diff --git a/agent/moa_loop.py b/agent/moa_loop.py index 9700f4abe85d..82080c1194f1 100644 --- a/agent/moa_loop.py +++ b/agent/moa_loop.py @@ -104,6 +104,16 @@ def __init__( "you should not try to or apologize for being unable to. A separate " "aggregator/orchestrator model holds those capabilities and will take the " "actual actions.\n\n" + "CRITICAL: You must NEVER claim or imply that you have executed a command, " + "downloaded a file, accessed a URL, or performed any action. You can only " + "analyze and advise based on the conversation context. Examples of what to " + "avoid:\n" + "- Bad: \"I ran curl and got 404.\"\n" + "- Bad: \"I downloaded the file successfully.\"\n" + "- Bad: \"I checked the repository and found...\"\n" + "- Good: \"Based on the error pattern, a curl request to that URL would likely return 404.\"\n" + "- Good: \"The conversation suggests downloading this file may help.\"\n" + "- Good: \"From the context, checking the repository would reveal...\"\n\n" "The conversation below is the current state of a task handled by that " "acting agent. Your job is to give your most intelligent analysis of that " "state: understand the goal, reason about the problem, and advise on what " @@ -114,7 +124,8 @@ def __init__( "asking for access.\n\n" "Respond with your advice directly — no preamble, no disclaimers about " "tools or access. Your response is private guidance handed to the " - "aggregator, not an answer shown to the user." + "aggregator, not an answer shown to the user. NEVER claim to have executed " + "anything." ) diff --git a/tests/agent/test_moa_reference_system_prompt.py b/tests/agent/test_moa_reference_system_prompt.py new file mode 100644 index 000000000000..89c7e9739eca --- /dev/null +++ b/tests/agent/test_moa_reference_system_prompt.py @@ -0,0 +1,73 @@ +""" +Test that the MoA reference system prompt contains explicit warnings +against claiming tool execution. + +Related issue: #61452 +""" + +from agent.moa_loop import _REFERENCE_SYSTEM_PROMPT + + +def test_reference_system_prompt_prohibits_claiming_execution(): + """ + Verify that the reference system prompt contains explicit warnings + against claiming tool execution. + + The prompt should: + 1. State that reference models cannot execute anything + 2. Warn against claiming/implying execution + 3. Provide bad/good examples + + This addresses #61452 where reference models were fabricating + tool execution in their text output. + """ + prompt_lower = _REFERENCE_SYSTEM_PROMPT.lower() + + # Critical constraints + assert "you cannot call tools" in prompt_lower or "you do not execute" in prompt_lower, \ + "Prompt must explicitly state that reference models cannot execute" + + assert "never claim" in prompt_lower or "never imply" in prompt_lower, \ + "Prompt must warn against claiming/implying execution" + + # Check for examples (helps models understand what NOT to do) + assert "bad:" in prompt_lower or "avoid:" in prompt_lower, \ + "Prompt should provide negative examples" + + # Specific action verbs that should NOT appear as claimed actions + # (these are common patterns of hallucinated execution) + forbidden_patterns = [ + "i ran", "i executed", "i downloaded", "i accessed", + "i checked", "i called", "i browsed" + ] + + # The prompt should mention these as bad examples + # (i.e., in the context of what to avoid, not as instruction) + has_any_forbidden = any( + f"bad: \"{pattern}" in _REFERENCE_SYSTEM_PROMPT.lower() or + f"avoid \"{pattern}" in _REFERENCE_SYSTEM_PROMPT.lower() + for pattern in forbidden_patterns + ) + + # At least one bad example pattern should exist + assert has_any_forbidden or "examples" in _REFERENCE_SYSTEM_PROMPT.lower(), \ + "Prompt should contain examples of what to avoid" + + +def test_reference_system_prompt_structure(): + """ + Verify the reference system prompt has a clear structure. + + A well-structured prompt helps models follow instructions better. + """ + # Prompt should not be empty + assert len(_REFERENCE_SYSTEM_PROMPT) > 100, \ + "Reference system prompt should be substantive" + + # Should have multiple paragraphs (structured guidance) + assert _REFERENCE_SYSTEM_PROMPT.count("\n\n") >= 2, \ + "Prompt should be structured with multiple sections" + + # Should contain the word "advisor" (defines role) + assert "advisor" in _REFERENCE_SYSTEM_PROMPT.lower(), \ + "Prompt should clearly define the advisor role" \ No newline at end of file