WIP Integrate SWE agent into SWE-Bench

evaluation/benchmarks/swe_bench/run_infer.py

@@ -32,6 +32,7 @@
     codeact_user_response,
     codex_user_response,
     opencode_user_response,
+    swe_agent_user_response,
     get_default_sandbox_config_for_eval,
     get_metrics,
     get_openhands_config_for_eval,
@@ -108,6 +109,7 @@ def set_dataset_type(dataset_name: str) -> str:
     'CodeActAgent': codeact_user_response,
     'OpenCodeAgent': opencode_user_response,
     'CodexAgent': codex_user_response,
+    'SWEAgent': swe_agent_user_response,
 }
 
 

openhands/agenthub/__init__.py

@@ -11,6 +11,7 @@
     loc_agent,
     opencode_agent,
     readonly_agent,
+    swe_agent,
     visualbrowsing_agent,
 )
 from openhands.controller.agent import Agent  # noqa: E402
@@ -25,4 +26,5 @@
     'loc_agent',
     'opencode_agent',
     'codex_agent',
+    'swe_agent',
 ]

openhands/agenthub/swe_agent/__init__.py

@@ -0,0 +1,4 @@
+from openhands.agenthub.swe_agent.swe_agent import SWEAgent
+from openhands.controller.agent import Agent
+
+Agent.register('SWEAgent', SWEAgent)

openhands/agenthub/swe_agent/function_calling.py

@@ -0,0 +1,198 @@
+"""Function calling implementation for SWE-Agent.
+
+Maps SWE-Agent tool calls (bash, str_replace_editor, submit) to OpenHands actions.
+"""
+
+import json
+
+from litellm import ModelResponse
+
+from openhands.agenthub.codeact_agent.function_calling import combine_thought
+from openhands.agenthub.swe_agent.tools.bash import SWE_AGENT_BASH_TOOL_NAME
+from openhands.agenthub.swe_agent.tools.str_replace_editor import (
+    SWE_AGENT_STR_REPLACE_EDITOR_TOOL_NAME,
+    StrReplaceEditorTool,
+)
+from openhands.agenthub.swe_agent.tools.submit import SWE_AGENT_SUBMIT_TOOL_NAME
+from openhands.core.exceptions import (
+    FunctionCallNotExistsError,
+    FunctionCallValidationError,
+    LLMContextWindowExceedError,
+)
+from openhands.core.logger import openhands_logger as logger
+from openhands.events.action import (
+    Action,
+    AgentFinishAction,
+    CmdRunAction,
+    FileEditAction,
+    FileReadAction,
+    MessageAction,
+    ValidationFailureAction,
+)
+from openhands.events.action.mcp import MCPAction
+from openhands.events.event import FileEditSource, FileReadSource
+from openhands.events.tool import ToolCallMetadata
+
+
+def response_to_actions(
+    response: ModelResponse, mcp_tool_names: list[str] | None = None
+) -> list[Action]:
+    actions: list[Action] = []
+    assert len(response.choices) == 1
+    choice = response.choices[0]
+    assistant_msg = choice.message
+
+    has_content = assistant_msg.content is not None
+    has_tool_calls = hasattr(assistant_msg, 'tool_calls') and assistant_msg.tool_calls
+
+    if not has_content and not has_tool_calls:
+        raise LLMContextWindowExceedError(
+            'LLM returned empty response with no content and no tool calls.'
+        )
+
+    if hasattr(assistant_msg, 'tool_calls') and assistant_msg.tool_calls:
+        # Extract thought from content
+        thought = ''
+        if isinstance(assistant_msg.content, str):
+            thought = assistant_msg.content
+        elif isinstance(assistant_msg.content, list):
+            for msg in assistant_msg.content:
+                if msg['type'] == 'text':
+                    thought += msg['text']
+
+        for i, tool_call in enumerate(assistant_msg.tool_calls):
+            action: Action
+            logger.debug(f'SWE-Agent tool call: {tool_call}')
+
+            try:
+                try:
+                    arguments = json.loads(tool_call.function.arguments)
+                except json.decoder.JSONDecodeError as e:
+                    raise FunctionCallValidationError(
+                        f'Failed to parse tool call arguments: {tool_call.function.arguments}'
+                    ) from e
+
+                # ================================================
+                # Bash tool
+                # ================================================
+                if tool_call.function.name == SWE_AGENT_BASH_TOOL_NAME:
+                    if 'command' not in arguments:
+                        raise FunctionCallValidationError(
+                            f'Missing required argument "command" in tool call {tool_call.function.name}'
+                        )
+                    action = CmdRunAction(command=arguments['command'])
+
+                # ================================================
+                # str_replace_editor tool
+                # ================================================
+                elif tool_call.function.name == SWE_AGENT_STR_REPLACE_EDITOR_TOOL_NAME:
+                    if 'command' not in arguments:
+                        raise FunctionCallValidationError(
+                            f'Missing required argument "command" in tool call {tool_call.function.name}'
+                        )
+                    if 'path' not in arguments:
+                        raise FunctionCallValidationError(
+                            f'Missing required argument "path" in tool call {tool_call.function.name}'
+                        )
+
+                    path = arguments['path']
+                    command = arguments['command']
+                    other_kwargs = {
+                        k: v
+                        for k, v in arguments.items()
+                        if k not in ['command', 'path']
+                    }
+
+                    if command == 'view':
+                        action = FileReadAction(
+                            path=path,
+                            impl_source=FileReadSource.OH_ACI,
+                            view_range=other_kwargs.get('view_range', None),
+                        )
+                    else:
+                        if 'view_range' in other_kwargs:
+                            other_kwargs.pop('view_range')
+
+                        # Filter to valid str_replace_editor params
+                        valid_params = set(
+                            StrReplaceEditorTool['function']['parameters'][
+                                'properties'
+                            ].keys()
+                        )
+                        valid_kwargs_for_editor = {
+                            k: v
+                            for k, v in other_kwargs.items()
+                            if k in valid_params
+                        }
+
+                        action = FileEditAction(
+                            path=path,
+                            command=command,
+                            impl_source=FileEditSource.OH_ACI,
+                            **valid_kwargs_for_editor,
+                        )
+
+                # ================================================
+                # Submit tool
+                # ================================================
+                elif tool_call.function.name == SWE_AGENT_SUBMIT_TOOL_NAME:
+                    action = AgentFinishAction()
+
+                # ================================================
+                # MCP tools
+                # ================================================
+                elif mcp_tool_names and tool_call.function.name in mcp_tool_names:
+                    action = MCPAction(
+                        name=tool_call.function.name,
+                        arguments=arguments,
+                    )
+                else:
+                    raise FunctionCallNotExistsError(
+                        f'Tool {tool_call.function.name} is not registered. '
+                        f'(arguments: {arguments}). '
+                        f'Please check the tool name and retry with an existing tool.'
+                    )
+
+            except FunctionCallValidationError as e:
+                action = ValidationFailureAction(
+                    function_name=tool_call.function.name,
+                    error_message=str(e),
+                    thought=thought if i == 0 else '',
+                )
+
+            except FunctionCallNotExistsError as e:
+                action = MessageAction(
+                    content=str(e),
+                    wait_for_response=False,
+                )
+
+            # Add thought to first action
+            if i == 0 and not isinstance(
+                action, (ValidationFailureAction, MessageAction)
+            ):
+                action = combine_thought(action, thought)
+
+            # Add metadata for tool calling
+            action.tool_call_metadata = ToolCallMetadata(
+                tool_call_id=tool_call.id,
+                function_name=tool_call.function.name,
+                model_response=response,
+                total_calls_in_response=len(assistant_msg.tool_calls),
+            )
+            actions.append(action)
+    else:
+        message_action = MessageAction(
+            content=str(assistant_msg.content) if assistant_msg.content else '',
+            wait_for_response=True,
+        )
+        message_action.tool_call_metadata = ToolCallMetadata(
+            model_response=response,
+            total_calls_in_response=0,
+        )
+        actions.append(message_action)
+
+    for action in actions:
+        action.response_id = response.id
+
+    assert len(actions) >= 1
+    return actions

openhands/agenthub/swe_agent/prompts/additional_info.j2

@@ -0,0 +1,53 @@
+{% if repository_info %}
+<REPOSITORY_INFO>
+At the user's request, repository {{ repository_info.repo_name }} has been cloned to {{ repository_info.repo_directory }} in the current working directory.
+{% if repository_info.branch_name %}The repository has been checked out to branch "{{ repository_info.branch_name }}".
+
+IMPORTANT: You should work within the current branch "{{ repository_info.branch_name }}" unless:
+    1. the user explicitly instructs otherwise
+    2. the current branch is "main", "master", or another default branch where direct pushes may be unsafe
+{% endif %}
+</REPOSITORY_INFO>
+{% endif %}
+{% if repository_instructions -%}
+<REPOSITORY_INSTRUCTIONS>
+{{ repository_instructions }}
+</REPOSITORY_INSTRUCTIONS>
+{% endif %}
+{% if runtime_info -%}
+<RUNTIME_INFORMATION>
+{% if runtime_info.working_dir %}
+The current working directory is {{ runtime_info.working_dir }}
+{% endif %}
+{% if runtime_info.available_hosts %}
+The user has access to the following hosts for accessing a web application,
+each of which has a corresponding port:
+{% for host, port in runtime_info.available_hosts.items() -%}
+* {{ host }} (port {{ port }})
+{% endfor %}
+When starting a web server, use the corresponding ports. You should also
+set any options to allow iframes and CORS requests, and allow the server to
+be accessed from any host (e.g. 0.0.0.0).
+For example, if you are using vite.config.js, you should set server.host and server.allowedHosts to true
+{% endif %}
+{% if runtime_info.additional_agent_instructions %}
+{{ runtime_info.additional_agent_instructions }}
+{% endif %}
+{% if runtime_info.custom_secrets_descriptions %}
+<CUSTOM_SECRETS>
+You have access to the following environment variables
+{% for secret_name, secret_description in runtime_info.custom_secrets_descriptions.items() %}
+* **${{ secret_name }}**: {{ secret_description }}
+{% endfor %}
+</CUSTOM_SECRETS>
+{% endif %}
+{% if runtime_info.date %}
+Today's date is {{ runtime_info.date }} (UTC).
+{% endif %}
+</RUNTIME_INFORMATION>
+{% if conversation_instructions and conversation_instructions.content -%}
+<CONVERSATION_INSTRUCTIONS>
+{{ conversation_instructions.content }}
+</CONVERSATION_INSTRUCTIONS>
+{% endif %}
+{% endif %}

openhands/agenthub/swe_agent/prompts/microagent_info.j2

@@ -0,0 +1,8 @@
+{% for agent_info in triggered_agents %}
+<EXTRA_INFO>
+The following information has been included based on a keyword match for "{{ agent_info.trigger }}".
+It may or may not be relevant to the user's request.
+
+{{ agent_info.content }}
+</EXTRA_INFO>
+{% endfor %}

openhands/agenthub/swe_agent/prompts/system_prompt.j2

@@ -0,0 +1,4 @@
+You are a helpful assistant that can interact with a computer to solve tasks.
+
+{% include 'additional_info.j2' %}
+{% include 'microagent_info.j2' %}

openhands/agenthub/swe_agent/prompts/system_prompt_long_horizon.j2

@@ -0,0 +1 @@
+{% include "system_prompt.j2" %}

openhands/agenthub/swe_agent/prompts/user_prompt.j2

@@ -0,0 +1 @@
+