diff --git a/backend/aci/server/tests/routes/agent/test_agent_chat.py b/backend/aci/server/tests/routes/agent/test_agent_chat.py new file mode 100644 index 000000000..4ddd10ced --- /dev/null +++ b/backend/aci/server/tests/routes/agent/test_agent_chat.py @@ -0,0 +1,531 @@ +import json +from collections.abc import AsyncGenerator +from unittest.mock import MagicMock, patch + +import pytest +from fastapi import status +from fastapi.testclient import TestClient + +from aci.common.db.sql_models import Agent +from aci.common.schemas.function import OpenAIResponsesFunctionDefinition +from aci.server import config +from aci.server.agent.prompt import ( + ClientMessage, + convert_to_openai_messages, + openai_chat_stream, +) +from aci.server.agent.types import ToolInvocation + + +# --------------------------------------------------------------------------- +# Standalone async generator used as a mock for openai_chat_stream +# --------------------------------------------------------------------------- + + +async def _mock_chat_stream_success( + messages, tools +) -> AsyncGenerator[str, None]: + """Mock async generator that simulates a successful chat stream.""" + yield f'0:{json.dumps("Hello")}\n' + yield f'0:{json.dumps(" from mock!")}\n' + yield 'd:{"finishReason":"stop","usage":{"promptTokens":10,"completionTokens":5}}\n' + + +# --------------------------------------------------------------------------- +# Unit tests: convert_to_openai_messages +# --------------------------------------------------------------------------- + + +class TestConvertToOpenaiMessages: + """Unit tests for convert_to_openai_messages.""" + + def test_converts_simple_user_message(self) -> None: + messages = [ + ClientMessage(role="user", content="Hello, how are you?"), + ] + result = convert_to_openai_messages(messages) + + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["type"] == "message" + assert result[0]["content"] == [ + {"type": "input_text", "text": "Hello, how are you?"} + ] + + def test_converts_assistant_message(self) -> None: + messages = [ + ClientMessage(role="assistant", content="I'm doing well, thank you!"), + ] + result = convert_to_openai_messages(messages) + + assert len(result) == 1 + assert result[0]["role"] == "assistant" + assert result[0]["content"] == [ + {"type": "output_text", "text": "I'm doing well, thank you!"} + ] + + def test_converts_multiple_messages(self) -> None: + messages = [ + ClientMessage(role="user", content="Hello"), + ClientMessage(role="assistant", content="Hi there!"), + ClientMessage(role="user", content="What can you do?"), + ] + result = convert_to_openai_messages(messages) + + assert len(result) == 3 + assert result[0]["role"] == "user" + assert result[1]["role"] == "assistant" + assert result[2]["role"] == "user" + + def test_converts_message_with_tool_invocations(self) -> None: + messages = [ + ClientMessage( + role="assistant", + content="Let me check that for you", + toolInvocations=[ + ToolInvocation( + toolCallId="call_123", + toolName="get_weather", + step=0, + state="completed", + args={"location": "New York"}, + result={"temperature": 72, "condition": "sunny"}, + ) + ], + ), + ] + result = convert_to_openai_messages(messages) + + # Should produce two messages: function_call and function_call_output + assert len(result) == 2 + assert result[0]["type"] == "function_call" + assert result[0]["call_id"] == "call_123" + assert result[0]["name"] == "get_weather" + assert result[0]["arguments"] == json.dumps({"location": "New York"}) + + assert result[1]["type"] == "function_call_output" + assert result[1]["call_id"] == "call_123" + assert result[1]["output"] == json.dumps( + {"temperature": 72, "condition": "sunny"} + ) + + def test_converts_message_with_tool_invocation_no_result(self) -> None: + """Tool invocation without a result still produces function_call but no output.""" + messages = [ + ClientMessage( + role="assistant", + content="Calling a function", + toolInvocations=[ + ToolInvocation( + toolCallId="call_456", + toolName="search", + step=0, + state="running", + args={"query": "test"}, + result=None, + ) + ], + ), + ] + result = convert_to_openai_messages(messages) + + # Only function_call (no output since result is None) + assert len(result) == 1 + assert result[0]["type"] == "function_call" + assert result[0]["call_id"] == "call_456" + assert result[0]["name"] == "search" + + def test_converts_mixed_messages_with_and_without_tools(self) -> None: + """Messages list with both regular and tool invocation messages.""" + messages = [ + ClientMessage(role="user", content="Search for something"), + ClientMessage( + role="assistant", + content="", + toolInvocations=[ + ToolInvocation( + toolCallId="call_789", + toolName="search", + step=0, + state="completed", + args={"query": "something"}, + result={"results": ["item1", "item2"]}, + ) + ], + ), + ClientMessage(role="user", content="Thanks!"), + ] + result = convert_to_openai_messages(messages) + + # user message, function_call, function_call_output, user message + assert len(result) == 4 + assert result[0]["role"] == "user" + assert result[1]["type"] == "function_call" + assert result[2]["type"] == "function_call_output" + assert result[3]["role"] == "user" + + def test_empty_messages_list(self) -> None: + """Empty messages list should return empty list.""" + result = convert_to_openai_messages([]) + assert result == [] + + +# --------------------------------------------------------------------------- +# Unit tests: openai_chat_stream +# --------------------------------------------------------------------------- + + +class TestOpenaiChatStream: + """Unit tests for openai_chat_stream.""" + + @pytest.mark.asyncio + async def test_streams_text_delta_events(self) -> None: + """Text delta events are properly yielded as SSE strings. + + NOTE: openai_chat_stream uses a nested for-loop that consumes the + first event in the outer loop (discarding it) and processes the + remaining events in the inner loop. + """ + # Dummy first event (discarded by the outer for-loop) + dummy_event = MagicMock() + dummy_event.type = "response.output_item.added" + dummy_event.output_index = 0 + dummy_event.item = MagicMock() + + text_event = MagicMock() + text_event.type = "response.output_text.delta" + text_event.delta = "Hello" + + text_event2 = MagicMock() + text_event2.type = "response.output_text.delta" + text_event2.delta = " world!" + + usage_mock = MagicMock() + usage_mock.prompt_tokens = 10 + usage_mock.completion_tokens = 5 + + complete_event = MagicMock() + complete_event.type = "response.completed" + complete_event.usage = usage_mock + + mock_stream = [dummy_event, text_event, text_event2, complete_event] + + mock_client = MagicMock() + mock_client.responses.create.return_value = mock_stream + + with patch( + "aci.server.agent.prompt.OpenAI", return_value=mock_client + ): + messages = [ + { + "role": "user", + "type": "message", + "content": [{"type": "input_text", "text": "Hi"}], + } + ] + tools: list[OpenAIResponsesFunctionDefinition] = [] + + results = [] + async for chunk in openai_chat_stream(messages, tools): + results.append(chunk) + + assert len(results) == 3 + assert results[0] == f'0:{json.dumps("Hello")}\n' + assert results[1] == f'0:{json.dumps(" world!")}\n' + assert '"finishReason":"stop"' in results[2] + assert '"promptTokens":10' in results[2] + assert '"completionTokens":5' in results[2] + + @pytest.mark.asyncio + async def test_streams_tool_call_events(self) -> None: + """Tool call events are properly yielded as SSE strings.""" + # Dummy first event (discarded by the outer for-loop) + dummy_event = MagicMock() + dummy_event.type = "response.output_text.delta" + dummy_event.delta = "" + + # Tool call events + added_event = MagicMock() + added_event.type = "response.output_item.added" + added_event.output_index = 0 + tool_item = MagicMock() + tool_item.call_id = "call_abc123" + tool_item.name = "get_weather" + tool_item.arguments = '{"loc' + added_event.item = tool_item + + args_delta_event = MagicMock() + args_delta_event.type = "response.function_call_arguments.delta" + args_delta_event.output_index = 0 + args_delta_event.delta = 'ation":' + + args_delta_event2 = MagicMock() + args_delta_event2.type = "response.function_call_arguments.delta" + args_delta_event2.output_index = 0 + args_delta_event2.delta = '"New York"}' + + args_done_event = MagicMock() + args_done_event.type = "response.function_call_arguments.done" + args_done_event.output_index = 0 + + usage_mock = MagicMock() + usage_mock.prompt_tokens = 15 + usage_mock.completion_tokens = 8 + + complete_event = MagicMock() + complete_event.type = "response.completed" + complete_event.usage = usage_mock + + mock_stream = [ + dummy_event, + added_event, + args_delta_event, + args_delta_event2, + args_done_event, + complete_event, + ] + + mock_client = MagicMock() + mock_client.responses.create.return_value = mock_stream + + with patch( + "aci.server.agent.prompt.OpenAI", return_value=mock_client + ): + messages = [ + { + "role": "user", + "type": "message", + "content": [ + {"type": "input_text", "text": "What's the weather?"} + ], + } + ] + tools: list[OpenAIResponsesFunctionDefinition] = [] + + results = [] + async for chunk in openai_chat_stream(messages, tools): + results.append(chunk) + + # Should have tool call + completion events + assert len(results) == 2 + assert results[0].startswith("9:") + tool_call_data = json.loads(results[0][2:]) # skip "9:" prefix + assert tool_call_data["toolCallId"] == "call_abc123" + assert tool_call_data["toolName"] == "get_weather" + assert tool_call_data["args"] == '{"location":"New York"}' + + assert '"finishReason":"tool-calls"' in results[1] + + @pytest.mark.asyncio + async def test_stream_handles_empty_stream(self) -> None: + """When all events are consumed by outer loop, inner loop yields nothing.""" + dummy_event = MagicMock() + dummy_event.type = "response.completed" + usage_mock = MagicMock() + usage_mock.prompt_tokens = 0 + usage_mock.completion_tokens = 0 + dummy_event.usage = usage_mock + + mock_stream = [dummy_event] + + mock_client = MagicMock() + mock_client.responses.create.return_value = mock_stream + + with patch( + "aci.server.agent.prompt.OpenAI", return_value=mock_client + ): + messages = [ + { + "role": "user", + "type": "message", + "content": [{"type": "input_text", "text": "Hi"}], + } + ] + tools: list[OpenAIResponsesFunctionDefinition] = [] + + results = [] + async for chunk in openai_chat_stream(messages, tools): + results.append(chunk) + + assert len(results) == 0 + + +# --------------------------------------------------------------------------- +# Integration tests: /chat endpoint +# --------------------------------------------------------------------------- + + +class TestChatEndpoint: + """Integration tests for the /chat endpoint.""" + + def test_chat_endpoint_returns_streaming_response( + self, + test_client: TestClient, + dummy_agent_1_with_all_apps_allowed: Agent, + ) -> None: + """Happy path: POST to /chat returns a streaming response with SSE data.""" + request_body = { + "id": "test-chat-1", + "linked_account_owner_id": "test-owner-1", + "selected_apps": [], + "selected_functions": ["ACI_TEST__HELLO_WORLD_NO_ARGS"], + "messages": [{"role": "user", "content": "Hello"}], + } + + api_key = dummy_agent_1_with_all_apps_allowed.api_keys[0].key + + with patch( + "aci.server.routes.agent.openai_chat_stream", + new=_mock_chat_stream_success, + ): + response = test_client.post( + f"{config.ROUTER_PREFIX_AGENT}/chat", + json=request_body, + headers={"x-api-key": api_key}, + ) + + assert response.status_code == status.HTTP_200_OK + assert response.headers["x-vercel-ai-data-stream"] == "v1" + + content = response.text + assert '0:"Hello"' in content + assert '0:" from mock!"' in content + assert '"finishReason":"stop"' in content + + def test_chat_endpoint_with_empty_messages( + self, + test_client: TestClient, + dummy_agent_1_with_all_apps_allowed: Agent, + ) -> None: + """Edge case: empty messages list should still succeed.""" + request_body = { + "id": "test-chat-2", + "linked_account_owner_id": "test-owner-2", + "selected_apps": [], + "selected_functions": [], + "messages": [], + } + + api_key = dummy_agent_1_with_all_apps_allowed.api_keys[0].key + + with patch( + "aci.server.routes.agent.openai_chat_stream", + new=_mock_chat_stream_success, + ): + response = test_client.post( + f"{config.ROUTER_PREFIX_AGENT}/chat", + json=request_body, + headers={"x-api-key": api_key}, + ) + + assert response.status_code == status.HTTP_200_OK + + def test_chat_endpoint_missing_api_key( + self, + test_client: TestClient, + ) -> None: + """Edge case: request without API key should be rejected.""" + request_body = { + "id": "test-chat-3", + "linked_account_owner_id": "test-owner-3", + "selected_apps": [], + "selected_functions": [], + "messages": [{"role": "user", "content": "Hello"}], + } + + response = test_client.post( + f"{config.ROUTER_PREFIX_AGENT}/chat", + json=request_body, + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_chat_endpoint_invalid_api_key( + self, + test_client: TestClient, + ) -> None: + """Edge case: request with non-existent API key should be rejected.""" + request_body = { + "id": "test-chat-4", + "linked_account_owner_id": "test-owner-4", + "selected_apps": [], + "selected_functions": [], + "messages": [{"role": "user", "content": "Hello"}], + } + + response = test_client.post( + f"{config.ROUTER_PREFIX_AGENT}/chat", + json=request_body, + headers={"x-api-key": "non_existent_key_abc123"}, + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + def test_chat_endpoint_missing_required_fields( + self, + test_client: TestClient, + dummy_agent_1_with_all_apps_allowed: Agent, + ) -> None: + """Edge case: request body missing required fields should return 422.""" + api_key = dummy_agent_1_with_all_apps_allowed.api_keys[0].key + + # Missing "messages" field + request_body = { + "id": "test-chat-5", + "linked_account_owner_id": "test-owner-5", + "selected_apps": [], + "selected_functions": [], + } + + response = test_client.post( + f"{config.ROUTER_PREFIX_AGENT}/chat", + json=request_body, + headers={"x-api-key": api_key}, + ) + + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY + + def test_chat_endpoint_with_tool_invocations_in_messages( + self, + test_client: TestClient, + dummy_agent_1_with_all_apps_allowed: Agent, + ) -> None: + """Messages with tool invocations should be properly accepted.""" + request_body = { + "id": "test-chat-6", + "linked_account_owner_id": "test-owner-6", + "selected_apps": [], + "selected_functions": ["ACI_TEST__HELLO_WORLD_NO_ARGS"], + "messages": [ + {"role": "user", "content": "Search for weather"}, + { + "role": "assistant", + "content": "Let me check", + "toolInvocations": [ + { + "toolCallId": "call_test", + "toolName": "get_weather", + "step": 0, + "state": "completed", + "args": {"location": "NYC"}, + "result": {"temp": 70}, + } + ], + }, + {"role": "user", "content": "Thanks"}, + ], + } + + api_key = dummy_agent_1_with_all_apps_allowed.api_keys[0].key + + with patch( + "aci.server.routes.agent.openai_chat_stream", + new=_mock_chat_stream_success, + ): + response = test_client.post( + f"{config.ROUTER_PREFIX_AGENT}/chat", + json=request_body, + headers={"x-api-key": api_key}, + ) + + assert response.status_code == status.HTTP_200_OK