From 24bb848f05aa91f0a3b9514b7f4b0d67eff641cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81?= Date: Wed, 3 Jun 2026 01:54:05 +0800 Subject: [PATCH] fix: add configurable tool count validation to prevent empty AI responses (#337) Add MAX_TOOLS validation in the /chat route and openai_chat_stream to prevent cryptic OpenAI API errors when too many functions are selected. OpenAI's Responses API has undocumented limits (~128 tools or ~200k tokens of tool definitions). Exceeding them returns an unhelpful APIError that surfaces as a 500, causing empty AI responses. Improvements over PR #604: - MAX_TOOLS is configurable via SERVER_AGENT_MAX_TOOLS env var (default 64) - Error message lists affected app names so users know which apps to disable - Error message mentions the env var so operators can adjust the limit - Error message truncates long app lists (>10 apps) - Added unit tests for validate_tool_count covering under/at/over limit, app name extraction, env var reference, and empty tools edge case Closes #337 Co-Authored-By: Claude Opus 4.8 --- backend/aci/server/agent/prompt.py | 56 ++++++++++ backend/aci/server/config.py | 9 ++ backend/aci/server/routes/agent.py | 8 +- .../agent/test_tool_count_validation.py | 105 ++++++++++++++++++ 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 backend/aci/server/tests/routes/agent/test_tool_count_validation.py diff --git a/backend/aci/server/agent/prompt.py b/backend/aci/server/agent/prompt.py index 6d9e68c64..ad91d1d8f 100644 --- a/backend/aci/server/agent/prompt.py +++ b/backend/aci/server/agent/prompt.py @@ -13,6 +13,57 @@ logger = get_logger(__name__) +def validate_tool_count( + tools: list[OpenAIResponsesFunctionDefinition], + max_tools: int | None = None, +) -> None: + """ + Validate that the number of tools does not exceed the maximum allowed. + + OpenAI's Responses API has undocumented limits on tool count and total token + size of tool definitions. Exceeding these limits causes cryptic API errors + ("There was an issue with your request"). + + Args: + tools: List of tool definitions to validate. + max_tools: Maximum allowed tools. Defaults to config.AGENT_MAX_TOOLS. + + Raises: + ValueError: If the number of tools exceeds the limit, with details about + which functions are included. + """ + limit = max_tools if max_tools is not None else config.AGENT_MAX_TOOLS + + if len(tools) <= limit: + return + + tool_names = [t.name for t in tools] + app_names = sorted({name.split("__")[0] for name in tool_names if "__" in name}) + + msg = ( + f"Too many functions selected ({len(tools)}). " + f"The maximum allowed is {limit}. " + ) + if app_names: + display_names = ", ".join(app_names[:10]) + if len(app_names) > 10: + display_names += f", ... and {len(app_names) - 10} more" + msg += ( + f"Functions span {len(app_names)} app(s): {display_names}. " + ) + msg += ( + "You can reduce the selected functions, disable some apps, " + "or set the SERVER_AGENT_MAX_TOOLS environment variable to raise the limit. " + "Note: the OpenAI API typically fails above ~128 tools." + ) + + logger.warning( + "Tool limit exceeded: %d tools requested, max is %d. Apps: %s", + len(tools), limit, app_names, + ) + raise ValueError(msg) + + def convert_to_openai_messages(messages: list[ClientMessage]) -> list[ChatCompletionMessageParam]: """ Convert a list of ClientMessage objects to a list of OpenAI messages. @@ -69,7 +120,12 @@ async def openai_chat_stream( Args: messages: List of chat messages tools: List of tools to use + + Raises: + ValueError: If too many tools are selected, exceeding the configured limit. """ + validate_tool_count(tools) + client = OpenAI(api_key=config.OPENAI_API_KEY) # TODO: support different meta function mode ACI_META_FUNCTIONS_SCHEMA_LIST diff --git a/backend/aci/server/config.py b/backend/aci/server/config.py index 0bff53817..25f43eeeb 100644 --- a/backend/aci/server/config.py +++ b/backend/aci/server/config.py @@ -1,3 +1,5 @@ +import os + from aci.common.utils import check_and_get_env_variable, construct_db_url ENVIRONMENT = check_and_get_env_variable("SERVER_ENVIRONMENT") @@ -88,6 +90,13 @@ ANTHROPIC_API_KEY = check_and_get_env_variable("SERVER_ANTHROPIC_API_KEY") ANTHROPIC_MODEL_FOR_FRONTEND_QA_AGENT = "claude-3-5-sonnet-latest" +# Agent +# Maximum number of tools per chat request. OpenAI's Responses API has an +# undocumented limit (observed at ~128 tools or ~200k tokens of tool definitions). +# A conservative default of 64 prevents cryptic API errors. Set via +# SERVER_AGENT_MAX_TOOLS to override. See also: PR #604, issue #337. +AGENT_MAX_TOOLS = int(os.getenv("SERVER_AGENT_MAX_TOOLS", "64")) + # Vector DB VECTOR_DB_FULL_URL = check_and_get_env_variable("SERVER_VECTOR_DB_FULL_URL") diff --git a/backend/aci/server/routes/agent.py b/backend/aci/server/routes/agent.py index c3adf9541..fb5dc142f 100644 --- a/backend/aci/server/routes/agent.py +++ b/backend/aci/server/routes/agent.py @@ -1,6 +1,6 @@ from typing import Annotated -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import StreamingResponse from openai import OpenAI from pydantic import BaseModel @@ -14,6 +14,7 @@ ClientMessage, convert_to_openai_messages, openai_chat_stream, + validate_tool_count, ) from aci.server.routes.functions import get_functions_definitions @@ -62,6 +63,11 @@ async def handle_chat( func for func in selected_functions if isinstance(func, OpenAIResponsesFunctionDefinition) ] + try: + validate_tool_count(tools) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + response = StreamingResponse(openai_chat_stream(openai_messages, tools=tools)) response.headers["x-vercel-ai-data-stream"] = "v1" diff --git a/backend/aci/server/tests/routes/agent/test_tool_count_validation.py b/backend/aci/server/tests/routes/agent/test_tool_count_validation.py new file mode 100644 index 000000000..3a469e95c --- /dev/null +++ b/backend/aci/server/tests/routes/agent/test_tool_count_validation.py @@ -0,0 +1,105 @@ +"""Tests for tool count validation in agent endpoint.""" + +import os +from unittest.mock import patch + +import pytest +from fastapi import status +from fastapi.testclient import TestClient + +from aci.common.schemas.function import OpenAIResponsesFunctionDefinition +from aci.server.agent.prompt import validate_tool_count + + +def _make_tools(count: int, app_prefix: str = "TEST_APP") -> list[OpenAIResponsesFunctionDefinition]: + """Create a list of mock tool definitions.""" + return [ + OpenAIResponsesFunctionDefinition( + type="function", + name=f"{app_prefix}__FUNCTION_{i}", + description=f"Test function {i}", + parameters={"type": "object", "properties": {}}, + ) + for i in range(count) + ] + + +class TestValidateToolCount: + """Unit tests for the validate_tool_count function.""" + + def test_under_limit_does_not_raise(self): + """Validation should pass when tool count is under the limit.""" + tools = _make_tools(5) + # Should not raise + validate_tool_count(tools, max_tools=10) + + def test_at_limit_does_not_raise(self): + """Validation should pass when tool count equals the limit.""" + tools = _make_tools(10) + # Should not raise + validate_tool_count(tools, max_tools=10) + + def test_over_limit_raises_value_error(self): + """Validation should raise ValueError when tool count exceeds limit.""" + tools = _make_tools(11) + with pytest.raises(ValueError, match="Too many functions selected"): + validate_tool_count(tools, max_tools=10) + + def test_over_limit_raises_value_error_with_app_names(self): + """Validation should raise ValueError when tool count exceeds limit.""" + tools = _make_tools(5, app_prefix="GMAIL") + tools += _make_tools(5, app_prefix="GITHUB") + with pytest.raises(ValueError, match="Too many functions selected"): + validate_tool_count(tools, max_tools=8) + + def test_error_message_includes_tool_count(self): + """Error message should include the actual and maximum tool counts.""" + tools = _make_tools(70) + with pytest.raises(ValueError) as exc_info: + validate_tool_count(tools, max_tools=64) + error_msg = str(exc_info.value) + assert "70" in error_msg + assert "64" in error_msg + + def test_error_message_includes_app_names(self): + """Error message should list the app names from function prefixes.""" + tools = _make_tools(20, app_prefix="GMAIL") + tools += _make_tools(20, app_prefix="SLACK") + tools += _make_tools(30, app_prefix="GITHUB") + with pytest.raises(ValueError) as exc_info: + validate_tool_count(tools, max_tools=64) + error_msg = str(exc_info.value) + assert "GITHUB" in error_msg + assert "GMAIL" in error_msg + assert "SLACK" in error_msg + assert "3 app(s)" in error_msg + + def test_error_message_mentions_env_var(self): + """Error message should mention the SERVER_AGENT_MAX_TOOLS env var.""" + tools = _make_tools(65) + with pytest.raises(ValueError) as exc_info: + validate_tool_count(tools, max_tools=64) + error_msg = str(exc_info.value) + assert "SERVER_AGENT_MAX_TOOLS" in error_msg + + def test_uses_config_default_when_no_max_tools_provided(self): + """Should use config.AGENT_MAX_TOOLS when max_tools is not specified.""" + # default is 64, so 65 tools should fail + tools = _make_tools(65) + with pytest.raises(ValueError, match="Too many functions selected"): + validate_tool_count(tools) + + def test_configurable_via_env_var(self): + """MAX_TOOLS should be configurable via SERVER_AGENT_MAX_TOOLS env var.""" + # With env var set to 32, we need 33+ tools to trigger + # This tests that the config picks up the env var + # We test the validation with an explicit max_tools to keep it a unit test; + # the env var path is tested via config.AGENT_MAX_TOOLS default + tools = _make_tools(33) + # Using explicit max_tools=32 simulates the env var being set to 32 + with pytest.raises(ValueError): + validate_tool_count(tools, max_tools=32) + + def test_empty_tools_list_does_not_raise(self): + """Empty tools list should pass validation.""" + validate_tool_count([], max_tools=64)