From 90b9bb6cb1a098c9bdefe252a8b3be574cae2b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=BF=97=E8=BF=9C?= Date: Fri, 7 Aug 2026 18:18:29 +0800 Subject: [PATCH] feat(llm): LLM module protocol optimization --- openjiuwen/core/foundation/llm/__init__.py | 18 +- .../llm/inference_affinity_model.py | 10 +- openjiuwen/core/foundation/llm/model.py | 15 +- .../foundation/llm/model_clients/__init__.py | 57 +- .../model_clients/anthropic_model_client.py | 397 +++++- .../ascend_affinity_model_client.py | 1113 ---------------- .../llm/model_clients/base_model_client.py | 8 +- .../model_clients/dashscope_model_client.py | 579 --------- .../model_clients/deepseek_model_client.py | 83 -- .../inference_affinity_model_client.py | 911 -------------- .../llm/model_clients/openai_model_client.py | 1114 ++++++++++++++++- .../model_clients/openrouter_model_client.py | 337 ----- .../model_clients/siliconflow_model_client.py | 727 ----------- .../core/foundation/llm/schema/config.py | 55 +- .../foundation/llm/schema/message_chunk.py | 1 + .../foundation/llm/utils/endpoint_profiles.py | 178 +++ .../core/single_agent/agents/react_agent.py | 2 + .../test_kv_cache_outbound_payload.py | 21 +- ...st_kv_cache_swarmflow_stateful_outbound.py | 28 +- ...finity_kv_cache_release_with_processors.py | 12 +- .../llm/test_anthropic_client_pooling.py | 16 + .../llm/test_anthropic_model_client.py | 274 +++- .../llm/test_ascend_affinity_model_client.py | 305 ----- .../core/foundation/llm/test_message_chunk.py | 16 + .../llm/test_model_client_config.py | 50 +- .../llm/test_model_client_profile_routing.py | 177 +++ .../llm/test_model_client_tracer.py | 462 ------- .../llm/test_openai_model_client.py | 200 +++ .../llm/test_openrouter_model_client.py | 855 ------------- 29 files changed, 2522 insertions(+), 5499 deletions(-) delete mode 100644 openjiuwen/core/foundation/llm/model_clients/ascend_affinity_model_client.py delete mode 100644 openjiuwen/core/foundation/llm/model_clients/dashscope_model_client.py delete mode 100644 openjiuwen/core/foundation/llm/model_clients/deepseek_model_client.py delete mode 100644 openjiuwen/core/foundation/llm/model_clients/inference_affinity_model_client.py delete mode 100644 openjiuwen/core/foundation/llm/model_clients/openrouter_model_client.py delete mode 100644 openjiuwen/core/foundation/llm/model_clients/siliconflow_model_client.py create mode 100644 openjiuwen/core/foundation/llm/utils/endpoint_profiles.py delete mode 100644 tests/unit_tests/core/foundation/llm/test_ascend_affinity_model_client.py create mode 100644 tests/unit_tests/core/foundation/llm/test_model_client_profile_routing.py delete mode 100644 tests/unit_tests/core/foundation/llm/test_model_client_tracer.py delete mode 100644 tests/unit_tests/core/foundation/llm/test_openrouter_model_client.py diff --git a/openjiuwen/core/foundation/llm/__init__.py b/openjiuwen/core/foundation/llm/__init__.py index 3ab0af2b2..703fe6cd4 100644 --- a/openjiuwen/core/foundation/llm/__init__.py +++ b/openjiuwen/core/foundation/llm/__init__.py @@ -7,7 +7,15 @@ from openjiuwen.core.foundation.llm.output_parsers.output_parser import BaseOutputParser # Configuration -from openjiuwen.core.foundation.llm.schema.config import ModelRequestConfig, ModelClientConfig, ProviderType +from openjiuwen.core.foundation.llm.schema.config import ( + LLMApiMode, + LLMAuthMode, + LLMExtensionsConfig, + KVCacheExtensionConfig, + ModelRequestConfig, + ModelClientConfig, + ProviderType, +) from openjiuwen.core.foundation.llm.schema.mode_info import BaseModelInfo, ModelConfig # Messages from openjiuwen.core.foundation.llm.schema.message import ( @@ -26,8 +34,8 @@ from openjiuwen.core.foundation.llm.schema.tool_call import ToolCall # Built-in implementations +from openjiuwen.core.foundation.llm.model_clients.anthropic_model_client import AnthropicModelClient from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient -from openjiuwen.core.foundation.llm.model_clients.ascend_affinity_model_client import AscendAffinityModelClient from openjiuwen.core.foundation.llm.output_parsers.json_output_parser import JsonOutputParser from openjiuwen.core.foundation.llm.output_parsers.markdown_output_parser import MarkdownOutputParser @@ -47,6 +55,10 @@ "ModelRequestConfig", "ModelClientConfig", "ProviderType", + "LLMApiMode", + "LLMAuthMode", + "LLMExtensionsConfig", + "KVCacheExtensionConfig", "BaseModelInfo", "ModelConfig" ] @@ -73,8 +85,8 @@ # Built-in ModelClient implementations _PREBUILT_MODEL_CLIENTS = [ + "AnthropicModelClient", "OpenAIModelClient", - "AscendAffinityModelClient", ] # Built-in OutputParser implementations diff --git a/openjiuwen/core/foundation/llm/inference_affinity_model.py b/openjiuwen/core/foundation/llm/inference_affinity_model.py index 27301b4a2..645735b28 100644 --- a/openjiuwen/core/foundation/llm/inference_affinity_model.py +++ b/openjiuwen/core/foundation/llm/inference_affinity_model.py @@ -9,14 +9,14 @@ from openjiuwen.core.foundation.tool import ToolInfo from openjiuwen.core.foundation.llm.schema.config import ModelRequestConfig, ModelClientConfig from openjiuwen.core.foundation.llm.output_parsers.output_parser import BaseOutputParser -from openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client import InferenceAffinityModelClient +from openjiuwen.core.foundation.llm.model_clients import create_model_client class InferenceAffinityModel: """InferenceAffinity (vLLM) model unified invocation entry point Responsibilities: - 1. Manage InferenceAffinityModelClient instances + 1. Manage OpenAI-compatible clients configured for KV cache release 2. Provide unified asynchronous interfaces (ainvoke, astream) 3. Support release functionality @@ -38,10 +38,10 @@ def __init__( """ self.model_config = model_config self.model_client_config = model_client_config - self._client: Optional[InferenceAffinityModelClient] = None + self._client = None if model_client_config is not None: - self._client = InferenceAffinityModelClient(model_config, model_client_config) + self._client = create_model_client(model_client_config, model_config) else: raise build_error(StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg="model client config is none") @@ -187,4 +187,4 @@ def build_kv_cache_invoke_kwargs( extra["session_id"] = session.get_session_id() if enable_kv_cache_release: extra["enable_cache_sharing"] = True - return extra \ No newline at end of file + return extra diff --git a/openjiuwen/core/foundation/llm/model.py b/openjiuwen/core/foundation/llm/model.py index 5e76ef1ce..a952c9360 100644 --- a/openjiuwen/core/foundation/llm/model.py +++ b/openjiuwen/core/foundation/llm/model.py @@ -20,7 +20,6 @@ VideoGenerationResponse ) from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient -from openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client import InferenceAffinityModelClient from openjiuwen.core.runner.callback import trigger @@ -303,15 +302,13 @@ def build_kv_cache_invoke_kwargs( - session_id: use session.get_session_id() if provided - enable_cache_sharing: follow enable_kv_cache_release """ - if not isinstance(self._client, InferenceAffinityModelClient): + build_fn = getattr(self._client, "build_kv_cache_invoke_kwargs", None) + if not callable(build_fn): return {} - - extra: dict = {} - if session is not None and hasattr(session, "get_session_id"): - extra["session_id"] = session.get_session_id() - if enable_kv_cache_release: - extra["enable_cache_sharing"] = True - return extra + return build_fn( + session=session, + enable_kv_cache_release=enable_kv_cache_release, + ) def build_kv_cache_affinity_invoke_kwargs( self, diff --git a/openjiuwen/core/foundation/llm/model_clients/__init__.py b/openjiuwen/core/foundation/llm/model_clients/__init__.py index 314d94142..55a81eedd 100644 --- a/openjiuwen/core/foundation/llm/model_clients/__init__.py +++ b/openjiuwen/core/foundation/llm/model_clients/__init__.py @@ -4,7 +4,28 @@ from openjiuwen.core.common.exception.codes import StatusCode from openjiuwen.core.common.exception.errors import build_error from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelRequestConfig, ModelClientConfig, ProviderType +from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, + ModelRequestConfig, + ModelClientConfig, + ProviderType, +) +from openjiuwen.core.foundation.llm.utils.endpoint_profiles import normalize_model_client_config + + +def _value(value): + return value.value if hasattr(value, "value") else value + + +def _implementation_provider(client_config: ModelClientConfig) -> str: + provider = _value(client_config.client_provider) + if provider != ProviderType.OpenAI.value: + return provider + + auth_mode = _value(getattr(client_config, "auth_mode", LLMAuthMode.ApiKey.value)) + if auth_mode == LLMAuthMode.OpenAIAccountOAuth.value: + return ProviderType.OpenAIAccount.value + return ProviderType.OpenAI.value def _builtin_model_client(provider, client_config: ModelClientConfig, model_config: ModelRequestConfig): @@ -18,39 +39,10 @@ def _builtin_model_client(provider, client_config: ModelClientConfig, model_conf from openjiuwen.core.foundation.llm.model_clients.openai_account_model_client import OpenAIAccountModelClient return OpenAIAccountModelClient(model_config=model_config, model_client_config=client_config) - if provider == ProviderType.OpenRouter.value: - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import OpenRouterModelClient - return OpenRouterModelClient(model_config=model_config, model_client_config=client_config) - if provider == ProviderType.Anthropic.value: from openjiuwen.core.foundation.llm.model_clients.anthropic_model_client import AnthropicModelClient return AnthropicModelClient(model_config=model_config, model_client_config=client_config) - if provider == ProviderType.SiliconFlow.value: - from openjiuwen.core.foundation.llm.model_clients.siliconflow_model_client import \ - SiliconFlowModelClient - return SiliconFlowModelClient(model_config=model_config, model_client_config=client_config) - - if provider == ProviderType.DashScope.value: - from openjiuwen.core.foundation.llm.model_clients.dashscope_model_client import \ - DashScopeModelClient - return DashScopeModelClient(model_config=model_config, model_client_config=client_config) - - if provider == ProviderType.InferenceAffinity.value: - from openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client import \ - InferenceAffinityModelClient - return InferenceAffinityModelClient(model_config=model_config, model_client_config=client_config) - - if provider == ProviderType.AscendAffinity.value: - from openjiuwen.core.foundation.llm.model_clients.ascend_affinity_model_client import \ - AscendAffinityModelClient - return AscendAffinityModelClient(model_config=model_config, model_client_config=client_config) - - if provider == ProviderType.DeepSeek.value: - from openjiuwen.core.foundation.llm.model_clients.deepseek_model_client import \ - DeepSeekModelClient - return DeepSeekModelClient(model_config=model_config, model_client_config=client_config) - if provider == ProviderType.IntelliRouter.value: from openjiuwen.core.foundation.llm.model_clients.intelli_router_model_client import \ IntelliRouterModelClient @@ -78,7 +70,10 @@ def create_model_client(client_config: ModelClientConfig, model_config: ModelReq error_msg="model client config client_id is none") provider = client_config.client_provider.value if isinstance(client_config.client_provider, ProviderType)\ else client_config.client_provider - client = _builtin_model_client(provider, client_config, model_config) + normalized_config = normalize_model_client_config(client_config) + dispatch_provider = _implementation_provider(normalized_config) + dispatch_config = normalized_config + client = _builtin_model_client(dispatch_provider, dispatch_config, model_config) if client is not None: return client try: diff --git a/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py b/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py index 96aaba00d..74839653f 100644 --- a/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py +++ b/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py @@ -16,7 +16,21 @@ 3. messages """ -from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterable, List, Mapping, Optional, Tuple, Union +import copy +import json +import re +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Dict, + Iterable, + List, + Mapping, + Optional, + Tuple, + Union, +) import httpx @@ -38,6 +52,7 @@ VideoGenerationResponse, ) from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, ModelClientConfig, ModelRequestConfig, ProviderType, @@ -58,6 +73,160 @@ import anthropic +_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY = "anthropic_content_blocks" +_ANTHROPIC_INTERNAL_CONTENT_BLOCKS_KEY = "__anthropic_content_blocks" +_ANTHROPIC_CACHEABLE_BLOCK_TYPES = frozenset({ + "text", "image", "document", "tool_use", "tool_result", +}) +_IMAGE_DATA_URL_PATTERN = re.compile( + r"^data:(image/(?:jpeg|png|gif|webp));base64,(.+)$", + flags=re.IGNORECASE | re.DOTALL, +) + + +def _to_plain_data(value: Any) -> Any: + """Convert Anthropic SDK models into JSON-compatible data.""" + if hasattr(value, "model_dump"): + return value.model_dump(exclude_none=True) + if isinstance(value, Mapping): + return {key: _to_plain_data(item) for key, item in value.items()} + if isinstance(value, list): + return [_to_plain_data(item) for item in value] + return value + + +def _sanitize_replay_block(block: Any) -> Optional[dict]: + """Keep only fields accepted by Anthropic's message input schema.""" + data = _to_plain_data(block) + if not isinstance(data, Mapping): + return None + + block_type = data.get("type") + if block_type == "thinking": + return { + "type": "thinking", + "thinking": str(data.get("thinking") or ""), + "signature": str(data.get("signature") or ""), + } + if block_type == "redacted_thinking": + opaque_data = data.get("data") + if not opaque_data: + return None + return {"type": "redacted_thinking", "data": opaque_data} + if block_type == "text": + result = {"type": "text", "text": str(data.get("text") or "")} + if data.get("citations") is not None: + result["citations"] = copy.deepcopy(data["citations"]) + return result + if block_type == "tool_use": + return { + "type": "tool_use", + "id": str(data.get("id") or ""), + "name": str(data.get("name") or ""), + "input": copy.deepcopy(data.get("input") or {}), + } + return None + + +def _preserved_content_blocks(message: Mapping[str, Any]) -> List[dict]: + raw_blocks = message.get(_ANTHROPIC_INTERNAL_CONTENT_BLOCKS_KEY) + if raw_blocks is None: + metadata = message.get("metadata") + if isinstance(metadata, Mapping): + raw_blocks = metadata.get(_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY) + if not isinstance(raw_blocks, list): + return [] + blocks: List[dict] = [] + for block in raw_blocks: + sanitized = _sanitize_replay_block(block) + if sanitized is not None: + blocks.append(sanitized) + return blocks + + +def _copy_preserved_blocks_to_converted_messages( + source_messages: Union[str, List[BaseMessage], List[dict]], + converted_messages: List[dict], +) -> None: + """Restore provider-private metadata stripped by the common converter.""" + if not isinstance(source_messages, list) or len(source_messages) != len(converted_messages): + return + for source, converted in zip(source_messages, converted_messages): + metadata = source.get("metadata") if isinstance(source, dict) else getattr(source, "metadata", None) + if not isinstance(metadata, Mapping): + continue + raw_blocks = metadata.get(_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY) + if isinstance(raw_blocks, list): + converted[_ANTHROPIC_INTERNAL_CONTENT_BLOCKS_KEY] = copy.deepcopy(raw_blocks) + + +def _stream_blocks_metadata(block_acc: Mapping[int, dict]) -> dict[str, Any]: + blocks = [] + for _, block in sorted(block_acc.items()): + sanitized = _sanitize_replay_block(block) + if sanitized is not None: + blocks.append(sanitized) + if not blocks: + return {} + return {_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY: blocks} + + +def _convert_tool_choice(tool_choice: Any) -> Optional[dict]: + """Translate common/OpenAI tool-choice shapes into Anthropic's shape.""" + if tool_choice is None or tool_choice == "auto": + return None + if isinstance(tool_choice, str): + mapped = {"required": "any", "none": "none", "any": "any"}.get(tool_choice) + return {"type": mapped} if mapped else None + if not isinstance(tool_choice, Mapping): + return None + if tool_choice.get("type") == "function": + function = tool_choice.get("function") or {} + return {"type": "tool", "name": function.get("name", "")} + if tool_choice.get("type") in {"auto", "any", "none", "tool"}: + return copy.deepcopy(dict(tool_choice)) + return None + + +def _model_forbids_custom_sampling(model: str) -> bool: + """Return whether a current Claude family accepts only default sampling.""" + normalized = str(model or "").lower().replace(".", "-") + if not normalized.startswith("claude-"): + return False + if any(name in normalized for name in ("fable", "mythos")): + return True + if re.search(r"claude-(?:opus|sonnet)-5(?:-|$)", normalized): + return True + return bool(re.search(r"claude-opus-4-(?:7|8)(?:-|$)", normalized)) + + +def _image_source_block(value: Any) -> dict: + """Convert an OpenAI-style image value to an Anthropic image block.""" + if isinstance(value, Mapping): + value = value.get("url") + if not isinstance(value, str) or not value: + raise ValueError("Anthropic image input requires a non-empty image URL.") + + data_url = _IMAGE_DATA_URL_PATTERN.fullmatch(value) + if data_url: + return { + "type": "image", + "source": { + "type": "base64", + "media_type": data_url.group(1).lower(), + "data": data_url.group(2), + }, + } + if value.startswith("data:"): + raise ValueError( + "Anthropic image inputs support base64 JPEG, PNG, GIF, or WebP data URLs only." + ) + return { + "type": "image", + "source": {"type": "url", "url": value}, + } + + # --------------------------------------------------------------------------- # Shape converters: openJiuwen BaseMessage list <-> Anthropic Messages API payload # --------------------------------------------------------------------------- @@ -72,6 +241,15 @@ def _content_to_blocks(content: Any) -> List[dict]: blocks2: List[dict] = [] for item in content: if isinstance(item, dict): + item_type = item.get("type") + image_value = None + if item_type in {"image_url", "input_image"}: + image_value = item.get("image_url") + elif item_type == "image" and "source" not in item: + image_value = item.get("data_url", item.get("image")) + if image_value is not None or item_type in {"image_url", "input_image"}: + blocks2.append(_image_source_block(image_value)) + continue blocks2.append(dict(item)) elif isinstance(item, str): if item: @@ -88,12 +266,13 @@ def _mark_cache_control(blocks: List[dict], ttl: str) -> None: Anthropic caches the prefix up to and including the marked block; only the final block in a message needs the marker to anchor the prefix there. """ - if not blocks: - return marker: dict = {"type": "ephemeral"} if ttl == "1h": marker["ttl"] = "1h" - blocks[-1]["cache_control"] = marker + for block in reversed(blocks): + if block.get("type") in _ANTHROPIC_CACHEABLE_BLOCK_TYPES: + block["cache_control"] = marker + return def _convert_message_schemas( @@ -140,13 +319,17 @@ def _flush_tool_results(): _flush_tool_results() if role == "assistant": + preserved_blocks = _preserved_content_blocks(msg) + if preserved_blocks: + out.append({"role": "assistant", "content": preserved_blocks}) + continue + blocks = _content_to_blocks(msg.get("content", "")) tool_calls = msg.get("tool_calls") or [] for tc in tool_calls: fn = tc.get("function", {}) if isinstance(tc, dict) else {} args_str = fn.get("arguments", "{}") or "{}" try: - import json args_obj = json.loads(args_str) if isinstance(args_str, str) else args_str except Exception: args_obj = {"_raw_arguments": args_str} @@ -261,6 +444,14 @@ def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelC super().__init__(model_config, model_client_config) self._base_headers = build_base_headers(custom_headers=model_client_config.custom_headers) + def _validate_config(self): + super()._validate_config() + if not str(self.model_client_config.api_key or "").strip(): + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg="model client config api_key is required for Anthropic client.", + ) + def _get_client_name(self) -> str: return "Anthropic client" @@ -284,7 +475,9 @@ def connection_key(cls, model_client_config: ModelClientConfig) -> Tuple: :meth:`aclose_connections`. """ cfg = model_client_config + auth_mode = cfg.auth_mode.value if isinstance(cfg.auth_mode, LLMAuthMode) else cfg.auth_mode return ( + auth_mode, cfg.api_key, cls._normalize_base_url(cfg.api_base), cfg.verify_ssl, @@ -315,7 +508,9 @@ def _normalize_base_url(api_base: Optional[str]) -> Optional[str]: if not api_base: return None b = api_base.rstrip("/") - if b.endswith("/v1"): + if b.endswith("/v1/messages"): + b = b[:-12] + elif b.endswith("/v1"): b = b[:-3] return b or None @@ -462,6 +657,7 @@ def _build_anthropic_params( oai_messages: List[dict] = openai_params.get("messages") or [] oai_tools: Optional[List[dict]] = openai_params.get("tools") + _copy_preserved_blocks_to_converted_messages(messages, oai_messages) system_blocks, anthropic_messages = _convert_message_schemas(oai_messages) anthropic_tools = _convert_tool_schemas(oai_tools) @@ -483,20 +679,61 @@ def _build_anthropic_params( params["system"] = system_blocks if anthropic_tools: params["tools"] = anthropic_tools - # Anthropic rejects temperature and top_p together (400 invalid_request - # for models such as Claude Haiku 4.5). Both carry non-None defaults in - # ModelRequestConfig, so we cannot rely on the caller to unset one: - # prefer temperature and only forward top_p when temperature is absent. + + # Forward Anthropic-native controls that the common OpenAI-shaped + # builder intentionally treats as extras. + for key in ( + "thinking", "output_config", "metadata", "service_tier", "top_k", + ): + if key in openai_params: + params[key] = openai_params[key] + + # Compatible gateways may add fields that are not keyword parameters + # in the Anthropic SDK. Send them through extra_body so the SDK merges + # them into JSON instead of raising TypeError before the HTTP request. + extra_body = copy.deepcopy(openai_params.get("extra_body") or {}) + for key in ( + "reasoning_effort", "thinking_budget", "thinking_strategy", + "enable_thinking", + ): + if key in openai_params: + extra_body[key] = openai_params[key] + if extra_body: + params["extra_body"] = extra_body + anthropic_tool_choice = _convert_tool_choice(openai_params.get("tool_choice")) + if anthropic_tool_choice is not None: + params["tool_choice"] = anthropic_tool_choice + + # ModelRequestConfig carries OpenAI-oriented defaults. Treat those as + # "unset" for Anthropic unless the caller explicitly configured them; + # current Claude families reject non-default sampling with HTTP 400. + temperature_explicit = temperature is not None or "temperature" in self.model_config.model_fields_set + top_p_explicit = top_p is not None or "top_p" in self.model_config.model_fields_set temperature = openai_params.get("temperature") top_p = openai_params.get("top_p") - if temperature is not None: + thinking_payload = params.get("thinking") + thinking_type = ( + (thinking_payload or {}).get("type") + if isinstance(thinking_payload, Mapping) + else None + ) + sampling_forbidden = _model_forbids_custom_sampling(params["model"]) + thinking_restricts_sampling = thinking_type in {"enabled", "adaptive"} + + if sampling_forbidden or thinking_restricts_sampling: + if temperature_explicit or top_p_explicit: + llm_logger.debug( + "Anthropic: dropping sampling overrides that are incompatible " + "with this model/thinking mode." + ) + elif temperature_explicit and temperature is not None: params["temperature"] = temperature - if top_p is not None: + if top_p_explicit and top_p is not None: llm_logger.debug( "Anthropic: dropping top_p because temperature is set " "(the API forbids specifying both)." ) - elif top_p is not None and top_p != 1.0: + elif top_p_explicit and top_p is not None and top_p != 1.0: # top_p=1.0 is the default (no nucleus truncation); skip it so we # send the API only meaningful overrides. params["top_p"] = top_p @@ -679,7 +916,10 @@ async def stream( # Accumulator state across the stream current_text = "" - tool_use_acc: dict[int, dict] = {} # index -> {id, name, args_str} + # Ordered content-block state. Besides assembling tool arguments it + # retains thinking signatures so the next agent iteration can replay + # the assistant turn exactly as Anthropic returned it. + tool_use_acc: dict[int, dict] = {} last_usage: Optional[UsageMetadata] = None final_stop_reason: Optional[str] = None @@ -750,13 +990,23 @@ async def _parse_response( """Convert an Anthropic ``Message`` response into ``AssistantMessage``.""" content_blocks = list(getattr(response, "content", []) or []) text_parts: List[str] = [] + reasoning_parts: List[str] = [] + replay_blocks: List[dict] = [] tool_calls: List[ToolCall] = [] for idx, block in enumerate(content_blocks): btype = getattr(block, "type", None) + replay_block = _sanitize_replay_block(block) + if replay_block is not None: + replay_blocks.append(replay_block) if btype == "text": text_parts.append(getattr(block, "text", "") or "") + elif btype == "thinking": + reasoning_parts.append(getattr(block, "thinking", "") or "") + elif btype == "redacted_thinking": + # The encrypted data is retained in metadata for replay, but it + # has no displayable reasoning text. + continue elif btype == "tool_use": - import json input_obj = getattr(block, "input", None) or {} args_str = json.dumps(input_obj) if not isinstance(input_obj, str) else input_obj tool_calls.append(ToolCall( @@ -766,10 +1016,20 @@ async def _parse_response( arguments=args_str, index=idx, )) - # thinking / redacted_thinking blocks: ignored for now -- could be - # surfaced as reasoning_content if/when OJ wants to display them. content = "".join(text_parts) + reasoning_content = "".join(reasoning_parts) or None + if reasoning_content is None: + # A few Anthropic-compatible gateways expose an OpenAI-style extra + # response field. It is safe to normalize for display, but it must + # never replace signed Anthropic thinking blocks during replay. + extra_reasoning = getattr(response, "reasoning_content", None) + if extra_reasoning is None: + model_extra = getattr(response, "model_extra", None) + if isinstance(model_extra, Mapping): + extra_reasoning = model_extra.get("reasoning_content") + if isinstance(extra_reasoning, str) and extra_reasoning: + reasoning_content = extra_reasoning usage_metadata = self._usage_from_anthropic(getattr(response, "usage", None)) @@ -798,6 +1058,11 @@ async def _parse_response( usage_metadata=usage_metadata, finish_reason=finish_reason, parser_content=parser_content, + reasoning_content=reasoning_content, + metadata=( + {_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY: replay_blocks} + if replay_blocks else {} + ), ) def _usage_from_anthropic(self, usage: Any) -> Optional[UsageMetadata]: @@ -865,12 +1130,38 @@ def _event_to_chunk( if etype == "content_block_start": block = getattr(event, "content_block", None) idx = getattr(event, "index", None) - if block is not None and getattr(block, "type", None) == "tool_use" and idx is not None: + if block is None or idx is None: + return None + block_type = getattr(block, "type", None) + if block_type == "tool_use": tool_use_acc[idx] = { + "type": "tool_use", "id": getattr(block, "id", "") or "", "name": getattr(block, "name", "") or "", + "input": getattr(block, "input", None) or {}, "args_str": "", } + elif block_type == "thinking": + tool_use_acc[idx] = { + "type": "thinking", + "thinking": getattr(block, "thinking", "") or "", + "signature": getattr(block, "signature", "") or "", + } + elif block_type == "redacted_thinking": + tool_use_acc[idx] = { + "type": "redacted_thinking", + "data": getattr(block, "data", "") or "", + } + return AssistantMessageChunk( + content="", + metadata=_stream_blocks_metadata(tool_use_acc), + finish_reason="null", + ) + elif block_type == "text": + tool_use_acc[idx] = { + "type": "text", + "text": getattr(block, "text", "") or "", + } return None if etype == "content_block_delta": @@ -883,14 +1174,48 @@ def _event_to_chunk( text = getattr(delta, "text", "") or "" if not text: return None + if idx is not None: + state = tool_use_acc.setdefault(idx, {"type": "text", "text": ""}) + state["text"] = (state.get("text") or "") + text return AssistantMessageChunk( content=text, reasoning_content=None, + metadata=_stream_blocks_metadata(tool_use_acc), + tool_calls=None, + usage_metadata=None, + finish_reason="null", + ) + if dtype == "thinking_delta": + thinking = getattr(delta, "thinking", "") or "" + if not thinking: + return None + if idx is not None: + state = tool_use_acc.setdefault( + idx, {"type": "thinking", "thinking": "", "signature": ""} + ) + state["thinking"] = (state.get("thinking") or "") + thinking + return AssistantMessageChunk( + content="", + reasoning_content=thinking, + metadata=_stream_blocks_metadata(tool_use_acc), tool_calls=None, usage_metadata=None, finish_reason="null", ) - if dtype == "input_json_delta" and idx is not None and idx in tool_use_acc: + if dtype == "signature_delta": + signature = getattr(delta, "signature", "") or "" + if idx is None or not signature: + return None + state = tool_use_acc.setdefault( + idx, {"type": "thinking", "thinking": "", "signature": ""} + ) + state["signature"] = (state.get("signature") or "") + signature + return AssistantMessageChunk( + content="", + metadata=_stream_blocks_metadata(tool_use_acc), + finish_reason="null", + ) + if self._is_tool_input_json_delta(dtype, idx, tool_use_acc): tool_use_acc[idx]["args_str"] += getattr(delta, "partial_json", "") or "" return None return None @@ -899,17 +1224,29 @@ def _event_to_chunk( idx = getattr(event, "index", None) if idx is None or idx not in tool_use_acc: return None - tu = tool_use_acc.pop(idx) + block = tool_use_acc[idx] + if block.get("type") != "tool_use": + return AssistantMessageChunk( + content="", + metadata=_stream_blocks_metadata(tool_use_acc), + finish_reason="null", + ) + args_str = block.get("args_str") or "{}" + try: + block["input"] = json.loads(args_str) + except (TypeError, ValueError): + block["input"] = {"_raw_arguments": args_str} return AssistantMessageChunk( content="", reasoning_content=None, tool_calls=[ToolCall( - id=tu["id"], + id=block["id"], type="function", - name=tu["name"], - arguments=tu["args_str"] or "{}", + name=block["name"], + arguments=args_str, index=idx, )], + metadata=_stream_blocks_metadata(tool_use_acc), usage_metadata=None, finish_reason="null", ) @@ -930,6 +1267,7 @@ def _event_to_chunk( content="", reasoning_content=None, tool_calls=None, + metadata=_stream_blocks_metadata(tool_use_acc), usage_metadata=usage_metadata, finish_reason=finish_reason, ) @@ -939,6 +1277,19 @@ def _event_to_chunk( return None + @staticmethod + def _is_tool_input_json_delta( + dtype: str, + idx: Optional[int], + tool_use_acc: Mapping[int, dict], + ) -> bool: + return ( + dtype == "input_json_delta" + and idx is not None + and idx in tool_use_acc + and tool_use_acc[idx].get("type") == "tool_use" + ) + # ------------------------------------------------------------------ # unsupported media methods (mirror OpenAI client's stub style) # ------------------------------------------------------------------ diff --git a/openjiuwen/core/foundation/llm/model_clients/ascend_affinity_model_client.py b/openjiuwen/core/foundation/llm/model_clients/ascend_affinity_model_client.py deleted file mode 100644 index d11a8adaf..000000000 --- a/openjiuwen/core/foundation/llm/model_clients/ascend_affinity_model_client.py +++ /dev/null @@ -1,1113 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -import asyncio -import json -from contextlib import asynccontextmanager -from typing import Any, AsyncIterator, Dict, List, Optional, Union - -import aiohttp - -from openjiuwen.core.common.exception.codes import StatusCode -from openjiuwen.core.common.exception.errors import build_error -from openjiuwen.core.common.logging import LogEventType, llm_logger -from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient -from openjiuwen.core.foundation.llm.output_parsers.output_parser import BaseOutputParser -from openjiuwen.core.foundation.llm.schema import ( - AudioGenerationResponse, - ImageGenerationResponse, - VideoGenerationResponse, -) -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig, ProviderType -from openjiuwen.core.foundation.llm.schema.message import AssistantMessage, BaseMessage, UsageMetadata, UserMessage -from openjiuwen.core.foundation.llm.schema.message_chunk import AssistantMessageChunk -from openjiuwen.core.foundation.llm.schema.tool_call import ToolCall -from openjiuwen.core.foundation.kv_cache.kv_cache_config import ( - KVC_MANAGEMENT_MAX_ATTEMPTS, - resolve_kvc_action_timeout, -) -from openjiuwen.core.foundation.tool import ToolInfo -from openjiuwen.core.runner.callback import trigger -from openjiuwen.core.runner.callback.events import LLMCallEvents - - -_KV_ACTIONS = {"evict", "offload", "prefetch"} -_KV_TARGETS = {"messages", "tools", "session"} - - -class AscendAffinityModelClient(BaseModelClient): - """OpenAI-compatible client for Ascend KV-cache affinity. - - The client keeps the framework-facing behavior of ``BaseModelClient`` while - using the aiohttp transport pattern already verified against the Ascend - inference service. Normal inference and KV-cache management share the - ``/v1/chat/completions`` endpoint; affinity intent is carried by the - top-level ``agent_hint`` request field. - """ - - __client_name__ = ProviderType.AscendAffinity.value - - def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelClientConfig): - super().__init__(model_config, model_client_config) - - def _get_client_name(self) -> str: - return "AscendAffinity client" - - def supports_kv_cache_affinity(self) -> bool: - return True - - @staticmethod - def _raise_config_error(message: str): - raise build_error( - StatusCode.MODEL_CONFIG_ERROR, - error_msg=f"[AscendAffinityModelClient] {message}" - ) - - @classmethod - def _validate_action_target(cls, action: str, target: str) -> None: - """Validate protocol-level KV action and target values.""" - if action not in _KV_ACTIONS: - cls._raise_config_error(f"unsupported KV affinity action: {action}") - if target not in _KV_TARGETS: - cls._raise_config_error(f"unsupported KV affinity target: {target}") - - @classmethod - def _range_edit( - cls, - *, - action: str, - target: str, - start: Optional[int] = None, - end: Optional[int] = None, - ) -> dict[str, Any]: - if start is None or end is None: - cls._raise_config_error( - f"target={target} requires both start and end" - ) - if not isinstance(start, int) or isinstance(start, bool): - cls._raise_config_error(f"target={target} start must be an integer") - if not isinstance(end, int) or isinstance(end, bool): - cls._raise_config_error(f"target={target} end must be an integer") - if start < 0 or end < 0: - cls._raise_config_error(f"target={target} range must be non-negative") - if start >= end: - cls._raise_config_error( - f"target={target} half-open range requires start < end" - ) - return {"type": action, "target": target, "start": start, "end": end} - - @staticmethod - def _has_any_range(**ranges: Optional[int]) -> bool: - return any(value is not None for value in ranges.values()) - - @classmethod - def _build_target_edits( - cls, - *, - action: str, - target: str, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - ) -> list[dict[str, Any]]: - """Build context-management edits for one protocol target. - - ``session`` operations are identity-scoped and therefore reject - message/tool ranges. ``messages`` may optionally include a tools edit - in the same management request. - """ - cls._validate_action_target(action, target) - - if target == "session": - if cls._has_any_range( - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - ): - cls._raise_config_error("target=session does not accept message/tool ranges") - if include_tools: - cls._raise_config_error("target=session does not accept include_tools=True") - return [{"type": action, "target": "session"}] - - if target == "messages": - edits = [ - cls._range_edit(action=action, target="messages", start=msg_start, end=msg_end) - ] - if include_tools: - edits.append( - cls._range_edit(action=action, target="tools", start=tools_start, end=tools_end) - ) - elif cls._has_any_range(tools_start=tools_start, tools_end=tools_end): - cls._raise_config_error("tools range requires include_tools=True or target=tools") - return edits - - if include_tools: - cls._raise_config_error("target=tools should not also set include_tools=True") - if cls._has_any_range(msg_start=msg_start, msg_end=msg_end): - cls._raise_config_error("messages range is invalid for target=tools") - return [ - cls._range_edit(action=action, target="tools", start=tools_start, end=tools_end) - ] - - @classmethod - def _build_agent_hint( - cls, - *, - session_id: Optional[str] = None, - parent_session_id: Optional[str] = None, - action: Optional[str] = None, - target: str = "session", - manage_request: Optional[bool] = None, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - ) -> dict[str, Any]: - """Build the required Ascend affinity extension. - - Normal inference carries only ``session_id`` and - ``parent_session_id``. When ``action`` is provided, - ``context_management`` is added and ``manage_request`` must be supplied - explicitly: - - - ``True``: execute a pure KV-cache management request; - - ``False``: reserved for a dedicated inference-then-management API, - such as a future ``invoke_then_evict_kvc`` implementation. - - The client deliberately provides no default for this distinction. - Current ``evict_kvc``, ``offload_kvc``, and ``prefetch_kvc`` methods - always use the pure-management path. - """ - if not session_id: - cls._raise_config_error("session_id is required") - if not parent_session_id: - cls._raise_config_error("parent_session_id is required") - - hint: dict[str, Any] = { - "session_id": session_id, - "parent_session_id": parent_session_id, - } - - if action is None: - if manage_request is not None: - cls._raise_config_error( - "manage_request is only valid when kv_action is set" - ) - return hint - - if not isinstance(manage_request, bool): - cls._raise_config_error( - "manage_request must be explicitly set when kv_action is set" - ) - - context_management: dict[str, Any] = { - "manage_request": manage_request, - "edits": cls._build_target_edits( - action=action, - target=target, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - include_tools=include_tools, - ), - } - hint["context_management"] = context_management - return hint - - def _build_ascend_affinity_request_params( - self, - *, - messages: Union[str, List[BaseMessage], List[dict]], - tools: Union[List[ToolInfo], List[dict], None], - temperature: Optional[float], - top_p: Optional[float], - model: Optional[str], - stop: Union[Optional[str], None], - max_tokens: Optional[int], - stream: bool, - session_id: str, - parent_session_id: str, - action: Optional[str] = None, - target: str = "session", - manage_request: Optional[bool] = None, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - **kwargs, - ) -> Dict[str, Any]: - """Build one Ascend-affinity Chat Completion request payload. - - This method owns protocol construction only. It validates affinity - identity and management fields, delegates standard Chat Completion - normalization to ``BaseModelClient``, and attaches the top-level - ``agent_hint`` extension. It performs no network I/O. - """ - if action is None and manage_request is not None: - self._raise_config_error( - "manage_request is only valid when kv_action is set" - ) - if action is not None and not isinstance(manage_request, bool): - self._raise_config_error( - "manage_request must be explicitly set when kv_action is set" - ) - if action is not None: - if not session_id: - self._raise_config_error("session_id is required") - if not parent_session_id: - self._raise_config_error("parent_session_id is required") - elif parent_session_id and not session_id: - self._raise_config_error( - "session_id is required when parent_session_id is set" - ) - - # Lifecycle-level session management may not have access to the original - # messages or tools. BaseModelClient rejects an empty message list, so a - # temporary message is used only during normalization and then removed. - is_session_manage_request = bool( - action and manage_request is True and target == "session" - ) - build_messages = ( - [{"role": "user", "content": ""}] - if is_session_manage_request - else messages - ) - - params = super()._build_request_params( - messages=build_messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=stream, - **kwargs, - ) - - if isinstance(params.get("messages"), list): - params["messages"] = self._sanitize_tool_calls(params["messages"]) - - if is_session_manage_request: - params["messages"] = [] - params.pop("tools", None) - params.pop("tool_choice", None) - - if session_id: - params["agent_hint"] = self._build_agent_hint( - session_id=session_id, - parent_session_id=parent_session_id or session_id, - action=action, - target=target, - manage_request=manage_request, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - include_tools=include_tools, - ) - return params - - def _build_request_params( - self, - *, - messages: Union[str, List[BaseMessage], List[dict]], - tools: Union[List[ToolInfo], List[dict], None], - temperature: Optional[float], - top_p: Optional[float], - model: Optional[str], - stop: Union[Optional[str], None], - max_tokens: Optional[int], - stream: bool, - **kwargs, - ) -> Dict[str, Any]: - """Compatibility adapter for framework code using the base method name.""" - session_id = kwargs.pop("session_id", None) - parent_session_id = kwargs.pop("parent_session_id", None) or session_id - return self._build_ascend_affinity_request_params( - messages=messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=stream, - session_id=session_id, - parent_session_id=parent_session_id, - action=kwargs.pop("kv_action", None), - target=kwargs.pop("target", "session"), - manage_request=kwargs.pop("manage_request", None), - msg_start=kwargs.pop("msg_start", None), - msg_end=kwargs.pop("msg_end", None), - tools_start=kwargs.pop("tools_start", None), - tools_end=kwargs.pop("tools_end", None), - include_tools=bool(kwargs.pop("include_tools", False)), - **kwargs, - ) - - def build_kv_cache_affinity_invoke_kwargs( - self, - *, - session: object = None, - session_id: Optional[str] = None, - parent_session_id: Optional[str] = None, - enable_kv_cache_affinity: bool = False, - **_: Any, - ) -> dict: - if not enable_kv_cache_affinity: - return {} - cache_id = session_id - if cache_id is None and session is not None and hasattr(session, "get_session_id"): - cache_id = session.get_session_id() - if not cache_id: - self._raise_config_error( - "session_id is required when KV cache affinity is enabled" - ) - return { - "session_id": cache_id, - "parent_session_id": parent_session_id or cache_id, - } - - @asynccontextmanager - async def _create_session(self, timeout: Optional[float] = None): - """Create a request-scoped aiohttp session. - - This transport shape intentionally follows the previously verified - InferenceAffinityModelClient path used in the restricted intranet - environment. - """ - final_timeout = timeout if timeout is not None else self.model_client_config.timeout - timeout_obj = aiohttp.ClientTimeout( - total=final_timeout, - connect=getattr(self.model_client_config, "connect_timeout", 30), - sock_read=final_timeout, - ) - async with aiohttp.ClientSession(timeout=timeout_obj) as session: - yield session - - async def invoke( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - output_parser: Optional[BaseOutputParser] = None, - timeout: Optional[float] = None, - **kwargs - ) -> AssistantMessage: - tracer_record_data = kwargs.pop("tracer_record_data", None) - params = self._build_ascend_affinity_request_params( - messages=messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=False, - session_id=kwargs.pop("session_id", None), - parent_session_id=kwargs.pop("parent_session_id", None), - action=kwargs.pop("kv_action", None), - target=kwargs.pop("target", "session"), - manage_request=kwargs.pop("manage_request", None), - msg_start=kwargs.pop("msg_start", None), - msg_end=kwargs.pop("msg_end", None), - tools_start=kwargs.pop("tools_start", None), - tools_end=kwargs.pop("tools_end", None), - include_tools=bool(kwargs.pop("include_tools", False)), - **kwargs, - ) - if tracer_record_data: - await tracer_record_data(llm_params=params) - - try: - await trigger( - LLMCallEvents.LLM_INPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - ) - response_data = await self._make_ascend_affinity_request(params, timeout=timeout) - assistant_message = await self._parse_response(response_data, output_parser) - if tracer_record_data: - await tracer_record_data(llm_response=assistant_message) - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - response=assistant_message.content, - usage=assistant_message.usage_metadata, - tool_calls=assistant_message.tool_calls, - ) - return assistant_message - except Exception as exc: - await trigger( - LLMCallEvents.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=False, - error=exc, - ) - llm_logger.error( - "AscendAffinity API async invoke error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - is_stream=False, - exception=str(exc), - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=f"AscendAffinity API async invoke error: {str(exc)}" - ) from exc - - async def stream( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - output_parser: Optional[BaseOutputParser] = None, - timeout: Optional[float] = None, - **kwargs - ) -> AsyncIterator[AssistantMessageChunk]: - tracer_record_data = kwargs.pop("tracer_record_data", None) - params = self._build_ascend_affinity_request_params( - messages=messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=True, - session_id=kwargs.pop("session_id", None), - parent_session_id=kwargs.pop("parent_session_id", None), - action=kwargs.pop("kv_action", None), - target=kwargs.pop("target", "session"), - manage_request=kwargs.pop("manage_request", None), - msg_start=kwargs.pop("msg_start", None), - msg_end=kwargs.pop("msg_end", None), - tools_start=kwargs.pop("tools_start", None), - tools_end=kwargs.pop("tools_end", None), - include_tools=bool(kwargs.pop("include_tools", False)), - **kwargs, - ) - if tracer_record_data: - await tracer_record_data(llm_params=params) - - final_message = None - try: - await trigger( - LLMCallEvents.LLM_INPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=True, - ) - stream_iter = ( - self._astream_with_parser(params, output_parser, timeout=timeout) - if output_parser - else self._stream_response(params, timeout=timeout) - ) - async for chunk in stream_iter: - if chunk: - final_message = chunk if final_message is None else final_message + chunk - yield chunk - if tracer_record_data: - await tracer_record_data(llm_response=final_message) - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=True, - response=final_message.content if final_message else None, - usage=final_message.usage_metadata if final_message else None, - tool_calls=final_message.tool_calls if final_message else None, - ) - except Exception as exc: - await trigger( - LLMCallEvents.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=True, - error=exc, - ) - llm_logger.error( - "AscendAffinity API async stream error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - is_stream=True, - exception=str(exc), - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=f"AscendAffinity API async stream error: {str(exc)}" - ) from exc - - async def _make_ascend_affinity_request( - self, - params: Dict[str, Any], - *, - timeout: Optional[float] = None, - max_attempts: Optional[int] = None, - ) -> Dict[str, Any]: - """Execute one non-streaming Ascend-affinity protocol request. - - The method intentionally remains thin: protocol payloads are built by - ``_build_ascend_affinity_request_params`` and raw HTTP transport is - delegated to ``_make_async_request``. - """ - return await self._make_async_request( - params, - timeout=timeout, - max_attempts=max_attempts, - ) - - async def _make_async_request( - self, - params: Dict[str, Any], - timeout: Optional[float] = None, - max_attempts: Optional[int] = None, - ) -> Dict[str, Any]: - """Send raw JSON over aiohttp with bounded exponential-backoff retries.""" - url = f"{self.model_client_config.api_base.rstrip('/')}/v1/chat/completions" - headers = {"Content-Type": "application/json"} - last_error = None - - attempts = ( - self.model_client_config.max_retries - if max_attempts is None - else max(1, int(max_attempts)) - ) - for attempt in range(attempts): - try: - async with self._create_session(timeout=timeout) as http_session: - async with http_session.post(url, headers=headers, json=params) as response: - response_text = await response.text() - if response.status != 200: - raise Exception(f"API returned error {response.status}: {response_text}") - try: - return json.loads(response_text) - except json.JSONDecodeError as exc: - raise Exception(f"API returned invalid JSON: {response_text}") from exc - except Exception as exc: - last_error = exc - if isinstance(exc, asyncio.TimeoutError): - llm_logger.warning( - "AscendAffinity request timeout.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={"attempt": attempt + 1}, - ) - else: - llm_logger.error( - "AscendAffinity request failed.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={"attempt": attempt + 1}, - exception=str(exc), - ) - if attempt < attempts - 1: - await asyncio.sleep(2 ** attempt) - - raise Exception(f"Request failed after {attempts} attempts: {last_error}") - - async def _stream_response( - self, - params: Dict[str, Any], - timeout: Optional[float] = None, - ) -> AsyncIterator[AssistantMessageChunk]: - """Yield parsed chunks from an SSE-style chat-completions response.""" - url = f"{self.model_client_config.api_base.rstrip('/')}/v1/chat/completions" - headers = {"Content-Type": "application/json"} - async with self._create_session(timeout=timeout) as http_session: - async with http_session.post(url, headers=headers, json=params) as response: - if response.status != 200: - error_text = await response.text() - raise Exception(f"API returned error {response.status}: {error_text}") - async for line in response.content: - line_str = line.decode("utf-8").strip() - if not line_str: - continue - chunk = self._parse_stream_chunk(line_str) - if chunk: - yield chunk - - async def _parse_response( - self, - response: Any, - parser: Optional[BaseOutputParser] = None, - ) -> AssistantMessage: - if not response.get("choices"): - raise ValueError("API did not return a valid response") - - choice = response.get("choices", [{}])[0] - message = choice.get("message", {}) - content = "" if message.get("content") is None else message.get("content") - reasoning_content = message.get("reasoning_content", None) - - tool_calls = [] - for idx, tc in enumerate(message.get("tool_calls") or []): - function = tc.get("function", {}) - tool_calls.append(ToolCall( - id=tc.get("id", "") or "", - type="function", - name=function.get("name", "") or "", - arguments=function.get("arguments", "") or "", - index=tc.get("index", idx), - )) - - usage_metadata = self._build_usage_metadata(response.get("usage")) - parser_content = None - if parser and content: - try: - parser_content = await parser.parse(content) - except Exception as exc: - llm_logger.warning( - "Parser parse error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - exception=str(exc), - ) - - return AssistantMessage( - content=content, - tool_calls=tool_calls if tool_calls else None, - usage_metadata=usage_metadata, - finish_reason="tool_calls" if tool_calls else "stop", - reasoning_content=reasoning_content, - parser_content=parser_content, - ) - - def _build_usage_metadata(self, usage: Any) -> Optional[UsageMetadata]: - if not usage: - return None - get_value = usage.get if isinstance(usage, dict) else lambda key, default=0: getattr(usage, key, default) - input_cost, output_cost, total_cost = self._extract_cost_info(usage) - return UsageMetadata( - model_name=self.model_config.model_name, - input_tokens=get_value("prompt_tokens", 0) or 0, - output_tokens=get_value("completion_tokens", 0) or 0, - total_tokens=get_value("total_tokens", 0) or 0, - cache_tokens=self._extract_cache_tokens(usage), - input_cost=input_cost, - output_cost=output_cost, - total_cost=total_cost, - ) - - async def _astream_with_parser( - self, - params: Dict[str, Any], - output_parser: BaseOutputParser, - timeout: Optional[float] = None, - ) -> AsyncIterator[AssistantMessageChunk]: - accumulated_content = "" - async for chunk_item in self._stream_response(params, timeout=timeout): - if chunk_item.content: - accumulated_content += chunk_item.content - parser_content = None - if accumulated_content: - try: - parsed = await output_parser.parse(accumulated_content) - if parsed is not None: - parser_content = parsed - accumulated_content = "" - except Exception: - parser_content = None - yield AssistantMessageChunk( - content=chunk_item.content, - reasoning_content=chunk_item.reasoning_content, - tool_calls=chunk_item.tool_calls, - usage_metadata=chunk_item.usage_metadata, - finish_reason=chunk_item.finish_reason, - parser_content=parser_content, - ) - - def _parse_stream_chunk(self, line: str) -> Optional[AssistantMessageChunk]: - if not line.startswith("data: "): - return None - data_str = line[6:] - if data_str.strip() == "[DONE]": - return None - try: - chunk_data = json.loads(data_str) - except json.JSONDecodeError as exc: - llm_logger.warning( - "Error parsing stream chunk.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=True, - line_content=line[:200], - exception=str(exc), - ) - return None - - choices = chunk_data.get("choices") or [] - usage_metadata = self._build_usage_metadata(chunk_data.get("usage")) - if not choices: - if usage_metadata: - return AssistantMessageChunk( - content="", - usage_metadata=usage_metadata, - finish_reason="null", - ) - return None - - choice = choices[0] - delta = choice.get("delta", {}) - tool_calls = [] - for tc_delta in delta.get("tool_calls") or []: - function = tc_delta.get("function", {}) - tool_calls.append(ToolCall( - id=tc_delta.get("id", "") or "", - type="function", - name=function.get("name", "") or "", - arguments=function.get("arguments", "") or "", - index=tc_delta.get("index", 0), - )) - - content = delta.get("content", None) or "" - reasoning_content = delta.get("reasoning_content", None) - if not any((content, reasoning_content, tool_calls, usage_metadata)): - return None - return AssistantMessageChunk( - content=content, - reasoning_content=reasoning_content, - tool_calls=tool_calls if tool_calls else None, - usage_metadata=usage_metadata, - finish_reason=choice.get("finish_reason") or "null", - ) - - def _sanitize_tool_calls(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Normalize assistant tool calls to the OpenAI-compatible wire schema. - - The list is normalized in place, matching the behavior of the verified - InferenceAffinityModelClient implementation. - """ - for msg in messages: - if msg.get("role") != "assistant": - continue - tool_calls = msg.get("tool_calls") - if not isinstance(tool_calls, list): - continue - cleaned = [] - for tc in tool_calls: - if not isinstance(tc, dict): - continue - function = tc.get("function", {}) - cleaned.append({ - "id": tc.get("id", ""), - "type": "function", - "index": tc.get("index"), - "function": { - "name": function.get("name", ""), - "arguments": function.get("arguments", ""), - }, - }) - msg["tool_calls"] = cleaned - return messages - - @staticmethod - def _validate_management_response(response: Any) -> None: - """Validate the agreed Chat Completion response envelope.""" - if not isinstance(response, dict) or not response.get("choices"): - raise ValueError( - "KV management request did not return a valid Chat Completion response" - ) - - async def _manage_kvc( - self, - action: str, - *, - session_id: str, - parent_session_id: Optional[str] = None, - target: str = "session", - messages: Union[str, List[BaseMessage], List[dict], None] = None, - tools: Union[List[ToolInfo], List[dict], None] = None, - model: Optional[str] = None, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - timeout: Optional[float] = None, - ) -> bool: - """Execute a pure KV-cache management request. - - Management requests are protocol peers of ``invoke`` rather than model - inference calls. They share request construction and aiohttp transport, - but do not emit normal LLM input/output callbacks or build an - ``AssistantMessage`` that would immediately be discarded. - """ - if not session_id: - self._raise_config_error("session_id is required") - if target == "messages" and messages is None: - self._raise_config_error("messages is required for target=messages") - if target == "tools" and messages is None: - self._raise_config_error("messages is required for target=tools") - if target == "tools" and tools is None: - self._raise_config_error("tools is required for target=tools") - if include_tools and tools is None: - self._raise_config_error("tools is required when include_tools=True") - - resolved_parent_session_id = parent_session_id or session_id - action_timeout = resolve_kvc_action_timeout(action, target, timeout) - params = self._build_ascend_affinity_request_params( - messages=[] if target == "session" else messages, - tools=None if target == "session" else tools, - temperature=None, - top_p=None, - model=model, - stop=None, - max_tokens=None, - stream=False, - session_id=session_id, - parent_session_id=resolved_parent_session_id, - action=action, - target=target, - manage_request=True, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - include_tools=include_tools, - ) - - try: - llm_logger.info( - "AscendAffinity KV management request started.", - event_type=LogEventType.LLM_CALL_START, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={ - "action": action, - "target": target, - "session_id": session_id, - "parent_session_id": resolved_parent_session_id, - "msg_start": msg_start, - "msg_end": msg_end, - "tools_start": tools_start, - "tools_end": tools_end, - }, - ) - # The transport timeout is per attempt. This outer deadline owns - # the whole management action and therefore also bounds retries - # and exponential backoff. wait_for cancels and reaps the request - # coroutine on expiry, so no retry task is left running. - response_data = await asyncio.wait_for( - self._make_ascend_affinity_request( - params, - timeout=action_timeout, - max_attempts=KVC_MANAGEMENT_MAX_ATTEMPTS, - ), - timeout=action_timeout, - ) - self._validate_management_response(response_data) - llm_logger.info( - "AscendAffinity KV management request completed.", - event_type=LogEventType.LLM_CALL_END, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={ - "action": action, - "target": target, - "session_id": session_id, - "parent_session_id": resolved_parent_session_id, - }, - ) - return True - except Exception as exc: - llm_logger.error( - "AscendAffinity KV management request failed.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={ - "action": action, - "target": target, - "session_id": session_id, - "parent_session_id": resolved_parent_session_id, - }, - exception=str(exc), - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=( - "AscendAffinity KV management request failed: " - f"{str(exc)}" - ), - ) from exc - - async def evict_kvc( - self, - *, - session_id: str, - parent_session_id: Optional[str] = None, - target: str = "session", - messages: Union[str, List[BaseMessage], List[dict], None] = None, - tools: Union[List[ToolInfo], List[dict], None] = None, - model: Optional[str] = None, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - timeout: Optional[float] = None, - ) -> bool: - return await self._manage_kvc( - "evict", - session_id=session_id, - parent_session_id=parent_session_id, - target=target, - messages=messages, - tools=tools, - model=model, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - include_tools=include_tools, - timeout=timeout, - ) - - async def offload_kvc( - self, - *, - session_id: str, - parent_session_id: Optional[str] = None, - target: str = "session", - messages: Union[str, List[BaseMessage], List[dict], None] = None, - tools: Union[List[ToolInfo], List[dict], None] = None, - model: Optional[str] = None, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - timeout: Optional[float] = None, - ) -> bool: - return await self._manage_kvc( - "offload", - session_id=session_id, - parent_session_id=parent_session_id, - target=target, - messages=messages, - tools=tools, - model=model, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - include_tools=include_tools, - timeout=timeout, - ) - - async def prefetch_kvc( - self, - *, - session_id: str, - parent_session_id: Optional[str] = None, - target: str = "session", - messages: Union[str, List[BaseMessage], List[dict], None] = None, - tools: Union[List[ToolInfo], List[dict], None] = None, - model: Optional[str] = None, - msg_start: Optional[int] = None, - msg_end: Optional[int] = None, - tools_start: Optional[int] = None, - tools_end: Optional[int] = None, - include_tools: bool = False, - timeout: Optional[float] = None, - ) -> bool: - return await self._manage_kvc( - "prefetch", - session_id=session_id, - parent_session_id=parent_session_id, - target=target, - messages=messages, - tools=tools, - model=model, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - include_tools=include_tools, - timeout=timeout, - ) - - async def generate_image( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - size: Optional[str] = "1664*928", - negative_prompt: Optional[str] = None, - n: Optional[int] = 1, - prompt_extend: bool = True, - watermark: bool = False, - seed: int = 0, - **kwargs - ) -> ImageGenerationResponse: - pass - - async def generate_video( - self, - messages: List[UserMessage], - *, - img_url: Optional[str] = None, - audio_url: Optional[str] = None, - model: Optional[str] = None, - size: Optional[str] = None, - resolution: Optional[str] = None, - duration: Optional[int] = 5, - prompt_extend: bool = True, - watermark: bool = False, - negative_prompt: Optional[str] = None, - seed: Optional[int] = None, - **kwargs - ) -> VideoGenerationResponse: - pass - - async def generate_speech( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - voice: Optional[str] = "Cherry", - language_type: Optional[str] = "Auto", - **kwargs - ) -> AudioGenerationResponse: - pass diff --git a/openjiuwen/core/foundation/llm/model_clients/base_model_client.py b/openjiuwen/core/foundation/llm/model_clients/base_model_client.py index 6e2e75476..7728905d0 100644 --- a/openjiuwen/core/foundation/llm/model_clients/base_model_client.py +++ b/openjiuwen/core/foundation/llm/model_clients/base_model_client.py @@ -23,6 +23,7 @@ from openjiuwen.core.common.security.user_config import UserConfig from openjiuwen.core.foundation.llm.output_parsers.output_parser import BaseOutputParser from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, ModelClientConfig, ModelRequestConfig, ) @@ -255,7 +256,12 @@ def _validate_config(self): """Validate configuration parameters (subclasses can optionally override)""" client_name = self._get_client_name() - if not self.model_client_config.api_key: + auth_mode = getattr( + self.model_client_config, + "auth_mode", + LLMAuthMode.ApiKey.value, + ) + if auth_mode in (LLMAuthMode.ApiKey, LLMAuthMode.ApiKey.value) and not self.model_client_config.api_key: raise build_error(StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg=f"model client config api_key is required for {client_name}.") if not self.model_client_config.api_base: diff --git a/openjiuwen/core/foundation/llm/model_clients/dashscope_model_client.py b/openjiuwen/core/foundation/llm/model_clients/dashscope_model_client.py deleted file mode 100644 index 7a1059a27..000000000 --- a/openjiuwen/core/foundation/llm/model_clients/dashscope_model_client.py +++ /dev/null @@ -1,579 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - -from typing import Optional, List - -import dashscope -from dashscope import MultiModalConversation, VideoSynthesis - -from openjiuwen.core.common.exception.errors import ValidationError, ModelError -from openjiuwen.core.common.exception.codes import StatusCode -from openjiuwen.core.common.logging import logger -from openjiuwen.core.foundation.llm.schema.message import UserMessage -from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig, ProviderType -from openjiuwen.core.foundation.llm.schema.generation_response import ( - ImageGenerationResponse, - AudioGenerationResponse, - VideoGenerationResponse -) - - -DASHSCOPE_VOICE = ["Cherry", "Serena", "Ethan", "Chelsie", "Momo", "Vivian", "Moon", "Maia", "Kai", "Nofish", - "Bella", "Jennifer", "Ryan", "Katerina", "Aiden", "Eldric Sage", "Mia", "Mochi", "Bellona", - "Vincent", "Bunny", "Neil", "Elias", "Arthur", "Nini" "Ebona", "Seren", "Pip", "Stella", "Bodega", - "Sonrisa", "Alek", "Dolce", "Sohee", "Ono Anna", "Lenn", "Emilien", "Andre", "Radio Gol", "Jada", - "Dylan", "Li", "Marcus", "Roy", "Peter", "Sunny", "Eric", "Rocky", "Kiki"] - -DASHSCOPE_LANGUAGE_TYPE = [ - "Chinese", "English", "German", "Italian", "Portuguese", - "Spanish", "Japanese", "Korean", "French", "Russian"] - - -class DashScopeModelClient(OpenAIModelClient): - """Alibaba Cloud DashScope Model Client - - This client extends OpenAIModelClient to support DashScope-specific multimodal generation APIs. - DashScope (通义千问) provides text-to-image, text-to-speech, and text-to-video capabilities - through Alibaba Cloud's proprietary APIs. - - For chat completions, it inherits all functionality from OpenAIModelClient since DashScope - provides OpenAI-compatible chat API endpoints. - """ - __client_name__ = ProviderType.DashScope.value - - def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelClientConfig): - super().__init__(model_config, model_client_config) - - def _get_client_name(self) -> str: - """Get client name.""" - return "DashScope client" - - async def generate_image( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - size: Optional[str] = "1664*928", - negative_prompt: Optional[str] = None, - n: Optional[int] = 1, - prompt_extend: bool = True, - watermark: bool = False, - seed: int = 0, - **kwargs - ) -> ImageGenerationResponse: - """Generate image using DashScope Wanx (通义万相) API - - DashScope provides text-to-image generation through the Wanx service. - - Args: - messages: List of messages, must only contain UserMessage type - model: Model to use (e.g., "qwen-image-max", "wanx-v1") - size: Size of the generated image (e.g., "1664*928", "1024*1024") - negative_prompt: Negative prompt to avoid certain features in the image - n: Number of images to generate (default: 1) - prompt_extend: Whether to extend the prompt (default: True) - watermark: Whether to add watermark (default: False) - seed: Random seed for reproducibility (default: 0) - **kwargs: Additional DashScope-specific parameters - - Returns: - ImageGenerationResponse: Generated image response - - Raises: - JiuWenBaseException: If messages contain non-UserMessage types or validation fails - """ - try: - # (1) Validate messages parameter - must have exactly one UserMessage - if not messages or len(messages) != 1: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Image generation requires exactly one message, but got {len(messages) if messages else 0}." - ) - - if not isinstance(messages[0], UserMessage): - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Image generation requires a UserMessage, but got {type(messages[0]).__name__}." - ) - - # (2) Validate and convert message content to DashScope format - msg = messages[0] - content_list = [] - image_count = 0 - text_count = 0 - - # Handle content: can be string or list of dicts - if isinstance(msg.content, str): - # Simple text prompt - content_list.append({"text": msg.content}) - text_count = 1 - elif isinstance(msg.content, list): - # Complex content with text and/or images - for item in msg.content: - if isinstance(item, str): - content_list.append({"text": item}) - text_count += 1 - elif isinstance(item, dict): - # Validate dict structure - if "text" in item: - content_list.append({"text": item["text"]}) - text_count += 1 - elif "image" in item: - content_list.append({"image": item["image"]}) - image_count += 1 - else: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Content dict must contain 'text' or 'image' key, but got: {list(item.keys())}" - ) - else: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Content item must be string or dict, but got {type(item).__name__}." - ) - else: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Message content must be string or list, but got {type(msg.content).__name__}." - ) - - # Validate content requirements - if text_count == 0: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg="Image generation requires at least one text prompt." - ) - - if image_count > 3: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Image generation supports at most 3 input images, but got {image_count}." - ) - - dashscope_messages = [{ - "role": msg.role, - "content": content_list - }] - - # Use default model if not specified - if model is None: - model = self.model_config.model_name - - # Prepare API parameters - api_params = { - "api_key": self.model_client_config.api_key, - "model": model, - "messages": dashscope_messages, - "result_format": "message", - "stream": False, - "watermark": watermark, - "prompt_extend": prompt_extend, - "size": size, - "n": n, - } - - # Add optional parameters - if negative_prompt: - api_params["negative_prompt"] = negative_prompt - - if seed: - api_params["seed"] = seed - - # Add any additional kwargs - api_params.update(kwargs) - - # Log request - logger.info( - f"Calling DashScope image generation API with model: {model}, size: {size}" - ) - - # Call DashScope API - dashscope.base_http_api_url = self.model_client_config.api_base - - response = MultiModalConversation.call(**api_params) - - # Handle response - if response.status_code != 200: - error_msg = ( - f"DashScope image generation failed. " - f"HTTP status: {response.status_code}, " - f"Error code: {response.code}, " - f"Error message: {response.message}" - ) - logger.error(error_msg) - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg=error_msg - ) - - # Extract image URLs from response - image_urls = [] - if response.output and response.output.get("choices"): - for choice in response.output["choices"]: - if choice.get("message") and choice["message"].get("content"): - for content_item in choice["message"]["content"]: - if isinstance(content_item, dict) and "image" in content_item: - image_urls.append(content_item["image"]) - - if not image_urls: - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg="No images returned from DashScope API." - ) - - # Log success - logger.info( - f"DashScope image generation succeeded. Generated {len(image_urls)} image(s)." - ) - - # Return ImageGenerationResponse - return ImageGenerationResponse( - model=model, - images=image_urls, - created=None # DashScope doesn't provide creation timestamp - ) - - except Exception as e: - error_msg = f"Unexpected error during DashScope image generation: {str(e)}" - logger.error(error_msg, exc_info=True) - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg=error_msg, - cause=e - ) from e - - async def generate_speech( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - voice: Optional[str] = "Cherry", - language_type: Optional[str] = "Auto", - **kwargs - ) -> AudioGenerationResponse: - """Generate speech using DashScope Cosyvoice API - - DashScope provides text-to-speech generation through the Cosyvoice service. - - Args: - messages: List of UserMessage containing text to convert to speech - model: Model to use for generation - voice: Voice to use for speech synthesis (required), refer to supported voices - language_type: Language type for synthesized audio, defaults to "Auto" for automatic detection - **kwargs: Additional parameters - - Returns: - AudioGenerationResponse: Generated audio response - - Raises: - JiuWenBaseException: If messages contain non-UserMessage types or validation fails - """ - try: - # (1) Validate messages parameter - must have at least one UserMessage - if not messages or len(messages) > 1: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg="Speech generation requires at least one message, but got 0." - ) - - # Validate all messages are UserMessage type - for idx, msg in enumerate(messages): - if not isinstance(msg, UserMessage): - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Speech generation requires UserMessage types," - f" but message at index {idx} is {type(msg).__name__}." - ) - - if len(messages) > 1: - pass - - text_to_synthesize = messages[0].content - - if not text_to_synthesize or not text_to_synthesize.strip(): - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg="Speech generation requires non-empty text content." - ) - - if model is None: - model = self.model_config.model_name - - dashscope.base_http_api_url = self.model_client_config.api_base - - # Prepare API parameters - api_params = { - "api_key": self.model_client_config.api_key, - "model": model, - "text": text_to_synthesize, - "voice": voice, - "language_type": language_type, - } - - # Add any additional kwargs - api_params.update(kwargs) - - # Log request - logger.info( - f"Calling DashScope speech generation API with model: {model}," - f" voice: {voice}, language: {language_type}" - ) - - # Call DashScope API - response = MultiModalConversation.call(**api_params) - - # Handle response - if response.status_code != 200: - error_msg = ( - f"DashScope speech generation failed. " - f"HTTP status: {response.status_code}, " - f"Error code: {response.code}, " - f"Error message: {response.message}" - ) - logger.error(error_msg) - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg=error_msg - ) - - # Extract audio information from response - # Response format: response.output.audio.url, response.output.audio.data - audio_url = None - audio_data = None - audio_format = None - - if response.output and response.output.get("audio"): - audio_info = response.output["audio"] - audio_url = audio_info.get("url") - audio_data_str = audio_info.get("data") - - # Convert audio data string to bytes if present - if audio_data_str: - audio_data = audio_data_str.encode('utf-8') if isinstance(audio_data_str, str) else audio_data_str - - # Infer audio format from URL extension - if audio_url: - if audio_url.endswith('.wav'): - audio_format = "wav" - elif audio_url.endswith('.mp3'): - audio_format = "mp3" - elif audio_url.endswith('.pcm'): - audio_format = "pcm" - - if not audio_url and not audio_data: - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg="No audio URL or data returned from DashScope API." - ) - - # Log success - logger.info( - f"DashScope speech generation succeeded. Audio format: {audio_format or 'unknown'}, " - f"URL present: {bool(audio_url)}, Data present: {bool(audio_data)}" - ) - - # Return AudioGenerationResponse - return AudioGenerationResponse( - model=model, - audio_url=audio_url, - audio_data=audio_data, - format=audio_format - ) - - except Exception as e: - error_msg = f"Unexpected error during DashScope speech generation: {str(e)}" - logger.error(error_msg, exc_info=True) - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg=error_msg, - cause=e - ) from e - - async def generate_video( - self, - messages: List[UserMessage], - *, - img_url: Optional[str] = None, - audio_url: Optional[str] = None, - model: Optional[str] = None, - size: Optional[str] = None, - resolution: Optional[str] = None, - duration: Optional[int] = 5, - prompt_extend: bool = True, - watermark: bool = False, - negative_prompt: Optional[str] = None, - seed: Optional[int] = None, - **kwargs - ) -> VideoGenerationResponse: - """Generate video using DashScope video generation API - - DashScope provides text-to-video (t2v) and image-to-video (i2v) generation capabilities. - When img_url is provided, it performs image-to-video generation; otherwise, text-to-video. - - Args: - messages: List of UserMessage containing text description of the video to generate - img_url: Optional URL/path of the first frame image for image-to-video generation. - Supports: public URL, local file path (file:// prefix), or base64 encoded image - audio_url: Optional URL of audio to add to the video - model: Model to use (e.g., "wan2.6-t2v" for text-to-video, "wan2.6-i2v-flash" for image-to-video) - size: Video size for text-to-video (e.g., "1280*720"). Use '*' as separator. - resolution: Video resolution for image-to-video (e.g., "720P", "1080P") - duration: Duration of the video in seconds (default: 5) - prompt_extend: Whether to automatically extend/enhance the prompt (default: True) - watermark: Whether to add watermark to generated video (default: False) - negative_prompt: Negative prompt to guide what not to generate - seed: Random seed for reproducible generation - **kwargs: Additional DashScope-specific parameters - - Returns: - VideoGenerationResponse: Generated video response containing video_url - - Raises: - JiuWenBaseException: If messages contain non-UserMessage types or validation fails - """ - try: - # (1) Validate messages parameter - must have exactly one UserMessage - if not messages or len(messages) != 1: - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Video generation requires exactly one message, but got {len(messages) if messages else 0}." - ) - - # Validate message is UserMessage type - if not isinstance(messages[0], UserMessage): - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg=f"Video generation requires UserMessage type, but got {type(messages[0]).__name__}." - ) - - # Extract prompt from message content - prompt = messages[0].content - - # Validate prompt - if not prompt or not prompt.strip(): - raise ValidationError( - StatusCode.MODEL_INVOKE_PARAM_ERROR, - msg="Video generation requires non-empty text content." - ) - - # Use default model if not specified - if model is None: - model = self.model_config.model_name - - # Set DashScope base URL - dashscope.base_http_api_url = self.model_client_config.api_base - - # Build API parameters - api_params = { - "api_key": self.model_client_config.api_key, - "model": model, - "prompt": prompt, - "prompt_extend": prompt_extend, - "watermark": watermark, - } - - # Add duration if specified - if duration is not None: - api_params["duration"] = duration - - # Add negative prompt if specified - if negative_prompt: - api_params["negative_prompt"] = negative_prompt - - # Add seed if specified - if seed is not None: - api_params["seed"] = seed - - # Add audio URL if specified - if audio_url: - api_params["audio_url"] = audio_url - - # Determine if this is image-to-video or text-to-video - if img_url: - # Image-to-video generation - api_params["img_url"] = img_url - # For i2v, use resolution parameter (e.g., "720P") - if resolution: - api_params["resolution"] = resolution - elif size: - # If only size is provided, try to convert to resolution - api_params["size"] = size - - logger.info( - f"Calling DashScope image-to-video generation API with model: {model}, " - f"resolution: {resolution or size}, duration: {duration}" - ) - else: - # Text-to-video generation - # For t2v, use size parameter (e.g., "1280*720") - if size: - api_params["size"] = size - elif resolution: - # If only resolution is provided, use it - api_params["resolution"] = resolution - - logger.info( - f"Calling DashScope text-to-video generation API with model: {model}, " - f"size: {size or resolution}, duration: {duration}" - ) - - # Add any additional kwargs - api_params.update(kwargs) - - # Call DashScope VideoSynthesis API - response = VideoSynthesis.call(**api_params) - - # Handle response - if response.status_code != 200: - error_msg = ( - f"DashScope video generation failed. " - f"HTTP status: {response.status_code}, " - f"Error code: {response.code}, " - f"Error message: {response.message}" - ) - logger.error(error_msg) - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg=error_msg - ) - - # Extract video URL from response - video_url = None - video_duration = None - video_resolution = None - - if response.output: - video_url = getattr(response.output, 'video_url', None) - - if response.usage: - video_duration = response.usage.get('duration') or response.usage.get('output_video_duration') - video_resolution = response.usage.get('size') - - if not video_url: - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg="No video URL returned from DashScope API." - ) - - # Log success - logger.info( - f"DashScope video generation succeeded. Video URL: {video_url[:100]}..." - ) - - # Return VideoGenerationResponse - return VideoGenerationResponse( - model=model, - video_url=video_url, - duration=video_duration, - resolution=video_resolution, - format="mp4" - ) - except Exception as e: - error_msg = f"Unexpected error during DashScope video generation: {str(e)}" - logger.error(error_msg, exc_info=True) - raise ModelError( - StatusCode.MODEL_CALL_FAILED, - msg=error_msg, - cause=e - ) from e diff --git a/openjiuwen/core/foundation/llm/model_clients/deepseek_model_client.py b/openjiuwen/core/foundation/llm/model_clients/deepseek_model_client.py deleted file mode 100644 index 5515f9d40..000000000 --- a/openjiuwen/core/foundation/llm/model_clients/deepseek_model_client.py +++ /dev/null @@ -1,83 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -from typing import Optional, List, Union - -from openjiuwen.core.foundation.llm.schema.message import UserMessage, BaseMessage -from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig, ProviderType -from openjiuwen.core.foundation.llm.schema.generation_response import ( - ImageGenerationResponse, - AudioGenerationResponse, - VideoGenerationResponse -) - - -class DeepSeekModelClient(OpenAIModelClient): - """DeepSeek Model Client""" - __client_name__ = ProviderType.DeepSeek.value - - def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelClientConfig): - super().__init__(model_config, model_client_config) - - def _get_client_name(self) -> str: - """Get client name.""" - return "DeepSeek client" - - @classmethod - def _convert_messages_to_dict(cls, messages: Union[str, List[BaseMessage], List[dict]]) -> List[dict]: - new_messages = super()._convert_messages_to_dict(messages=messages) - for msg in new_messages: - if msg.get("role") == "assistant" and msg.get("tool_calls") and "reasoning_content" not in msg: - msg["reasoning_content"] = "" - if msg.get("role") == "assistant" and "reasoning_content" not in msg: - msg["reasoning_content"] = "" - return new_messages - - - - async def generate_image( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - size: Optional[str] = "1664*928", - negative_prompt: Optional[str] = None, - n: Optional[int] = 1, - prompt_extend: bool = True, - watermark: bool = False, - seed: int = 0, - **kwargs - ) -> ImageGenerationResponse: - pass - - async def generate_speech( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - voice: Optional[str] = "Cherry", - language_type: Optional[str] = "Auto", - **kwargs - ) -> AudioGenerationResponse: - pass - - async def generate_video( - self, - messages: List[UserMessage], - *, - img_url: Optional[str] = None, - audio_url: Optional[str] = None, - model: Optional[str] = None, - size: Optional[str] = None, - resolution: Optional[str] = None, - duration: Optional[int] = 5, - prompt_extend: bool = True, - watermark: bool = False, - negative_prompt: Optional[str] = None, - seed: Optional[int] = None, - **kwargs - ) -> VideoGenerationResponse: - """Generate video using DashScope video generation API - """ - pass diff --git a/openjiuwen/core/foundation/llm/model_clients/inference_affinity_model_client.py b/openjiuwen/core/foundation/llm/model_clients/inference_affinity_model_client.py deleted file mode 100644 index b44cb433a..000000000 --- a/openjiuwen/core/foundation/llm/model_clients/inference_affinity_model_client.py +++ /dev/null @@ -1,911 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - -import json -import asyncio -from typing import List, Dict, Any, Optional, AsyncIterator, Union -from contextlib import asynccontextmanager -import aiohttp - -from openjiuwen.core.common.exception.codes import StatusCode -from openjiuwen.core.common.exception.errors import build_error -from openjiuwen.core.common.logging import llm_logger, LogEventType -from openjiuwen.core.foundation.llm.schema import ImageGenerationResponse, VideoGenerationResponse, \ - AudioGenerationResponse -from openjiuwen.core.foundation.llm.schema.message import ( - BaseMessage, - AssistantMessage, - UserMessage, - UsageMetadata -) -from openjiuwen.core.foundation.llm.schema.message_chunk import AssistantMessageChunk -from openjiuwen.core.foundation.llm.schema.tool_call import ToolCall -from openjiuwen.core.foundation.tool import ToolInfo -from openjiuwen.core.foundation.llm.output_parsers.output_parser import BaseOutputParser -from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig, ProviderType -from openjiuwen.core.runner.callback import trigger -from openjiuwen.core.runner.callback.events import LLMCallEvents - - -class InferenceAffinityModelClient(BaseModelClient): - """Inference Affinity (vLLM) API client with cache release support""" - __client_name__ = ProviderType.InferenceAffinity.value - - def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelClientConfig): - super().__init__(model_config, model_client_config) - - def _get_client_name(self) -> str: - """Get client name for error messages""" - return "InferenceAffinity client" - - def _build_and_sanitize_params( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - stream: bool = False, - session_id: str = None, - enable_cache_sharing: bool = False, - **kwargs - ) -> Dict[str, Any]: - """Build and sanitize request parameters""" - params = self._build_request_params( - messages=messages, - tools=tools, - model=model, - temperature=temperature, - top_p=top_p, - stop=stop, - max_tokens=max_tokens, - stream=stream, - **kwargs - ) - # Sanitize tool_calls in messages - params["messages"] = self._sanitize_tool_calls(params["messages"]) - if enable_cache_sharing and session_id: - params["cache_sharing"] = True - params["cache_salt"] = session_id - return params - - @asynccontextmanager - async def _create_session(self, timeout: Optional[float] = None): - """Create a new aiohttp session for each request - - Args: - timeout: Optional timeout override for this specific request - """ - final_timeout = timeout if timeout is not None else self.model_client_config.timeout - timeout_obj = aiohttp.ClientTimeout( - total=final_timeout, - connect=getattr(self.model_client_config, 'connect_timeout', 30), - sock_read=final_timeout - ) - async with aiohttp.ClientSession(timeout=timeout_obj) as session: - yield session - - async def invoke( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - output_parser: Optional[BaseOutputParser] = None, - timeout: Optional[float] = None, - session_id: str = None, - enable_cache_sharing: bool = False, - **kwargs - ) -> AssistantMessage: - """Async invoke InferenceAffinity API - - Args: - messages: Input messages - tools: Available tools - temperature: Sampling temperature - top_p: Nucleus sampling parameter - model: Model name override - max_tokens: Maximum tokens to generate - stop: Stop sequences - output_parser: Optional output parser - timeout: Request timeout in seconds - session_id: session id for cache sharing - enable_cache_sharing: enable cache sharing - **kwargs: Additional parameters - - Returns: - AssistantMessage: Model response - """ - tracer_record_data = kwargs.pop("tracer_record_data", None) - params = self._build_and_sanitize_params( - messages=messages, - tools=tools, - model=model, - temperature=temperature, - top_p=top_p, - stop=stop, - max_tokens=max_tokens, - stream=False, - session_id=session_id, - enable_cache_sharing=enable_cache_sharing, - **kwargs - ) - if tracer_record_data: - await tracer_record_data(llm_params=params) - - llm_logger.info( - "LLM request params ready.", - event_type=LogEventType.LLM_CALL_START, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=False - ) - - try: - await trigger( - LLMCallEvents.LLM_INPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - frequency_penalty=params.get("frequency_penalty"), - presence_penalty=params.get("presence_penalty"), - stop=params.get("stop")) - - response_data = await self._make_async_request(params, timeout=timeout) - - llm_logger.info( - "InferenceAffinity API response received.", - event_type=LogEventType.LLM_CALL_END, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=False, - metadata={"response": response_data} - ) - - # Parse response and apply output parser - llm_logger.info( - "Before parse response with output parser.", - event_type=LogEventType.LLM_CALL_END, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=False, - metadata={"output_parser": str(output_parser)} - ) - assistant_message = await self._parse_response(response_data, output_parser) - - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - response=assistant_message.content, - usage=assistant_message.usage_metadata, - tool_calls=assistant_message.tool_calls) - - return assistant_message - - except Exception as e: - await trigger( - LLMCallEvents.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=False, - error=e) - llm_logger.error( - "InferenceAffinity API async invoke error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=False, - exception=str(e) - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=f"InferenceAffinity API async invoke error: {str(e)}" - ) from e - - async def stream( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - output_parser: Optional[BaseOutputParser] = None, - timeout: Optional[float] = None, - session_id: str = None, - enable_cache_sharing: bool = False, - **kwargs - ) -> AsyncIterator[AssistantMessageChunk]: - """Async streaming invoke InferenceAffinity API - - Args: - messages: Input messages - tools: Available tools - temperature: Sampling temperature - top_p: Nucleus sampling parameter - model: Model name override - max_tokens: Maximum tokens to generate - stop: Stop sequences - output_parser: Optional output parser - timeout: Request timeout in seconds - session_id: session id for cache sharing - enable_cache_sharing: enable cache sharing - **kwargs: Additional parameters - - Yields: - AssistantMessageChunk: Streaming response chunk - """ - tracer_record_data = kwargs.pop("tracer_record_data", None) - params = self._build_and_sanitize_params( - messages=messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=True, - session_id=session_id, - enable_cache_sharing=enable_cache_sharing, - **kwargs - ) - - if tracer_record_data: - await tracer_record_data(llm_params=params) - - try: - await trigger( - LLMCallEvents.LLM_INPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - frequency_penalty=params.get("frequency_penalty"), - presence_penalty=params.get("presence_penalty"), - stop=params.get("stop"), - is_stream=True) - - if output_parser: - # Use streaming parser - async for parsed_result in self._astream_with_parser(params, output_parser, timeout=timeout): - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - result=parsed_result, - is_stream=True) - yield parsed_result - else: - # Direct return without parser - async for chunk in self._stream_response(params, timeout=timeout): - if chunk: - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - result=chunk, - is_stream=True) - yield chunk - - except Exception as e: - await trigger( - LLMCallEvents.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=True, - error=e) - llm_logger.error( - "InferenceAffinity API async stream error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=True, - exception=str(e) - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=f"InferenceAffinity API async stream error: {str(e)}" - ) from e - - async def generate_image( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - size: Optional[str] = "1664*928", - negative_prompt: Optional[str] = None, - n: Optional[int] = 1, - prompt_extend: bool = True, - watermark: bool = False, - seed: int = 0, - **kwargs - ) -> ImageGenerationResponse: - pass - - async def generate_speech( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - voice: Optional[str] = "Cherry", - language_type: Optional[str] = "Auto", - **kwargs - ) -> AudioGenerationResponse: - pass - - async def generate_video( - self, - messages: List[UserMessage], - *, - img_url: Optional[str] = None, - audio_url: Optional[str] = None, - model: Optional[str] = None, - size: Optional[str] = None, - resolution: Optional[str] = None, - duration: Optional[int] = 5, - prompt_extend: bool = True, - watermark: bool = False, - negative_prompt: Optional[str] = None, - seed: Optional[int] = None, - **kwargs - ) -> VideoGenerationResponse: - pass - - async def release( - self, - session_id: str, - messages: List, - messages_released_index: int, - *, - model: Optional[str] = None, - tools: Optional[List] = None, - tools_released_index: Optional[int] = None - ) -> bool: - """Release model cache or resources - - Args: - session_id: Cache salt value to identify specific cache - messages: Message list - messages_released_index: Message release index (0-based) - model: Model name (defaults to config model_name) - tools: Tool list - tools_released_index: Tool release index (0-based) - - Returns: - bool: Whether release was successful - - Raises: - BaseError: If release request fails - """ - try: - messages_dict = self._convert_messages_to_dict(messages) - tools_dict = self._convert_tools_to_dict(tools) - sanitized_messages = self._sanitize_tool_calls(messages_dict) - - release_params = { - "model": model if model else self.model_config.model_name, - "cache_salt": session_id, - "cache_sharing": True, - "messages": sanitized_messages, - "messages_released_index": messages_released_index, - } - - if tools_dict: - release_params["tools"] = tools_dict - - if tools_released_index is not None: - release_params["tools_released_index"] = tools_released_index - - client_name = self._get_client_name() - llm_logger.info( - "Before release KV cache, release request params ready.", - event_type=LogEventType.LLM_CALL_START, - model_name=model if model else self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - metadata={ - "client_name": client_name, - "session_id": session_id, - "messages_released_index": messages_released_index, - "tools_released_index": tools_released_index, - } - ) - - # Call vLLM release API - url = f"{self.model_client_config.api_base.rstrip('/')}/release_kv_cache" - headers = {"Content-Type": "application/json"} - - async with self._create_session() as http_session: - async with http_session.post(url, headers=headers, json=release_params) as response: - response_text = await response.text() - - if response.status == 200: - try: - result = json.loads(response_text) - llm_logger.info( - "KV cache release successful.", - event_type=LogEventType.LLM_CALL_END, - model_name=model if model else self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - metadata={ - "client_name": client_name, - "session_id": session_id, - "response": result - } - ) - return True - except json.JSONDecodeError: - llm_logger.info( - "KV cache release successful (non-JSON response).", - event_type=LogEventType.LLM_CALL_END, - model_name=model if model else self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - metadata={ - "client_name": client_name, - "session_id": session_id, - "response_text": response_text - } - ) - return True - else: - llm_logger.error( - f"KV cache release failed with status {response.status}.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=model if model else self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - metadata={ - "client_name": client_name, - "session_id": session_id, - "status_code": response.status, - "response_body": response_text - } - ) - return False - - except ValueError as ve: - # Log validation errors before re-raising - llm_logger.warning( - "KV cache release validation error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=model if model else self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - metadata={ - "client_name": self._get_client_name(), - "session_id": session_id, - "error": str(ve) - }, - exc_info=True - ) - raise # Preserve original traceback - except Exception as e: - client_name = self._get_client_name() - llm_logger.error( - f"KV cache release error: {str(e)}", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=model if model else self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - metadata={ - "client_name": client_name, - "session_id": session_id, - "error": str(e) - }, - exc_info=True - ) - raise build_error( - error_code=StatusCode.MODEL_CALL_FAILED, - error_msg=f"Release error: {str(e)}", - status=StatusCode.ERROR - ) from e - - async def _make_async_request(self, params: Dict, timeout: Optional[float] = None) -> Dict: - """Make async HTTP request with retry logic - - Args: - params: Request parameters - timeout: Optional timeout override for this specific request - """ - url = f"{self.model_client_config.api_base.rstrip('/')}/v1/chat/completions" - headers = {"Content-Type": "application/json"} - - last_error = None - for attempt in range(self.model_client_config.max_retries): - try: - llm_logger.debug( - f"Non-stream request (attempt {attempt + 1}/{self.model_client_config.max_retries})", - event_type=LogEventType.LLM_CALL_START, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={"attempt": attempt + 1} - ) - - async with self._create_session(timeout=timeout) as http_session: - async with http_session.post(url, headers=headers, json=params) as response: - if response.status != 200: - error_text = await response.text() - raise Exception(f"API returned error {response.status}: {error_text}") - - return await response.json() - - except Exception as e: - last_error = e - if isinstance(e, asyncio.TimeoutError): - llm_logger.warning( - f"Request timeout: {str(e)}", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={"attempt": attempt + 1} - ) - else: - llm_logger.error( - f"Request failed: {str(e)}", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={"attempt": attempt + 1}, - exception=str(e) - ) - - if attempt < self.model_client_config.max_retries - 1: - wait_time = 2 ** attempt - llm_logger.info( - f"Retrying in {wait_time} seconds...", - event_type=LogEventType.LLM_CALL_START, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - metadata={"wait_time": wait_time, "next_attempt": attempt + 2} - ) - await asyncio.sleep(wait_time) - - raise Exception(f"Request failed after {self.model_client_config.max_retries} attempts: {str(last_error)}") - - async def _parse_response( - self, - response: Any, - parser: Optional[BaseOutputParser] = None - ) -> AssistantMessage: - """Parse InferenceAffinity API response - - Args: - response: API response object (dict from JSON) - parser: Optional output parser, only parses content field - - Returns: - AssistantMessage: Parsed assistant message - """ - if not response.get("choices"): - raise ValueError("API did not return a valid response") - - choice = response.get("choices", [{}])[0] - message = choice.get("message", {}) - - # Get content - content = "" if message.get("content") is None else message.get("content") - - # Get reasoning_content (if exists) - reasoning_content = message.get("reasoning_content", None) - - # Parse tool_calls - tool_calls = [] - if message.get("tool_calls"): - for idx, tc in enumerate(message.get("tool_calls", [])): - function = tc.get("function", {}) - tool_call = ToolCall( - id=tc.get("id", "") or "", - type="function", - name=function.get("name", "") or "", - arguments=function.get("arguments", "") or "", - index=tc.get("index", idx) - ) - tool_calls.append(tool_call) - - # Build UsageMetadata - usage_metadata = None - usage = response.get("usage") - if usage: - input_tokens = usage.get("prompt_tokens", 0) or 0 - output_tokens = usage.get("completion_tokens", 0) or 0 - total_tokens = usage.get("total_tokens", 0) or 0 - - usage_metadata = UsageMetadata( - model_name=self.model_config.model_name, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - cache_tokens=self._extract_cache_tokens(usage), - reasoning_tokens=self._extract_reasoning_tokens(usage), - ) - - # Apply output parser (only parse content field) - parser_content = None - llm_logger.info( - "Before parse content with parser.", - event_type=LogEventType.LLM_CALL_END, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - response_content=content, - is_stream=False - ) - llm_logger.info( - "Before parse content with parser config.", - event_type=LogEventType.LLM_CALL_END, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - metadata={"parser": str(parser)} - ) - if parser and content: - try: - parser_content = await parser.parse(content) - llm_logger.info( - "Parser parse success.", - event_type=LogEventType.LLM_CALL_END, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - metadata={"parser_content": parser_content} - ) - except Exception as e: - llm_logger.warning( - "Parser parse error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - exception=str(e) - ) - parser_content = None - - return AssistantMessage( - content=content, - tool_calls=tool_calls if tool_calls else None, - usage_metadata=usage_metadata, - finish_reason="tool_calls" if tool_calls else "stop", - reasoning_content=reasoning_content, - parser_content=parser_content - ) - - async def _astream_with_parser( - self, - params: Dict, - output_parser: BaseOutputParser, - timeout: Optional[float] = None - ) -> AsyncIterator[AssistantMessageChunk]: - """Process streaming response with output parser - - Strategy: - 1. Immediately yield each raw chunk, maintaining streaming characteristics (content is incremental) - 2. Accumulate all content - 3. **Attempt to parse accumulated content every time a new chunk is received** - 4. When parsing succeeds, output parser_content and clear buffer (implementing incremental output) - 5. When parsing fails, parser_content is None, continue accumulating - """ - accumulated_content = "" - - async for chunk_item in self._stream_response(params, timeout=timeout): - if chunk_item: - # Accumulate content - if chunk_item.content: - accumulated_content += chunk_item.content - - # Attempt to parse accumulated content every time - parser_content = None - if accumulated_content and output_parser: - try: - current_parsed_result = await output_parser.parse(accumulated_content) - # When parsing succeeds, output result and clear buffer - if current_parsed_result is not None: - parser_content = current_parsed_result - accumulated_content = "" # Clear buffer to implement incremental output - except Exception as e: - llm_logger.debug( - "Stream parser attempt error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=True, - exception=str(e) - ) - parser_content = None - - # Create new chunk with original content and parser_content - chunk_with_parser = AssistantMessageChunk( - content=chunk_item.content, - reasoning_content=chunk_item.reasoning_content, - tool_calls=chunk_item.tool_calls, - usage_metadata=chunk_item.usage_metadata, - finish_reason=chunk_item.finish_reason, - parser_content=parser_content - ) - - yield chunk_with_parser - - async def _stream_response(self, params: Dict, timeout: Optional[float] = None) -> AsyncIterator[ - AssistantMessageChunk]: - """Stream response from API - - Args: - params: Request parameters - timeout: Optional timeout override for this specific request - """ - url = f"{self.model_client_config.api_base.rstrip('/')}/v1/chat/completions" - headers = {"Content-Type": "application/json"} - - async with self._create_session(timeout=timeout) as http_session: - async with http_session.post(url, headers=headers, json=params) as response: - if response.status != 200: - error_text = await response.text() - raise Exception(f"API returned error {response.status}: {error_text}") - - async for line in response.content: - line_str = line.decode('utf-8').strip() - if not line_str: - continue - - chunk = self._parse_stream_chunk(line_str) - if chunk: - yield chunk - - def _parse_stream_chunk(self, line: str) -> Optional[AssistantMessageChunk]: - """Parse streaming response line - - Args: - line: SSE format single line data (e.g., "data: {...}") - - Returns: - AssistantMessageChunk or None - """ - if not line.startswith("data: "): - return None - - data_str = line[6:] - if data_str.strip() == "[DONE]": - return None - - try: - chunk_data = json.loads(data_str) - - if "choices" in chunk_data and chunk_data["choices"]: - choice = (chunk_data.get("choices") or [{}])[0] - delta = choice.get("delta", {}) - - # Extract content - content = delta.get("content", None) or "" - reasoning_content = delta.get("reasoning_content", None) - - # Parse tool_calls delta - tool_calls = [] - tool_calls_delta = delta.get("tool_calls") - if tool_calls_delta: - for tc_delta in tool_calls_delta: - index = tc_delta.get("index", 0) - tool_call_id = tc_delta.get("id", "") - function_delta = tc_delta.get("function", {}) - name_delta = function_delta.get("name", "") - args_delta = function_delta.get("arguments", "") - - tool_call = ToolCall( - id=tool_call_id or "", - type="function", - name=name_delta or "", - arguments=args_delta or "", - index=index - ) - tool_calls.append(tool_call) - - # Build usage_metadata (usually only in the last chunk) - usage_metadata = None - usage = chunk_data.get("usage") - if usage: - usage_metadata = UsageMetadata( - model_name=self.model_config.model_name, - input_tokens=usage.get("prompt_tokens", 0) or 0, - output_tokens=usage.get("completion_tokens", 0) or 0, - total_tokens=usage.get("total_tokens", 0) or 0, - cache_tokens=self._extract_cache_tokens(usage), - reasoning_tokens=self._extract_reasoning_tokens(usage), - ) - - # Skip empty chunks - is_response_empty = ( - not content - and not reasoning_content - and not tool_calls - and not usage_metadata - ) - if is_response_empty: - return None - - return AssistantMessageChunk( - content=content, - reasoning_content=reasoning_content, - tool_calls=tool_calls if tool_calls else None, - usage_metadata=usage_metadata, - finish_reason=choice.get("finish_reason") or "null" - ) - return None - - except (json.JSONDecodeError, KeyError, IndexError, AttributeError) as e: - llm_logger.warning( - "Error parsing stream chunk.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=True, - line_content=line[:200] if len(line) <= 200 else f"{line[:200]}...", - exception=str(e) - ) - return None - - def _sanitize_tool_calls(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Sanitize tool_calls in messages, keep OpenAI standard fields: - id, type, function.name, function.arguments - Force type to "function" - - Args: - messages: List of message dictionaries - - Returns: - Sanitized message list - """ - for msg in messages: - if msg.get("role") != "assistant": - continue - tool_calls = msg.get("tool_calls") - if not isinstance(tool_calls, list): - continue - - cleaned = [] - for tc in tool_calls: - if not isinstance(tc, dict): - continue - func = tc.get("function", {}) - cleaned.append({ - "id": tc.get("id", ""), - "type": "function", - "index": tc.get("index"), - "function": { - "name": func.get("name", ""), - "arguments": func.get("arguments", "") - } - }) - msg["tool_calls"] = cleaned - return messages diff --git a/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py b/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py index 676a93604..5e2a00716 100644 --- a/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py +++ b/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py @@ -1,13 +1,14 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +from copy import deepcopy from dataclasses import dataclass from typing import TYPE_CHECKING, Any, AsyncIterator, Callable, Dict, Iterable, List, Mapping, Optional, Tuple, Union import httpx from openjiuwen.core.common.exception.codes import StatusCode -from openjiuwen.core.common.exception.errors import build_error +from openjiuwen.core.common.exception.errors import ModelError, build_error from openjiuwen.core.common.logging import llm_logger, logger, LogEventType from openjiuwen.core.common.security.ssl_utils import SslUtils from openjiuwen.core.common.security.url_utils import UrlUtils @@ -29,7 +30,13 @@ merge_request_headers, ) from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig, ProviderType +from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, + ModelClientConfig, + ModelRequestConfig, + ProviderType, +) +from openjiuwen.core.foundation.llm.utils.endpoint_profiles import apply_message_transforms from openjiuwen.core.runner.callback import trigger from openjiuwen.core.runner.callback.events import LLMCallEvents @@ -52,6 +59,179 @@ class ModelParamRule: ), ) +OPENROUTER_ATTRIBUTION_HEADER_KEYS = frozenset({ + "http-referer", + "x-openrouter-title", + "x-openrouter-categories", +}) +OPENROUTER_EXPLICIT_PROMPT_CACHING_PROVIDERS = frozenset({ + "anthropic", + "qwen", +}) +OPENROUTER_1H_PROMPT_CACHE_TTL_PROVIDERS = frozenset({ + "anthropic", +}) +DASHSCOPE_VOICE = frozenset({ + "Cherry", "Serena", "Ethan", "Chelsie", "Momo", "Vivian", "Moon", "Maia", "Kai", "Nofish", + "Bella", "Jennifer", "Ryan", "Katerina", "Aiden", "Eldric Sage", "Mia", "Mochi", "Bellona", + "Vincent", "Bunny", "Neil", "Elias", "Arthur", "Nini", "Ebona", "Seren", "Pip", "Stella", "Bodega", + "Sonrisa", "Alek", "Dolce", "Sohee", "Ono Anna", "Lenn", "Emilien", "Andre", "Radio Gol", "Jada", + "Dylan", "Li", "Marcus", "Roy", "Peter", "Sunny", "Eric", "Rocky", "Kiki", +}) +DASHSCOPE_LANGUAGE_TYPE = frozenset({ + "Auto", "Chinese", "English", "German", "Italian", "Portuguese", + "Spanish", "Japanese", "Korean", "French", "Russian", +}) +_KV_ACTIONS = {"evict", "offload", "prefetch"} +_KV_TARGETS = {"messages", "tools", "session"} +_OPENAI_EXTRA_BODY_EXTENSION_FIELDS = { + "agent_hint", + "cache_salt", + "cache_sharing", + "return_token_ids", +} + + +def _openrouter_model_provider(model: Optional[str]) -> Optional[str]: + if not model or "/" not in model: + return None + return model.split("/", 1)[0].lstrip("~").lower() + + +def _normalize_openrouter_provider_set(value: Any, default: frozenset[str]) -> frozenset[str]: + if value is None: + return default + values = value.split(",") if isinstance(value, str) else value + try: + return frozenset(str(provider).strip().lower() for provider in values if str(provider).strip()) + except TypeError: + return default + + +def _supports_openrouter_explicit_prompt_caching( + model: Optional[str], + supported_providers: frozenset[str] = OPENROUTER_EXPLICIT_PROMPT_CACHING_PROVIDERS, +) -> bool: + return _openrouter_model_provider(model) in supported_providers + + +def _supports_openrouter_1h_prompt_cache_ttl( + model: Optional[str], + supported_providers: frozenset[str] = OPENROUTER_1H_PROMPT_CACHE_TTL_PROVIDERS, +) -> bool: + return _openrouter_model_provider(model) in supported_providers + + +def _without_cache_control(value: Any) -> Any: + if isinstance(value, dict): + normalized = { + key: _without_cache_control(item) + for key, item in value.items() + if key != "cache_control" + } + if normalized.get("type") == "text" and set(normalized) <= {"type", "text"}: + return normalized.get("text", "") + content = normalized.get("content") + if isinstance(content, list) and len(content) == 1 and isinstance(content[0], str): + normalized["content"] = content[0] + return normalized + if isinstance(value, list): + return [_without_cache_control(item) for item in value] + return value + + +def _contains_cache_control(value: Any) -> bool: + if isinstance(value, dict): + if "cache_control" in value: + return True + return any(_contains_cache_control(item) for item in value.values()) + if isinstance(value, list): + return any(_contains_cache_control(item) for item in value) + return False + + +def _build_cache_control_marker(enable_1h_ttl: bool = False) -> dict: + marker = {"type": "ephemeral"} + if enable_1h_ttl: + marker["ttl"] = "1h" + return marker + + +def _add_cache_control_marker(block: dict, enable_1h_ttl: bool = False) -> dict: + block.setdefault("cache_control", _build_cache_control_marker(enable_1h_ttl)) + return block + + +def _mark_message_with_cache_control(message: dict, enable_1h_ttl: bool = False) -> None: + if _contains_cache_control(message): + return + + content = message.get("content") + if isinstance(content, list): + if not content: + return + last_index = len(content) - 1 + last_block = content[last_index] + if isinstance(last_block, dict): + _add_cache_control_marker(last_block, enable_1h_ttl) + else: + content[last_index] = _add_cache_control_marker({ + "type": "text", + "text": last_block if isinstance(last_block, str) else str(last_block), + }, enable_1h_ttl) + return + + message["content"] = [_add_cache_control_marker({ + "type": "text", + "text": content if isinstance(content, str) else ("" if content is None else str(content)), + }, enable_1h_ttl)] + + +def _longest_prefix_overlap_index(previous_messages: Optional[list], current_messages: list) -> Optional[int]: + if not previous_messages: + return None + + overlap = 0 + for previous, current in zip(previous_messages, current_messages): + if _without_cache_control(previous) != _without_cache_control(current): + break + overlap += 1 + + if overlap == 0: + return None + return overlap - 1 + + +def _apply_openrouter_prompt_cache_control( + params: dict, + previous_messages: Optional[list], + *, + enable_1h_ttl: bool = False, +) -> None: + tools = params.get("tools") + if isinstance(tools, list) and tools and isinstance(tools[-1], dict): + _add_cache_control_marker(tools[-1], enable_1h_ttl) + + messages = params.get("messages") + if not isinstance(messages, list) or not messages: + return + + prefix_index = _longest_prefix_overlap_index(previous_messages, messages) + + if isinstance(messages[0], dict): + _mark_message_with_cache_control(messages[0], enable_1h_ttl) + if prefix_index is not None and isinstance(messages[prefix_index], dict): + _mark_message_with_cache_control(messages[prefix_index], enable_1h_ttl) + if isinstance(messages[-1], dict): + _mark_message_with_cache_control(messages[-1], enable_1h_ttl) + + +def _resolved_api_key_for_config(model_client_config: ModelClientConfig) -> str: + auth_mode = getattr(model_client_config, "auth_mode", LLMAuthMode.ApiKey.value) + if auth_mode in (LLMAuthMode.NoneAuth, LLMAuthMode.NoneAuth.value): + return "EMPTY" + return model_client_config.api_key + class OpenAIModelClient(BaseModelClient): """OpenAI API client supporting GPT models and OpenAI-compatible services.""" @@ -72,6 +252,28 @@ def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelC self._base_headers = build_base_headers( custom_headers=model_client_config.custom_headers, ) + extra = model_client_config.__pydantic_extra__ or {} + self._enable_openrouter_explicit_caching = extra.get( + "openrouter_enable_explicit_prompt_caching", + True, + ) + self._enable_openrouter_prompt_cache_prefix_matching = extra.get( + "openrouter_enable_prompt_cache_prefix_matching", + True, + ) + self._enable_openrouter_1h_prompt_cache_ttl = extra.get( + "openrouter_enable_1h_prompt_cache_ttl", + False, + ) + self._openrouter_explicit_prompt_cache_providers = _normalize_openrouter_provider_set( + extra.get("openrouter_explicit_prompt_cache_providers"), + OPENROUTER_EXPLICIT_PROMPT_CACHING_PROVIDERS, + ) + self._openrouter_prompt_cache_1h_ttl_providers = _normalize_openrouter_provider_set( + extra.get("openrouter_prompt_cache_1h_ttl_providers"), + OPENROUTER_1H_PROMPT_CACHE_TTL_PROVIDERS, + ) + self._previous_openrouter_prompt_cache_messages: Optional[list] = None def _use_shared_client(self) -> bool: """Whether to reuse the process-wide cached client (default True). @@ -93,8 +295,9 @@ def connection_key(cls, model_client_config: ModelClientConfig) -> Tuple: """ cfg = model_client_config return ( - cfg.api_key, + _resolved_api_key_for_config(cfg), cfg.api_base, + getattr(cfg, "auth_mode", LLMAuthMode.ApiKey.value), cfg.verify_ssl, cfg.ssl_cert, ) @@ -119,6 +322,9 @@ def _apply_model_specific_params(self, model: Optional[str], params: dict) -> No def _client_cache_key(self) -> Tuple: return self.connection_key(self.model_client_config) + def _resolved_api_key(self) -> str: + return _resolved_api_key_for_config(self.model_client_config) + def _get_client_name(self) -> str: """Get client name.""" return "OpenAI client" @@ -130,7 +336,356 @@ def _build_request_headers( request_headers: Optional[Mapping[str, Any]], ) -> dict[str, str]: """Merge request-level headers with prebuilt config-level headers (request wins).""" - return merge_request_headers(base_headers, request_headers) + filtered_request_headers = request_headers + if request_headers: + filtered_request_headers = { + key: value + for key, value in request_headers.items() + if key.lower() not in OPENROUTER_ATTRIBUTION_HEADER_KEYS + } + return merge_request_headers(base_headers, filtered_request_headers) + + def _endpoint_profile_name(self) -> str: + return str(getattr(self.model_client_config, "endpoint_profile", "") or "").strip().lower() + + def _kv_cache_config(self): + extensions = getattr(self.model_client_config, "extensions", None) + return getattr(extensions, "kv_cache", None) if extensions is not None else None + + def _kv_cache_mode(self) -> str: + kv_cache = self._kv_cache_config() + mode = getattr(kv_cache, "mode", "none") + return mode.value if hasattr(mode, "value") else str(mode or "none") + + def supports_kv_cache_release(self) -> bool: + return self._kv_cache_mode() == "release" + + def supports_kv_cache_affinity(self) -> bool: + return self._kv_cache_mode() == "affinity" + + @staticmethod + def _sanitize_tool_calls(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + for msg in messages: + if msg.get("role") != "assistant": + continue + tool_calls = msg.get("tool_calls") + if not isinstance(tool_calls, list): + continue + + cleaned = [] + for tc in tool_calls: + if not isinstance(tc, dict): + continue + func = tc.get("function", {}) + cleaned.append({ + "id": tc.get("id", ""), + "type": "function", + "index": tc.get("index"), + "function": { + "name": func.get("name", ""), + "arguments": func.get("arguments", ""), + }, + }) + msg["tool_calls"] = cleaned + return messages + + @staticmethod + def _raise_kv_cache_error(message: str): + raise build_error( + StatusCode.MODEL_CONFIG_ERROR, + error_msg=f"[OpenAIModelClient kv_cache] {message}", + ) + + @classmethod + def _validate_kv_action_target(cls, action: str, target: str) -> None: + if action not in _KV_ACTIONS: + cls._raise_kv_cache_error(f"unsupported KV affinity action: {action}") + if target not in _KV_TARGETS: + cls._raise_kv_cache_error(f"unsupported KV affinity target: {target}") + + @classmethod + def _kv_range_edit( + cls, + *, + action: str, + target: str, + start: Optional[int] = None, + end: Optional[int] = None, + ) -> dict[str, Any]: + if start is None or end is None: + cls._raise_kv_cache_error(f"target={target} requires both start and end") + if not isinstance(start, int) or isinstance(start, bool): + cls._raise_kv_cache_error(f"target={target} start must be an integer") + if not isinstance(end, int) or isinstance(end, bool): + cls._raise_kv_cache_error(f"target={target} end must be an integer") + if start < 0 or end < 0: + cls._raise_kv_cache_error(f"target={target} range must be non-negative") + if start >= end: + cls._raise_kv_cache_error(f"target={target} half-open range requires start < end") + return {"type": action, "target": target, "start": start, "end": end} + + @staticmethod + def _has_any_kv_range(**ranges: Optional[int]) -> bool: + return any(value is not None for value in ranges.values()) + + @classmethod + def _build_kv_target_edits( + cls, + *, + action: str, + target: str, + msg_start: Optional[int] = None, + msg_end: Optional[int] = None, + tools_start: Optional[int] = None, + tools_end: Optional[int] = None, + include_tools: bool = False, + ) -> list[dict[str, Any]]: + cls._validate_kv_action_target(action, target) + + if target == "session": + if cls._has_any_kv_range( + msg_start=msg_start, + msg_end=msg_end, + tools_start=tools_start, + tools_end=tools_end, + ): + cls._raise_kv_cache_error("target=session does not accept message/tool ranges") + if include_tools: + cls._raise_kv_cache_error("target=session does not accept include_tools=True") + return [{"type": action, "target": "session"}] + + if target == "messages": + edits = [ + cls._kv_range_edit(action=action, target="messages", start=msg_start, end=msg_end) + ] + if include_tools: + edits.append( + cls._kv_range_edit(action=action, target="tools", start=tools_start, end=tools_end) + ) + elif cls._has_any_kv_range(tools_start=tools_start, tools_end=tools_end): + cls._raise_kv_cache_error("tools range requires include_tools=True or target=tools") + return edits + + if include_tools: + cls._raise_kv_cache_error("target=tools should not also set include_tools=True") + if cls._has_any_kv_range(msg_start=msg_start, msg_end=msg_end): + cls._raise_kv_cache_error("messages range is invalid for target=tools") + return [ + cls._kv_range_edit(action=action, target="tools", start=tools_start, end=tools_end) + ] + + @classmethod + def _build_agent_hint( + cls, + *, + session_id: Optional[str] = None, + parent_session_id: Optional[str] = None, + action: Optional[str] = None, + target: str = "session", + manage_request: Optional[bool] = None, + msg_start: Optional[int] = None, + msg_end: Optional[int] = None, + tools_start: Optional[int] = None, + tools_end: Optional[int] = None, + include_tools: bool = False, + ) -> dict[str, Any]: + if not session_id: + cls._raise_kv_cache_error("session_id is required") + if not parent_session_id: + cls._raise_kv_cache_error("parent_session_id is required") + + hint: dict[str, Any] = { + "session_id": session_id, + "parent_session_id": parent_session_id, + } + + if action is None: + if manage_request is not None: + cls._raise_kv_cache_error("manage_request is only valid when kv_action is set") + return hint + + if not isinstance(manage_request, bool): + cls._raise_kv_cache_error("manage_request must be explicitly set when kv_action is set") + + hint["context_management"] = { + "manage_request": manage_request, + "edits": cls._build_kv_target_edits( + action=action, + target=target, + msg_start=msg_start, + msg_end=msg_end, + tools_start=tools_start, + tools_end=tools_end, + include_tools=include_tools, + ), + } + return hint + + def build_kv_cache_affinity_invoke_kwargs( + self, + *, + session: object = None, + session_id: Optional[str] = None, + parent_session_id: Optional[str] = None, + enable_kv_cache_affinity: bool = False, + **_: Any, + ) -> dict: + if not enable_kv_cache_affinity or not self.supports_kv_cache_affinity(): + return {} + cache_id = session_id + if cache_id is None and session is not None and hasattr(session, "get_session_id"): + cache_id = session.get_session_id() + if not cache_id: + self._raise_kv_cache_error("session_id is required when KV cache affinity is enabled") + return { + "session_id": cache_id, + "parent_session_id": parent_session_id or cache_id, + } + + def build_kv_cache_invoke_kwargs( + self, + *, + session: object = None, + enable_kv_cache_release: bool = False, + **_: Any, + ) -> dict: + if not enable_kv_cache_release or not self.supports_kv_cache_release(): + return {} + extra: dict = {} + if session is not None and hasattr(session, "get_session_id"): + extra["session_id"] = session.get_session_id() + extra["enable_cache_sharing"] = True + return extra + + async def release( + self, + session_id: str, + messages: List, + messages_released_index: int, + *, + model: Optional[str] = None, + tools: Optional[List] = None, + tools_released_index: Optional[int] = None, + ) -> bool: + if not self.supports_kv_cache_release(): + return False + + kv_cache = self._kv_cache_config() + messages_dict = self._convert_messages_to_dict(messages) + tools_dict = self._convert_tools_to_dict(tools) + sanitized_messages = self._sanitize_tool_calls(messages_dict) + release_params = { + "model": model if model else self.model_config.model_name, + getattr(kv_cache, "session_field", "cache_salt"): session_id, + getattr(kv_cache, "enable_cache_sharing_field", "cache_sharing"): True, + "messages": sanitized_messages, + "messages_released_index": messages_released_index, + } + if tools_dict: + release_params["tools"] = tools_dict + if tools_released_index is not None: + release_params["tools_released_index"] = tools_released_index + + url = ( + f"{self.model_client_config.api_base.rstrip('/')}" + f"{getattr(kv_cache, 'release_endpoint', '/release_kv_cache')}" + ) + verify = ( + SslUtils.create_strict_ssl_context(self.model_client_config.ssl_cert) + if self.model_client_config.verify_ssl + else False + ) + headers = {"Content-Type": "application/json"} + async with httpx.AsyncClient( + proxy=UrlUtils.get_global_proxy_url(url), + verify=verify, + timeout=self.model_client_config.timeout, + ) as http_client: + response = await http_client.post(url, headers=headers, json=release_params) + if 200 <= response.status_code < 300: + return True + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg=( + f"OpenAI-compatible KV cache release failed: " + f"{response.status_code} {response.text}" + ), + ) + + async def evict_kvc(self, **kwargs) -> bool: + return await self._invoke_kv_cache_affinity_action("evict", **kwargs) + + async def offload_kvc(self, **kwargs) -> bool: + return await self._invoke_kv_cache_affinity_action("offload", **kwargs) + + async def prefetch_kvc(self, **kwargs) -> bool: + return await self._invoke_kv_cache_affinity_action("prefetch", **kwargs) + + async def _invoke_kv_cache_affinity_action( + self, + action: str, + *, + session_id: str, + parent_session_id: Optional[str] = None, + target: str = "session", + model: Optional[str] = None, + msg_start: Optional[int] = None, + msg_end: Optional[int] = None, + tools_start: Optional[int] = None, + tools_end: Optional[int] = None, + include_tools: bool = False, + timeout: Optional[float] = None, + max_attempts: Optional[int] = None, + **kwargs, + ) -> bool: + if not self.supports_kv_cache_affinity(): + return False + + params = self._build_request_params( + messages=[{"role": "user", "content": ""}], + tools=None, + temperature=None, + top_p=None, + model=model, + stop=None, + max_tokens=None, + stream=False, + session_id=session_id, + parent_session_id=parent_session_id or session_id, + kv_action=action, + target=target, + manage_request=True, + msg_start=msg_start, + msg_end=msg_end, + tools_start=tools_start, + tools_end=tools_end, + include_tools=include_tools, + **kwargs, + ) + self._move_openai_extra_body_extensions(params) + + attempts = self.model_client_config.max_retries if max_attempts is None else max(1, int(max_attempts)) + last_error = None + for attempt in range(attempts): + async_client = None + try: + async_client = self._create_async_openai_client(timeout=timeout) + if timeout is not None: + params["timeout"] = timeout + await async_client.chat.completions.create(**params) + return True + except Exception as exc: + last_error = exc + if attempt < attempts - 1: + continue + finally: + if async_client is not None and not self._use_shared_client(): + await async_client.close() + + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg=f"OpenAI-compatible KV cache {action} failed: {last_error}", + ) def _build_request_params( self, @@ -154,9 +709,30 @@ def _build_request_params( - if temperature is present, drop top_p - if temperature is not present but top_p is, keep top_p """ + session_id = kwargs.pop("session_id", None) + enable_cache_sharing = bool(kwargs.pop("enable_cache_sharing", False)) + parent_session_id = kwargs.pop("parent_session_id", None) + kv_action = kwargs.pop("kv_action", None) + kv_target = kwargs.pop("target", "session") + manage_request = kwargs.pop("manage_request", None) + msg_start = kwargs.pop("msg_start", None) + msg_end = kwargs.pop("msg_end", None) + tools_start = kwargs.pop("tools_start", None) + tools_end = kwargs.pop("tools_end", None) + include_tools = bool(kwargs.pop("include_tools", False)) + + is_session_manage_request = bool( + kv_action and manage_request is True and kv_target == "session" + ) + build_messages = ( + [{"role": "user", "content": ""}] + if is_session_manage_request + else messages + ) + # First, use the base implementation to build standard OpenAI-compatible params params = super()._build_request_params( - messages=messages, + messages=build_messages, tools=tools, temperature=temperature, top_p=top_p, @@ -177,8 +753,115 @@ def _build_request_params( params.pop("top_p", None) # If only one exists, keep as-is + params["messages"] = apply_message_transforms( + self.model_client_config, + params["messages"], + ) + + profile_name = self._endpoint_profile_name() + kv_mode = self._kv_cache_mode() + if profile_name == "siliconflow" or kv_mode in {"release", "affinity"}: + params["messages"] = self._sanitize_tool_calls(params["messages"]) + + if is_session_manage_request: + params["messages"] = [] + params.pop("tools", None) + params.pop("tool_choice", None) + + if kv_mode == "release" and enable_cache_sharing and session_id: + kv_cache = self._kv_cache_config() + params[getattr(kv_cache, "enable_cache_sharing_field", "cache_sharing")] = True + params[getattr(kv_cache, "session_field", "cache_salt")] = session_id + + if kv_mode == "affinity" and session_id: + kv_cache = self._kv_cache_config() + params[getattr(kv_cache, "affinity_field", "agent_hint")] = self._build_agent_hint( + session_id=session_id, + parent_session_id=parent_session_id or session_id, + action=kv_action, + target=kv_target, + manage_request=manage_request, + msg_start=msg_start, + msg_end=msg_end, + tools_start=tools_start, + tools_end=tools_end, + include_tools=include_tools, + ) + + self._apply_openrouter_profile(params) + return params + def _apply_openrouter_profile(self, params: dict) -> None: + if self._endpoint_profile_name() != "openrouter": + return + + model_name = params.get("model") + if not self._enable_openrouter_explicit_caching: + self._previous_openrouter_prompt_cache_messages = None + return + + if not _supports_openrouter_explicit_prompt_caching( + model_name, + self._openrouter_explicit_prompt_cache_providers, + ): + llm_logger.warning( + "OpenRouter explicit prompt caching is enabled but unsupported for model %s; " + "skipping cache_control markers.", + model_name, + ) + if self._enable_openrouter_1h_prompt_cache_ttl: + llm_logger.warning( + "OpenRouter 1h prompt-cache TTL is enabled but unsupported for model %s; " + "the ttl flag will not be added.", + model_name, + ) + self._previous_openrouter_prompt_cache_messages = None + return + + current_messages = params.get("messages") + if self._enable_openrouter_prompt_cache_prefix_matching: + previous_messages = self._previous_openrouter_prompt_cache_messages + self._previous_openrouter_prompt_cache_messages = ( + deepcopy(current_messages) if isinstance(current_messages, list) else None + ) + else: + previous_messages = None + self._previous_openrouter_prompt_cache_messages = None + + if isinstance(current_messages, list): + params["messages"] = deepcopy(current_messages) + if isinstance(params.get("tools"), list): + params["tools"] = deepcopy(params["tools"]) + + enable_1h_ttl = ( + self._enable_openrouter_1h_prompt_cache_ttl + and _supports_openrouter_1h_prompt_cache_ttl( + model_name, + self._openrouter_prompt_cache_1h_ttl_providers, + ) + ) + if self._enable_openrouter_1h_prompt_cache_ttl and not enable_1h_ttl: + llm_logger.warning( + "OpenRouter 1h prompt-cache TTL is enabled but unsupported for model %s; " + "using default ephemeral cache_control markers.", + model_name, + ) + _apply_openrouter_prompt_cache_control( + params, + previous_messages, + enable_1h_ttl=enable_1h_ttl, + ) + + @staticmethod + def _move_openai_extra_body_extensions(params: dict) -> None: + extra_body = dict(params.get("extra_body") or {}) + for key in list(_OPENAI_EXTRA_BODY_EXTENSION_FIELDS): + if key in params: + extra_body[key] = params.pop(key) + if extra_body: + params["extra_body"] = extra_body + def _create_async_openai_client(self, timeout: Optional[float] = None) -> "openai.AsyncOpenAI": """Acquire an ``AsyncOpenAI`` client for a request. @@ -246,7 +929,7 @@ def _build_async_openai_client(self, timeout: Optional[float] = None) -> "openai ) return AsyncOpenAI( - api_key=self.model_client_config.api_key, + api_key=self._resolved_api_key(), base_url=self.model_client_config.api_base, http_client=http_client, timeout=final_timeout, @@ -353,13 +1036,8 @@ async def invoke( if effective_headers: params["extra_headers"] = effective_headers - # OpenAI SDK drops unknown top-level create() args; vLLM needs return_token_ids in JSON body. - if "return_token_ids" in params: - extra_body = dict(params.get("extra_body") or {}) - extra_body["return_token_ids"] = params.pop("return_token_ids") - params["extra_body"] = extra_body - self._apply_model_specific_params(model, params) + self._move_openai_extra_body_extensions(params) if tracer_record_data: await tracer_record_data(llm_params=params) @@ -519,11 +1197,8 @@ async def stream( if effective_headers: params["extra_headers"] = effective_headers - if "return_token_ids" in params: - extra_body = dict(params.get("extra_body") or {}) - extra_body["return_token_ids"] = params.pop("return_token_ids") - params["extra_body"] = extra_body self._apply_model_specific_params(model, params) + self._move_openai_extra_body_extensions(params) if tracer_record_data: await tracer_record_data(llm_params=params) @@ -643,7 +1318,52 @@ async def generate_image( seed: int = 0, **kwargs ) -> ImageGenerationResponse: - pass + self._require_dashscope_media_profile("generate_image") + + try: + content_list = self._dashscope_image_content(messages) + import dashscope + from dashscope import MultiModalConversation + + api_params = { + "api_key": self.model_client_config.api_key, + "model": model or self.model_config.model_name, + "messages": [{"role": "user", "content": content_list}], + "result_format": "message", + "stream": False, + "size": size, + "n": n, + "prompt_extend": prompt_extend, + "watermark": watermark, + } + if negative_prompt: + api_params["negative_prompt"] = negative_prompt + if seed is not None: + api_params["seed"] = seed + api_params.update(kwargs) + + dashscope.base_http_api_url = self.model_client_config.api_base + response = MultiModalConversation.call(**api_params) + self._raise_for_dashscope_response(response, "image generation") + + image_urls = [] + for choice in (getattr(response, "output", None) or {}).get("choices", []): + content = (choice.get("message") or {}).get("content") or [] + for content_item in content: + if isinstance(content_item, dict) and content_item.get("image"): + image_urls.append(content_item["image"]) + if not image_urls: + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg="No images returned from DashScope API.", + ) + return ImageGenerationResponse( + model=api_params["model"], + images=image_urls, + created=None, + ) + except Exception as exc: + self._raise_dashscope_model_error("image generation", exc) async def generate_video( self, @@ -661,7 +1381,67 @@ async def generate_video( seed: Optional[int] = None, **kwargs ) -> VideoGenerationResponse: - pass + self._require_dashscope_media_profile("generate_video") + + try: + prompt = self._single_user_text(messages, "Video generation") + if not prompt.strip(): + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg="Video generation requires non-empty text content.", + ) + self._validate_dashscope_video_params(img_url=img_url, size=size, resolution=resolution) + import dashscope + from dashscope import VideoSynthesis + + api_params = { + "api_key": self.model_client_config.api_key, + "model": model or self.model_config.model_name, + "prompt": prompt, + "duration": duration, + "prompt_extend": prompt_extend, + "watermark": watermark, + } + if img_url: + api_params["img_url"] = img_url + if audio_url: + api_params["audio_url"] = audio_url + if size: + api_params["size"] = size + if resolution: + api_params["resolution"] = resolution + if negative_prompt: + api_params["negative_prompt"] = negative_prompt + if seed is not None: + api_params["seed"] = seed + api_params.update(kwargs) + + dashscope.base_http_api_url = self.model_client_config.api_base + response = VideoSynthesis.call(**api_params) + self._raise_for_dashscope_response(response, "video generation") + output = getattr(response, "output", None) or {} + video_url = self._get_mapping_or_attr(output, "video_url") or self._get_mapping_or_attr(output, "url") + if not video_url: + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg="No video URL returned from DashScope API.", + ) + usage = getattr(response, "usage", None) or {} + video_duration = ( + self._get_mapping_or_attr(usage, "duration") + or self._get_mapping_or_attr(usage, "output_video_duration") + or duration + ) + video_resolution = self._get_mapping_or_attr(usage, "size") or resolution or size + return VideoGenerationResponse( + model=api_params["model"], + video_url=video_url, + duration=video_duration, + resolution=video_resolution, + format="mp4", + ) + except Exception as exc: + self._raise_dashscope_model_error("video generation", exc) async def generate_speech( self, @@ -672,7 +1452,303 @@ async def generate_speech( language_type: Optional[str] = "Auto", **kwargs ) -> AudioGenerationResponse: - pass + self._require_dashscope_media_profile("generate_speech") + + try: + text = self._single_user_text(messages, "Speech generation") + if not text.strip(): + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg="Speech generation requires non-empty text content.", + ) + self._validate_dashscope_speech_params(voice=voice, language_type=language_type) + import dashscope + from dashscope import MultiModalConversation + + api_params = { + "api_key": self.model_client_config.api_key, + "model": model or self.model_config.model_name, + "text": text, + "voice": voice, + "language_type": language_type, + } + api_params.update(kwargs) + + dashscope.base_http_api_url = self.model_client_config.api_base + response = MultiModalConversation.call(**api_params) + self._raise_for_dashscope_response(response, "speech generation") + + audio_url, audio_data, audio_format = self._extract_dashscope_audio(response) + if not audio_url and not audio_data: + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg="No audio URL or data returned from DashScope API.", + ) + return AudioGenerationResponse( + model=api_params["model"], + audio_url=audio_url, + audio_data=audio_data, + format=audio_format, + ) + except Exception as exc: + self._raise_dashscope_model_error("speech generation", exc) + + def _require_dashscope_media_profile(self, operation: str) -> None: + if self._endpoint_profile_name() == "dashscope": + return + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg=f"{operation} is not supported by OpenAIModelClient for this endpoint_profile.", + ) + + @staticmethod + def _raise_dashscope_model_error(operation: str, exc: Exception): + if isinstance(exc, ModelError): + raise exc + error_msg = f"Unexpected error during DashScope {operation}: {str(exc)}" + logger.error(error_msg, exc_info=True) + raise ModelError( + StatusCode.MODEL_CALL_FAILED, + msg=error_msg, + cause=exc, + ) from exc + + @staticmethod + def _validate_dashscope_speech_params(*, voice: Optional[str], language_type: Optional[str]) -> None: + if voice not in DASHSCOPE_VOICE: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Unsupported DashScope voice: {voice}.", + ) + if language_type not in DASHSCOPE_LANGUAGE_TYPE: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Unsupported DashScope language_type: {language_type}.", + ) + + @staticmethod + def _validate_dashscope_video_params( + *, + img_url: Optional[str], + size: Optional[str], + resolution: Optional[str], + ) -> None: + if img_url: + if size: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg="Image-to-video generation uses resolution; do not pass size.", + ) + return + if resolution: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg="Text-to-video generation uses size; do not pass resolution.", + ) + + @staticmethod + def _single_user_message(messages: List[UserMessage], operation: str) -> UserMessage: + if not messages or len(messages) != 1: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"{operation} requires exactly one UserMessage.", + ) + message = messages[0] + if not isinstance(message, UserMessage): + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"{operation} requires a UserMessage.", + ) + return message + + @classmethod + def _single_user_text(cls, messages: List[UserMessage], operation: str) -> str: + message = cls._single_user_message(messages, operation) + content = message.content + if isinstance(content, str): + return content + if isinstance(content, list): + text_parts = [] + for item in content: + if isinstance(item, str): + text_parts.append(item) + elif isinstance(item, dict) and item.get("text"): + text_parts.append(str(item["text"])) + return "\n".join(text_parts) + return str(content) + + @classmethod + def _dashscope_image_content(cls, messages: List[UserMessage]) -> list[dict]: + message = cls._single_user_message(messages, "Image generation") + content = message.content + content_list: list[dict] = [] + image_count = 0 + text_count = 0 + + if isinstance(content, str): + content_list.append({"text": content}) + text_count += 1 + elif isinstance(content, list): + for item in content: + if isinstance(item, str): + content_list.append({"text": item}) + text_count += 1 + continue + if not isinstance(item, dict): + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Content item must be string or dict, but got {type(item).__name__}.", + ) + if cls._dashscope_is_text_item(item): + content_list.append({"text": item["text"]}) + text_count += 1 + continue + image_value = cls._dashscope_image_value(item) + if image_value: + content_list.append({"image": image_value}) + image_count += 1 + continue + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=( + "Content dict must contain a non-empty 'text', 'image', or " + f"'image_url' value, but got: {list(item.keys())}" + ), + ) + else: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Message content must be string or list, but got {type(content).__name__}.", + ) + + if text_count == 0: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg="Image generation requires at least one text prompt.", + ) + if image_count > 3: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Image generation supports at most 3 input images, but got {image_count}.", + ) + return content_list + + @staticmethod + def _dashscope_is_text_item(item: dict) -> bool: + keys = set(item) + if keys == {"text"}: + return True + if keys == {"type", "text"} and item.get("type") == "text": + return True + if "text" in item: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Content dict with 'text' must not contain extra keys, but got: {list(item.keys())}", + ) + return False + + @staticmethod + def _dashscope_image_value(item: dict) -> Optional[str]: + keys = set(item) + image_value = item.get("image") + if "image" in item: + if keys != {"image"}: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=f"Content dict with 'image' must not contain extra keys, but got: {list(item.keys())}", + ) + if isinstance(image_value, str) and image_value.strip(): + return image_value + return None + + image_url = item.get("image_url") + if "image_url" in item: + allowed_keys = {"image_url"} if "type" not in item else {"type", "image_url"} + if keys != allowed_keys: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=( + "Content dict with 'image_url' must not contain extra keys, " + f"but got: {list(item.keys())}" + ), + ) + if "type" in item and item.get("type") not in {"image_url", "input_image", "image"}: + return None + if isinstance(image_url, str) and image_url.strip(): + return image_url + if isinstance(image_url, dict): + url = image_url.get("url") + if isinstance(url, str) and url.strip(): + return url + return None + + if item.get("type") in {"image", "input_image"}: + if keys != {"type", "url"}: + raise build_error( + StatusCode.MODEL_INVOKE_PARAM_ERROR, + error_msg=( + "Content dict with image type must contain only 'type' and 'url', " + f"but got: {list(item.keys())}" + ), + ) + url = item.get("url") + if isinstance(url, str) and url.strip(): + return url + return None + + @classmethod + def _extract_dashscope_audio(cls, response: Any) -> tuple[Optional[str], Any, Optional[str]]: + audio_url = None + audio_data = None + audio_format = None + output = getattr(response, "output", None) or {} + + audio = cls._get_mapping_or_attr(output, "audio") + if audio: + audio_url = cls._get_mapping_or_attr(audio, "url") + audio_data = cls._get_mapping_or_attr(audio, "data") + if isinstance(audio_data, str): + audio_data = audio_data.encode("utf-8") + if audio_url: + lower_url = str(audio_url).lower() + if lower_url.endswith(".wav"): + audio_format = "wav" + elif lower_url.endswith(".mp3"): + audio_format = "mp3" + elif lower_url.endswith(".pcm"): + audio_format = "pcm" + + choices = cls._get_mapping_or_attr(output, "choices") or [] + for choice in choices: + content = (cls._get_mapping_or_attr(cls._get_mapping_or_attr(choice, "message") or {}, "content") or []) + for content_item in content: + if not isinstance(content_item, dict): + continue + audio_url = content_item.get("audio") or content_item.get("audio_url") or audio_url + audio_data = content_item.get("audio_data") or audio_data + audio_format = content_item.get("format") or audio_format + return audio_url, audio_data, audio_format + + @staticmethod + def _get_mapping_or_attr(value: Any, key: str) -> Any: + if isinstance(value, dict): + return value.get(key) + return getattr(value, key, None) + + @staticmethod + def _raise_for_dashscope_response(response: Any, operation: str) -> None: + status_code = getattr(response, "status_code", None) + if status_code == 200: + return + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg=( + f"DashScope {operation} failed. " + f"HTTP status: {status_code}, " + f"Error code: {getattr(response, 'code', None)}, " + f"Error message: {getattr(response, 'message', None)}" + ), + ) async def _astream_with_parser( self, diff --git a/openjiuwen/core/foundation/llm/model_clients/openrouter_model_client.py b/openjiuwen/core/foundation/llm/model_clients/openrouter_model_client.py deleted file mode 100644 index 5b25bc22b..000000000 --- a/openjiuwen/core/foundation/llm/model_clients/openrouter_model_client.py +++ /dev/null @@ -1,337 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -from copy import deepcopy -from typing import Any, Mapping, Optional, Union - -from openjiuwen.core.common.logging import llm_logger -from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient -from openjiuwen.core.foundation.llm.schema.message import BaseMessage -from openjiuwen.core.foundation.llm.schema.config import ProviderType -from openjiuwen.core.foundation.tool import ToolInfo - - -OPENROUTER_ATTRIBUTION_HEADER_KEYS = frozenset({ - "http-referer", - "x-openrouter-title", - "x-openrouter-categories", -}) -OPENROUTER_EXPLICIT_PROMPT_CACHING_PROVIDERS = frozenset({ - "anthropic", - "qwen", -}) -OPENROUTER_1H_PROMPT_CACHE_TTL_PROVIDERS = frozenset({ - "anthropic", -}) - - -def _openrouter_model_provider(model: Optional[str]) -> Optional[str]: - if not model or "/" not in model: - return None - return model.split("/", 1)[0].lstrip("~").lower() - - -def _normalize_openrouter_provider_set(value: Any, default: frozenset[str]) -> frozenset[str]: - if value is None: - return default - if isinstance(value, str): - values = value.split(",") - else: - values = value - try: - return frozenset(str(provider).strip().lower() for provider in values if str(provider).strip()) - except TypeError: - return default - - -def _supports_openrouter_explicit_prompt_caching( - model: Optional[str], - supported_providers: frozenset[str] = OPENROUTER_EXPLICIT_PROMPT_CACHING_PROVIDERS, -) -> bool: - provider = _openrouter_model_provider(model) - return provider in supported_providers - - -def _supports_openrouter_1h_prompt_cache_ttl( - model: Optional[str], - supported_providers: frozenset[str] = OPENROUTER_1H_PROMPT_CACHE_TTL_PROVIDERS, -) -> bool: - return _openrouter_model_provider(model) in supported_providers - - -def _without_cache_control(value: Any) -> Any: - """Remove cache_control and normalize marked text blocks for comparison. - - OpenRouter prompt-cache markers can turn plain text content into a typed text block. - For the purpose of prefix matching & caching, these are equivalent and normalized: - - ``{"role": "user", "content": "text content"}`` - ``{"role": "user", "content": [{"type": "text", "text": "text content"}]}`` - - Only used for longest prefix comparison - """ - if isinstance(value, dict): - normalized = { - key: _without_cache_control(item) - for key, item in value.items() - if key != "cache_control" - } - if normalized.get("type") == "text" and set(normalized) <= {"type", "text"}: - return normalized.get("text", "") - content = normalized.get("content") - if isinstance(content, list) and len(content) == 1 and isinstance(content[0], str): - normalized["content"] = content[0] - return normalized - if isinstance(value, list): - return [_without_cache_control(item) for item in value] - return value - - -def _contains_cache_control(value: Any) -> bool: - if isinstance(value, dict): - if "cache_control" in value: - return True - return any(_contains_cache_control(item) for item in value.values()) - if isinstance(value, list): - return any(_contains_cache_control(item) for item in value) - return False - - -def _build_cache_control_marker(enable_1h_ttl: bool = False) -> dict: - marker = {"type": "ephemeral"} - if enable_1h_ttl: - marker["ttl"] = "1h" - return marker - - -def _add_cache_control_marker(block: dict, enable_1h_ttl: bool = False) -> dict: - block.setdefault("cache_control", _build_cache_control_marker(enable_1h_ttl)) - return block - - -def _mark_message_with_cache_control(message: dict, enable_1h_ttl: bool = False) -> None: - """Attach OpenRouter prompt-cache metadata to the final content block.""" - if _contains_cache_control(message): - return - - content = message.get("content") - if isinstance(content, list): - if not content: - return - - last_index = len(content) - 1 - last_block = content[last_index] - if isinstance(last_block, dict): - _add_cache_control_marker(last_block, enable_1h_ttl) - else: - content[last_index] = _add_cache_control_marker({ - "type": "text", - "text": last_block if isinstance(last_block, str) else str(last_block), - }, enable_1h_ttl) - return - - message["content"] = [_add_cache_control_marker({ - "type": "text", - "text": content if isinstance(content, str) else ("" if content is None else str(content)), - }, enable_1h_ttl)] - - -def _longest_prefix_overlap_index(previous_messages: Optional[list], current_messages: list) -> Optional[int]: - if not previous_messages: - return None - - overlap = 0 - for previous, current in zip(previous_messages, current_messages): - if _without_cache_control(previous) != _without_cache_control(current): - break - overlap += 1 - - if overlap == 0: - return None - return overlap - 1 - - -def _apply_openrouter_prompt_cache_control( - params: dict, - previous_messages: Optional[list], - *, - enable_1h_ttl: bool = False, -) -> None: - tools = params.get("tools") - if isinstance(tools, list) and tools and isinstance(tools[-1], dict): - _add_cache_control_marker(tools[-1], enable_1h_ttl) - - messages = params.get("messages") - if not isinstance(messages, list) or not messages: - return - - prefix_index = _longest_prefix_overlap_index(previous_messages, messages) - - if isinstance(messages[0], dict): - _mark_message_with_cache_control(messages[0], enable_1h_ttl) - - if prefix_index is not None and isinstance(messages[prefix_index], dict): - _mark_message_with_cache_control(messages[prefix_index], enable_1h_ttl) - - if isinstance(messages[-1], dict): - _mark_message_with_cache_control(messages[-1], enable_1h_ttl) - - -class OpenRouterModelClient(OpenAIModelClient): - """OpenRouter-specific model client with configurable App Attribution headers. - - This client does NOT provide default attribution header values. The calling - application (e.g. JiuwenSwarm) is responsible for injecting attribution - headers via ``ModelClientConfig.custom_headers``. - - This client provides a protection mechanism: once attribution headers are - configured at the config level, they cannot be overridden by request-level - headers, following the OpenRouter App Attribution specification. - """ - __client_name__ = [ProviderType.OpenRouter.value] - - _ATTRIBUTION_PROTECTED_KEYS: frozenset[str] = OPENROUTER_ATTRIBUTION_HEADER_KEYS - - def __init__( - self, - model_config, - model_client_config, - ): - super().__init__(model_config, model_client_config) - extra = model_client_config.__pydantic_extra__ or {} - self._enable_explicit_caching = extra.get( - "openrouter_enable_explicit_prompt_caching", - True, - ) - # Prefix matching stores previous messages on this client instance, so - # parallel calls on one shared client can compare against another call's - # messages and miss the intended prefix. Disable it in that case. - # Note that Anthropic and Qwen providers allow up to 4 cache_control flags, - # with no penalty for using extra flags. Hence, there's no reason not - # to try to add a 4th flag in a reasonable place. - self._enable_prompt_cache_prefix_matching = extra.get( - "openrouter_enable_prompt_cache_prefix_matching", - True, - ) - self._enable_1h_prompt_cache_ttl = extra.get( - "openrouter_enable_1h_prompt_cache_ttl", - False, - ) - self._explicit_prompt_cache_providers = _normalize_openrouter_provider_set( - extra.get("openrouter_explicit_prompt_cache_providers"), - OPENROUTER_EXPLICIT_PROMPT_CACHING_PROVIDERS, - ) - self._prompt_cache_1h_ttl_providers = _normalize_openrouter_provider_set( - extra.get("openrouter_prompt_cache_1h_ttl_providers"), - OPENROUTER_1H_PROMPT_CACHE_TTL_PROVIDERS, - ) - self._previous_prompt_cache_messages: Optional[list] = None - - @classmethod - def _build_request_headers( - cls, - base_headers: Optional[Mapping[str, Any]], - request_headers: Optional[Mapping[str, Any]], - ) -> dict[str, str]: - """Merge request-level headers but protect attribution keys from override.""" - effective = dict(base_headers or {}) - if request_headers: - protected_lower = cls._ATTRIBUTION_PROTECTED_KEYS - for key, value in request_headers.items(): - if key.lower() in protected_lower: - continue - effective[key] = str(value) - return effective - - def _build_request_params( - self, - *, - messages: Union[str, list[BaseMessage], list[dict]], - tools: Union[list[ToolInfo], list[dict], None], - temperature: Optional[float], - top_p: Optional[float], - model: Optional[str], - stop: Union[Optional[str], None], - max_tokens: Optional[int], - stream: bool, - **kwargs - ) -> dict: - """Build params and add OpenRouter cache breakpoints. - - For supported models, explicit prompt caching marks the final tool, the - first message, and the final message. When configured, the client also - marks the longest message-prefix overlap with the previous request. - """ - params = super()._build_request_params( - messages=messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=stream, - **kwargs, - ) - - model_name = params.get("model") - if not self._enable_explicit_caching: - self._previous_prompt_cache_messages = None - return params - - if not _supports_openrouter_explicit_prompt_caching( - model_name, - self._explicit_prompt_cache_providers, - ): - llm_logger.warning( - "OpenRouter explicit prompt caching is enabled but unsupported for model %s; " - "skipping cache_control markers.", - model_name, - ) - if self._enable_1h_prompt_cache_ttl: - llm_logger.warning( - "OpenRouter 1h prompt-cache TTL is enabled but unsupported for model %s; " - "the ttl flag will not be added.", - model_name, - ) - self._previous_prompt_cache_messages = None - return params - - current_messages = params.get("messages") - if self._enable_prompt_cache_prefix_matching: - previous_messages = self._previous_prompt_cache_messages - self._previous_prompt_cache_messages = ( - deepcopy(current_messages) if isinstance(current_messages, list) else None - ) - else: - previous_messages = None - self._previous_prompt_cache_messages = None - - if isinstance(current_messages, list): - params["messages"] = deepcopy(current_messages) - if isinstance(params.get("tools"), list): - params["tools"] = deepcopy(params["tools"]) - - enable_1h_ttl = ( - self._enable_1h_prompt_cache_ttl - and _supports_openrouter_1h_prompt_cache_ttl( - model_name, - self._prompt_cache_1h_ttl_providers, - ) - ) - if self._enable_1h_prompt_cache_ttl and not enable_1h_ttl: - llm_logger.warning( - "OpenRouter 1h prompt-cache TTL is enabled but unsupported for model %s; " - "using default ephemeral cache_control markers.", - model_name, - ) - _apply_openrouter_prompt_cache_control( - params, - previous_messages, - enable_1h_ttl=enable_1h_ttl, - ) - return params - - @staticmethod - def _extract_reasoning_content(msg_or_delta: Any) -> Optional[str]: - return getattr(msg_or_delta, 'reasoning', None) or getattr(msg_or_delta, 'reasoning_content', None) diff --git a/openjiuwen/core/foundation/llm/model_clients/siliconflow_model_client.py b/openjiuwen/core/foundation/llm/model_clients/siliconflow_model_client.py deleted file mode 100644 index 37eb25d68..000000000 --- a/openjiuwen/core/foundation/llm/model_clients/siliconflow_model_client.py +++ /dev/null @@ -1,727 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - -from typing import List, Optional, AsyncIterator, Union, Dict, Any -from contextlib import asynccontextmanager -import aiohttp - -from openjiuwen.core.common.exception.codes import StatusCode -from openjiuwen.core.common.exception.errors import build_error -from openjiuwen.core.common.logging import llm_logger, LogEventType -from openjiuwen.core.common.security.ssl_utils import SslUtils -from openjiuwen.core.common.security.url_utils import UrlUtils -from openjiuwen.core.foundation.llm.schema.config import ProviderType -from openjiuwen.core.foundation.llm.schema import ImageGenerationResponse, VideoGenerationResponse, \ - AudioGenerationResponse -from openjiuwen.core.foundation.llm.schema.message import ( - BaseMessage, - AssistantMessage, - UserMessage, - UsageMetadata -) -from openjiuwen.core.foundation.llm.schema.message_chunk import AssistantMessageChunk -from openjiuwen.core.foundation.llm.schema.tool_call import ToolCall -from openjiuwen.core.foundation.tool import ToolInfo -from openjiuwen.core.foundation.llm.output_parsers.output_parser import BaseOutputParser -from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig -from openjiuwen.core.runner.callback import trigger -from openjiuwen.core.runner.callback.events import LLMCallEvents - - -class SiliconFlowModelClient(BaseModelClient): - """SiliconFlow API client supporting GPT models and OpenAI-compatible services.""" - __client_name__ = ProviderType.SiliconFlow.value - - def __init__(self, model_config: ModelRequestConfig, model_client_config: ModelClientConfig): - super().__init__(model_config, model_client_config) - - def _get_client_name(self) -> str: - """Get client name for error messages""" - return "SiliconFlow client" - - def _build_and_sanitize_params( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - stream: bool = False, - **kwargs - ) -> Dict[str, Any]: - params = self._build_request_params( - messages=messages, - tools=tools, - model=model, - temperature=temperature, - top_p=top_p, - stop=stop, - max_tokens=max_tokens, - stream=stream, - **kwargs - ) - # Sanitize tool_calls in messages - params["messages"] = self._sanitize_tool_calls(params["messages"]) - return params - - @asynccontextmanager - async def _apost(self, params: Dict[str, Any], timeout: Optional[float] = None): - """Create a POST request context for SiliconFlow API. - - Args: - params: Request parameters - timeout: Optional timeout override for this specific request - """ - # Validate API base URL - UrlUtils.check_url_is_valid(self.model_client_config.api_base) - - # Build complete API URL - auto-append /chat/completions if not present - api_url = self.model_client_config.api_base.rstrip('/') - if not api_url.endswith('/chat/completions'): - api_url = f"{api_url}/chat/completions" - - ssl_verify, ssl_cert = self.model_client_config.verify_ssl, self.model_client_config.ssl_cert - if ssl_verify: - ssl_context = SslUtils.create_strict_ssl_context(ssl_cert) - connector = aiohttp.TCPConnector(ssl=ssl_context) - else: - connector = aiohttp.TCPConnector(ssl=False) - - # Use method-level timeout if provided, otherwise use config timeout - final_timeout = timeout if timeout is not None else self.model_client_config.timeout - timeout_obj = aiohttp.ClientTimeout(total=final_timeout) - - llm_logger.info( - "Before create siliconflow client, model client config params ready.", - event_type=LogEventType.LLM_CALL_START, - timeout=final_timeout - ) - - async with aiohttp.ClientSession(connector=connector) as session: - async with session.post( - url=api_url, - proxy=UrlUtils.get_global_proxy_url(api_url), - headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {self.model_client_config.api_key}" - }, - json=params, - allow_redirects=False, - timeout=timeout_obj - ) as response: - response.raise_for_status() - yield response - - async def invoke( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - output_parser: Optional[BaseOutputParser] = None, - timeout: float = None, - **kwargs - ) -> AssistantMessage: - """Async invoke SiliconFlow API - - Args: - :param output_parser: - :param model: - :param stop: - :param temperature: - :param tools: - :param messages: - :param top_p: - :param max_tokens: - :param timeout: - **kwargs: Additional parameters - - Returns: - AssistantMessage: Model response - """ - tracer_record_data = kwargs.pop("tracer_record_data", None) - params = self._build_and_sanitize_params( - messages=messages, - tools=tools, - model=model, - temperature=temperature, - top_p=top_p, - stop=stop, - max_tokens=max_tokens, - stream=False, - **kwargs - ) - if tracer_record_data: - await tracer_record_data(llm_params=params) - llm_logger.info( - "LLM request params ready.", - event_type=LogEventType.LLM_CALL_START, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=False - ) - - try: - await trigger( - LLMCallEvents.LLM_INPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - frequency_penalty=params.get("frequency_penalty"), - presence_penalty=params.get("presence_penalty"), - stop=params.get("stop")) - - async with self._apost(params, timeout=timeout) as response: - data = await response.json() - llm_logger.info( - "SiliconFlow API response received.", - event_type=LogEventType.LLM_CALL_END, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=False, - metadata={"response": data} - ) - - # Parse response and apply output parser - llm_logger.info( - "Before parse response with output parser.", - event_type=LogEventType.LLM_CALL_END, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=False, - metadata={"output_parser": str(output_parser)} - ) - assistant_message = await self._parse_response(data, output_parser) - - if tracer_record_data: - await tracer_record_data(llm_response=assistant_message) - - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - response=assistant_message.content, - usage=assistant_message.usage_metadata, - tool_calls=assistant_message.tool_calls) - - return assistant_message - - except Exception as e: - await trigger( - LLMCallEvents.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=False, - error=e) - llm_logger.error( - "SiliconFlow API async invoke error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=False, - exception=str(e) - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=f"siliconFlow API async invoke error: {str(e)}" - ) from e - - async def stream( - self, - messages: Union[str, List[BaseMessage], List[dict]], - *, - tools: Union[List[ToolInfo], List[dict], None] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - model: str = None, - max_tokens: Optional[int] = None, - stop: Union[Optional[str], None] = None, - output_parser: Optional[BaseOutputParser] = None, - timeout: float = None, - **kwargs - ) -> AsyncIterator[AssistantMessageChunk]: - """Async streaming invoke silicon flow API - - Args: - :param output_parser: - :param model: - :param stop: - :param temperature: - :param tools: - :param messages: - :param top_p: - :param max_tokens: - :param timeout: - **kwargs: Additional parameters - - Yields: - AssistantMessageChunk: Streaming response chunk - """ - tracer_record_data = kwargs.pop("tracer_record_data", None) - - params = self._build_and_sanitize_params( - messages=messages, - tools=tools, - temperature=temperature, - top_p=top_p, - model=model, - stop=stop, - max_tokens=max_tokens, - stream=True, - **kwargs - ) - - if tracer_record_data: - await tracer_record_data(llm_params=params) - - try: - await trigger( - LLMCallEvents.LLM_INPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - frequency_penalty=params.get("frequency_penalty"), - presence_penalty=params.get("presence_penalty"), - stop=params.get("stop"), - is_stream=True) - - final_message = None - async with self._apost(params, timeout=timeout) as response: - if output_parser: - # Use streaming parser - async for parsed_result in self._astream_with_parser(response, output_parser): - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - result=parsed_result, - is_stream=True) - if final_message: - final_message = final_message + parsed_result - else: - final_message = parsed_result - yield parsed_result - else: - # Direct return without parser - async for line in response.content: - if line: - parsed_chunk = self._parse_stream_chunk(line) - if parsed_chunk: - await trigger( - LLMCallEvents.LLM_OUTPUT, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - result=parsed_chunk, - is_stream=True) - if final_message: - final_message = final_message + parsed_chunk - else: - final_message = parsed_chunk - yield parsed_chunk - if tracer_record_data: - await tracer_record_data(llm_response=final_message) - - except Exception as e: - await trigger( - LLMCallEvents.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - is_stream=True, - error=e) - llm_logger.error( - "SiliconFlow API async stream error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=params.get("model"), - model_provider=self.model_client_config.client_provider, - messages=params.get("messages"), - tools=params.get("tools"), - temperature=params.get("temperature"), - top_p=params.get("top_p"), - max_tokens=params.get("max_tokens"), - is_stream=True, - exception=str(e) - ) - raise build_error( - StatusCode.MODEL_CALL_FAILED, - error_msg=f"siliconFlow API async stream error: {str(e)}" - ) from e - - async def generate_image( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - size: Optional[str] = "1664*928", - negative_prompt: Optional[str] = None, - n: Optional[int] = 1, - prompt_extend: bool = True, - watermark: bool = False, - seed: int = 0, - **kwargs - ) -> ImageGenerationResponse: - pass - - async def generate_video( - self, - messages: List[UserMessage], - *, - img_url: Optional[str] = None, - audio_url: Optional[str] = None, - model: Optional[str] = None, - size: Optional[str] = None, - resolution: Optional[str] = None, - duration: Optional[int] = 5, - prompt_extend: bool = True, - watermark: bool = False, - negative_prompt: Optional[str] = None, - seed: Optional[int] = None, - **kwargs - ) -> VideoGenerationResponse: - pass - - async def generate_speech( - self, - messages: List[UserMessage], - *, - model: Optional[str] = None, - voice: Optional[str] = "Cherry", - language_type: Optional[str] = "Auto", - **kwargs - ) -> AudioGenerationResponse: - pass - - async def _astream_with_parser( - self, - response_stream, - output_parser: BaseOutputParser - ) -> AsyncIterator[AssistantMessageChunk]: - """Process streaming response with output parser - - Strategy: - 1. Immediately yield each raw chunk, maintaining streaming characteristics (content is incremental) - 2. Accumulate all content - 3. **Attempt to parse accumulated content every time a new chunk is received** - 4. When parsing succeeds, output parser_content and clear buffer (implementing incremental output) - 5. When parsing fails, parser_content is None, continue accumulating - """ - accumulated_content = "" - - async for line in response_stream.content: - if line: - parsed_chunk = self._parse_stream_chunk(line) - if parsed_chunk: - # Accumulate content - if parsed_chunk.content: - accumulated_content += parsed_chunk.content - - # Attempt to parse accumulated content every time - parser_content = None - if accumulated_content and output_parser: - try: - current_parsed_result = await output_parser.parse(accumulated_content) - # When parsing succeeds, output result and clear buffer - if current_parsed_result is not None: - parser_content = current_parsed_result - accumulated_content = "" # Clear buffer to implement incremental output - except Exception as e: - llm_logger.debug( - "Stream parser attempt error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=True, - exception=str(e) - ) - parser_content = None - - # Create new chunk with original content and parser_content - chunk_with_parser = AssistantMessageChunk( - content=parsed_chunk.content, # Keep original content increment unchanged - reasoning_content=parsed_chunk.reasoning_content, - tool_calls=parsed_chunk.tool_calls, - usage_metadata=parsed_chunk.usage_metadata, - finish_reason=parsed_chunk.finish_reason, - parser_content=parser_content # Has value when parsing succeeds, otherwise None - ) - - yield chunk_with_parser - - async def _parse_response( - self, - response: Any, - parser: Optional[BaseOutputParser] = None - ) -> AssistantMessage: - """Parse SiliconFlow API response - - Args: - response: SiliconFlow API response object (dict from JSON) - parser: Optional output parser, only parses content field - - Returns: - AssistantMessage: Parsed assistant message - - Note: - Non-streaming finish_reason can only be "stop" or "tool_calls": - - stop: Model generation completed without tool calls - - tool_calls: Model generation completed with tool calls - """ - choice = response.get("choices", [{}])[0] - message = choice.get("message", {}) - - # Get content - content = "" if message.get("content") is None else message.get("content") - - # Get reasoning_content (if exists) - reasoning_content = message.get("reasoning_content", None) - - # Parse tool_calls - tool_calls = [] - if message.get("tool_calls"): - for idx, tc in enumerate(message.get("tool_calls", [])): - function = tc.get("function", {}) - tool_call = ToolCall( - id=tc.get("id", "") or "", - type="function", - name=function.get("name", "") or "", - arguments=function.get("arguments", "") or "", - index=tc.get("index", idx) - ) - tool_calls.append(tool_call) - - # Build UsageMetadata - usage_metadata = None - usage = response.get("usage") - if usage: - # Extract basic token information - input_tokens = usage.get("prompt_tokens", 0) or 0 - output_tokens = usage.get("completion_tokens", 0) or 0 - total_tokens = usage.get("total_tokens", 0) or 0 - - # Extract cost information if available - input_cost, output_cost, total_cost = self._extract_cost_info(usage) - - usage_metadata = UsageMetadata( - model_name=self.model_config.model_name, - input_tokens=input_tokens, - output_tokens=output_tokens, - total_tokens=total_tokens, - cache_tokens=self._extract_cache_tokens(usage), - reasoning_tokens=self._extract_reasoning_tokens(usage), - input_cost=input_cost, - output_cost=output_cost, - total_cost=total_cost, - ) - - # Apply output parser (only parse content field) - parser_content = None - llm_logger.info( - "Before parse content with parser.", - event_type=LogEventType.LLM_CALL_END, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - response_content=content, - is_stream=False - ) - llm_logger.info( - "Before parse content with parser config.", - event_type=LogEventType.LLM_CALL_END, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - metadata={"parser": str(parser)} - ) - if parser and content: - try: - parser_content = await parser.parse(content) - llm_logger.info( - "Parser parse success.", - event_type=LogEventType.LLM_CALL_END, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - metadata={"parser_content": parser_content} - ) - except Exception as e: - llm_logger.warning( - "Parser parse error.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=False, - exception=str(e) - ) - parser_content = None - - return AssistantMessage( - content=content, - tool_calls=tool_calls if tool_calls else None, - usage_metadata=usage_metadata, - finish_reason="tool_calls" if tool_calls else "stop", - reasoning_content=reasoning_content, - parser_content=parser_content - ) - - def _parse_stream_chunk(self, chunk: Any) -> Optional[AssistantMessageChunk]: - """Parse SiliconFlow streaming response chunk - - Args: - chunk: SiliconFlow streaming response chunk (bytes) - - Returns: - AssistantMessageChunk or None - """ - import json - - # Handle SSE format: data: {...} - if chunk.startswith(b"data: "): - chunk = chunk[6:] - - # Handle [DONE] marker - if chunk.strip() == b"[DONE]": - return None - - try: - data = json.loads(chunk.decode("utf-8")) - choice = data.get("choices", [{}])[0] - delta = choice.get("delta", {}) - - # Extract content - content = delta.get("content", None) or "" - reasoning_content = delta.get("reasoning_content", None) - - # Parse tool_calls delta - tool_calls = [] - tool_calls_delta = delta.get("tool_calls") - if tool_calls_delta: - for tc_delta in tool_calls_delta: - index = tc_delta.get("index", 0) - tool_call_id = tc_delta.get("id", "") - function_delta = tc_delta.get("function", {}) - name_delta = function_delta.get("name", "") - args_delta = function_delta.get("arguments", "") - - tool_call = ToolCall( - id=tool_call_id or "", - type="function", - name=name_delta or "", - arguments=args_delta or "", - index=index - ) - tool_calls.append(tool_call) - - # Build usage_metadata (usually only in the last chunk) - usage_metadata = None - usage = data.get("usage") - if usage: - # Extract cost information if available - input_cost, output_cost, total_cost = self._extract_cost_info(usage) - usage_metadata = UsageMetadata( - model_name=self.model_config.model_name, - input_tokens=usage.get("prompt_tokens", 0) or 0, - output_tokens=usage.get("completion_tokens", 0) or 0, - total_tokens=usage.get("total_tokens", 0) or 0, - cache_tokens=self._extract_cache_tokens(usage), - reasoning_tokens=self._extract_reasoning_tokens(usage), - input_cost=input_cost, - output_cost=output_cost, - total_cost=total_cost, - ) - - # Skip empty chunks - is_response_empty = ( - not content - and not reasoning_content - and not tool_calls - and not usage_metadata - ) - - if is_response_empty: - return None - - return AssistantMessageChunk( - content=content, - reasoning_content=reasoning_content, - tool_calls=tool_calls if tool_calls else None, - usage_metadata=usage_metadata, - finish_reason=choice.get("finish_reason") or "null" - ) - except json.JSONDecodeError: - return None - except Exception as e: - llm_logger.warning( - "Error parsing stream chunk.", - event_type=LogEventType.LLM_CALL_ERROR, - model_name=self.model_config.model_name, - model_provider=self.model_client_config.client_provider, - is_stream=True, - exception=str(e) - ) - return None - - def _sanitize_tool_calls(self, messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Sanitize tool_calls in messages, keep OpenAI standard fields: - id, type, function.name, function.arguments - Force type to "function" - - Args: - messages: List of message dictionaries - - Returns: - Sanitized message list - """ - for msg in messages: - if msg.get("role") != "assistant": - continue - tool_calls = msg.get("tool_calls") - if not isinstance(tool_calls, list): - continue - - cleaned = [] - for tc in tool_calls: - if not isinstance(tc, dict): - continue - # Extract only valid fields - func = tc.get("function", {}) - cleaned.append({ - "id": tc.get("id", ""), - "type": "function", - "index": tc.get("index"), - "function": { - "name": func.get("name", ""), - "arguments": func.get("arguments", "") - } - }) - msg["tool_calls"] = cleaned - return messages diff --git a/openjiuwen/core/foundation/llm/schema/config.py b/openjiuwen/core/foundation/llm/schema/config.py index f88057533..d7842b083 100644 --- a/openjiuwen/core/foundation/llm/schema/config.py +++ b/openjiuwen/core/foundation/llm/schema/config.py @@ -2,7 +2,7 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. import uuid from enum import Enum -from typing import Optional, Union, Any, Self +from typing import Optional, Union, Any, Self, Literal from pydantic import BaseModel, Field, model_validator @@ -24,6 +24,34 @@ class ProviderType(str, Enum): IntelliRouter = "intelli_router" +class LLMApiMode(str, Enum): + ChatCompletions = "chat_completions" + Responses = "responses" + AnthropicMessages = "anthropic_messages" + + +class LLMAuthMode(str, Enum): + ApiKey = "api_key" + NoneAuth = "none" + CustomHeaders = "custom_headers" + OpenAIAccountOAuth = "openai_account_oauth" + + +class KVCacheExtensionConfig(BaseModel): + mode: Literal["none", "release", "affinity"] = "none" + release_endpoint: str = "/release_kv_cache" + session_field: str = "cache_salt" + enable_cache_sharing_field: str = "cache_sharing" + affinity_field: str = "agent_hint" + + +class LLMExtensionsConfig(BaseModel): + prompt_cache: dict[str, Any] = Field(default_factory=dict) + kv_cache: KVCacheExtensionConfig = Field(default_factory=KVCacheExtensionConfig) + response_fields: dict[str, Any] = Field(default_factory=dict) + request_extra_body: dict[str, Any] = Field(default_factory=dict) + + _TOP_LEVEL_API_KEY_PROVIDERS = { ProviderType.OpenAI.value, ProviderType.OpenRouter.value, @@ -34,6 +62,10 @@ class ProviderType(str, Enum): ProviderType.InferenceAffinity.value, } _TOP_LEVEL_API_BASE_PROVIDERS = _TOP_LEVEL_API_KEY_PROVIDERS | {ProviderType.OpenAIAccount.value} +_DEFAULT_API_BASE_BY_ENDPOINT_PROFILE = { + "ollama": "http://localhost:11434/v1", + "lmstudio": "http://localhost:1234/v1", +} class ModelClientConfig(BaseModel): @@ -82,7 +114,14 @@ class ModelClientConfig(BaseModel): default=None, description="Developer-provided headers merged per LLM call" ) - model_config = {"extra": "allow"} # Allow extra fields injected by core/provider (e.g. default headers) + api_mode: Optional[LLMApiMode] = Field(default=None, description="LLM API mode") + auth_mode: LLMAuthMode = Field(default=LLMAuthMode.ApiKey, description="LLM authentication mode") + endpoint_profile: Optional[str] = Field(default=None, description="OpenAI-compatible endpoint profile name") + extensions: LLMExtensionsConfig = Field(default_factory=LLMExtensionsConfig) + legacy_client_provider: Optional[str] = Field(default=None, description="Original provider before normalization") + model_config = { + "extra": "allow", + } @model_validator(mode='after') def validate_client_provider(self) -> Self: @@ -123,7 +162,17 @@ def validate_client_provider(self) -> Self: ) def _validate_top_level_provider_config(self, provider: str) -> None: - if provider in _TOP_LEVEL_API_KEY_PROVIDERS and not str(self.api_key or "").strip(): + if not str(self.api_base or "").strip() and self.endpoint_profile: + default_api_base = _DEFAULT_API_BASE_BY_ENDPOINT_PROFILE.get( + str(self.endpoint_profile).strip().lower() + ) + if default_api_base: + self.api_base = default_api_base + if self.auth_mode != LLMAuthMode.ApiKey.value and self.auth_mode != LLMAuthMode.ApiKey: + requires_api_key = False + else: + requires_api_key = provider in _TOP_LEVEL_API_KEY_PROVIDERS + if requires_api_key and not str(self.api_key or "").strip(): raise build_error( StatusCode.MODEL_SERVICE_CONFIG_ERROR, error_msg=f"api_key is required for provider {provider}." diff --git a/openjiuwen/core/foundation/llm/schema/message_chunk.py b/openjiuwen/core/foundation/llm/schema/message_chunk.py index de23cb4f2..5b2c01f0c 100644 --- a/openjiuwen/core/foundation/llm/schema/message_chunk.py +++ b/openjiuwen/core/foundation/llm/schema/message_chunk.py @@ -216,6 +216,7 @@ def __add__(self, other: Any) -> "AssistantMessageChunk": return AssistantMessageChunk( role=self.role, content=combined_content, + metadata=other.metadata or self.metadata, tool_calls=merged_tool_calls if merged_tool_calls else None, usage_metadata=other.usage_metadata or self.usage_metadata, finish_reason=merged_finish_reason, diff --git a/openjiuwen/core/foundation/llm/utils/endpoint_profiles.py b/openjiuwen/core/foundation/llm/utils/endpoint_profiles.py new file mode 100644 index 000000000..2579d0fa0 --- /dev/null +++ b/openjiuwen/core/foundation/llm/utils/endpoint_profiles.py @@ -0,0 +1,178 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +from copy import deepcopy +from typing import Any, Callable + +from pydantic import BaseModel, Field + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import build_error +from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, + LLMApiMode, + ModelClientConfig, + ProviderType, +) + + +class EndpointProfile(BaseModel): + """Structured rules for an OpenAI-compatible endpoint profile.""" + + name: str + protocol: str = ProviderType.OpenAI.value + api_mode: str = LLMApiMode.ChatCompletions.value + default_api_base: str | None = None + message_transforms: list[str] = Field(default_factory=list) + request_transforms: list[str] = Field(default_factory=list) + response_transforms: list[str] = Field(default_factory=list) + stream_transforms: list[str] = Field(default_factory=list) + extensions: dict[str, Any] = Field(default_factory=dict) + + +def _deepseek_reasoning_content(messages: list[dict]) -> list[dict]: + for message in messages: + if message.get("role") == "assistant": + message.setdefault("reasoning_content", "") + return messages + + +MESSAGE_TRANSFORMS: dict[str, Callable[[list[dict]], list[dict]]] = { + "deepseek_reasoning_content": _deepseek_reasoning_content, +} + + +ENDPOINT_PROFILES: dict[str, EndpointProfile] = { + "openai": EndpointProfile(name="openai"), + "openai_compatible": EndpointProfile(name="openai_compatible"), + "deepseek": EndpointProfile( + name="deepseek", + message_transforms=["deepseek_reasoning_content"], + ), + "openrouter": EndpointProfile(name="openrouter"), + "siliconflow": EndpointProfile(name="siliconflow"), + "dashscope": EndpointProfile(name="dashscope"), + "ollama": EndpointProfile( + name="ollama", + default_api_base="http://localhost:11434/v1", + ), + "lmstudio": EndpointProfile( + name="lmstudio", + default_api_base="http://localhost:1234/v1", + ), + "vllm": EndpointProfile(name="vllm"), + "sglang": EndpointProfile(name="sglang"), +} + + +LEGACY_PROVIDER_ALIASES: dict[str, dict[str, Any]] = { + ProviderType.DeepSeek.value: { + "client_provider": ProviderType.OpenAI.value, + "endpoint_profile": "deepseek", + }, + ProviderType.OpenRouter.value: { + "client_provider": ProviderType.OpenAI.value, + "endpoint_profile": "openrouter", + }, + ProviderType.SiliconFlow.value: { + "client_provider": ProviderType.OpenAI.value, + "endpoint_profile": "siliconflow", + }, + ProviderType.DashScope.value: { + "client_provider": ProviderType.OpenAI.value, + "endpoint_profile": "dashscope", + }, + ProviderType.InferenceAffinity.value: { + "client_provider": ProviderType.OpenAI.value, + "endpoint_profile": "openai_compatible", + "extensions": { + "kv_cache": {"mode": "release"}, + }, + }, + ProviderType.AscendAffinity.value: { + "client_provider": ProviderType.OpenAI.value, + "endpoint_profile": "openai_compatible", + "auth_mode": LLMAuthMode.CustomHeaders.value, + "extensions": { + "kv_cache": {"mode": "affinity"}, + }, + }, + ProviderType.OpenAIAccount.value: { + "client_provider": ProviderType.OpenAI.value, + "api_mode": LLMApiMode.Responses.value, + "auth_mode": LLMAuthMode.OpenAIAccountOAuth.value, + }, +} + + +def deep_merge_defaults(target: dict[str, Any], defaults: dict[str, Any]) -> dict[str, Any]: + """Merge default values without overriding explicit user-provided values.""" + merged = deepcopy(defaults) + for key, value in target.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = deep_merge_defaults(value, merged[key]) + else: + merged[key] = value + return merged + + +def normalize_model_client_config(config: ModelClientConfig) -> ModelClientConfig: + """Return a normalized config carrying protocol/profile/auth/api-mode metadata.""" + provider = ( + config.client_provider.value + if isinstance(config.client_provider, ProviderType) + else config.client_provider + ) + provider = str(provider or "").strip() + alias = LEGACY_PROVIDER_ALIASES.get(provider) + if not alias: + return config + + data = config.model_dump() + explicit_data = config.model_dump(exclude_unset=True) + explicit_fields = config.model_fields_set + data["legacy_client_provider"] = provider + for key, value in alias.items(): + if key == "extensions": + continue + if key == "client_provider" or key not in explicit_fields: + data[key] = value + + if alias.get("extensions"): + normalized_extensions = deep_merge_defaults( + alias["extensions"], + data.get("extensions") or {}, + ) + explicit_extensions = explicit_data.get("extensions") or {} + data["extensions"] = deep_merge_defaults( + explicit_extensions, + normalized_extensions, + ) + return ModelClientConfig(**data) + + +def resolve_endpoint_profile(config: ModelClientConfig) -> EndpointProfile: + profile_name = getattr(config, "endpoint_profile", None) + if not profile_name: + profile_name = "openai" if config.client_provider == ProviderType.OpenAI.value else "openai_compatible" + try: + return ENDPOINT_PROFILES[str(profile_name)] + except KeyError as exc: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"unknown endpoint_profile: {profile_name}" + ) from exc + + +def apply_message_transforms(config: ModelClientConfig, messages: list[dict]) -> list[dict]: + profile = resolve_endpoint_profile(config) + transformed = messages + for name in profile.message_transforms: + transform = MESSAGE_TRANSFORMS.get(name) + if transform is None: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=f"unknown message transform: {name}" + ) + transformed = transform(transformed) + return transformed diff --git a/openjiuwen/core/single_agent/agents/react_agent.py b/openjiuwen/core/single_agent/agents/react_agent.py index cf54b558a..afe38adcc 100644 --- a/openjiuwen/core/single_agent/agents/react_agent.py +++ b/openjiuwen/core/single_agent/agents/react_agent.py @@ -1079,6 +1079,7 @@ async def _railed_model_call(self, ctx: AgentCallbackContext) -> AssistantMessag else: ai_message = AssistantMessage( content=accumulated_chunk.content or "", + metadata=accumulated_chunk.metadata, tool_calls=accumulated_chunk.tool_calls or [], usage_metadata=accumulated_chunk.usage_metadata, reasoning_content=accumulated_chunk.reasoning_content, @@ -2012,6 +2013,7 @@ async def _inner_invoke(self, session, inputs, query, need_cleanup, conversation await context.add_messages( AssistantMessage( content=ai_message.content, + metadata=ai_message.metadata, tool_calls=ai_message.tool_calls, reasoning_content=ai_message.reasoning_content, usage_metadata=ai_message.usage_metadata, diff --git a/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_outbound_payload.py b/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_outbound_payload.py index cc3f04065..8f57cf05c 100644 --- a/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_outbound_payload.py +++ b/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_outbound_payload.py @@ -21,10 +21,9 @@ KVCacheAffinityConfig, KVCacheIdentity, ) -from openjiuwen.core.foundation.llm.model_clients.ascend_affinity_model_client import ( - AscendAffinityModelClient, -) +from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, ModelClientConfig, ModelRequestConfig, ProviderType, @@ -36,13 +35,14 @@ from openjiuwen.core.single_agent.schema.agent_card import AgentCard -def _client() -> AscendAffinityModelClient: - return AscendAffinityModelClient( +def _client() -> OpenAIModelClient: + return OpenAIModelClient( model_config=ModelRequestConfig(model="test-model"), model_client_config=ModelClientConfig( - client_provider=ProviderType.AscendAffinity, - api_key="test-key", + client_provider=ProviderType.OpenAI, + auth_mode=LLMAuthMode.CustomHeaders, api_base="https://example.test", + extensions={"kv_cache": {"mode": "affinity"}}, verify_ssl=False, ), ) @@ -169,7 +169,12 @@ def current_kv_cache_identity(self) -> KVCacheIdentity: (True, True, True), ], ) -async def test_team_member_outbound_payload_gate_and_registry_noop(role: str, enabled: bool, supports: bool, expect_hint: bool) -> None: +async def test_team_member_outbound_payload_gate_and_registry_noop( + role: str, + enabled: bool, + supports: bool, + expect_hint: bool, +) -> None: member_id = "leader-card" if role == "leader" else "teammate-card" cache_id = f"team:team-sid:team:team-a:member:{member_id}" diff --git a/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_swarmflow_stateful_outbound.py b/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_swarmflow_stateful_outbound.py index b2b85d512..abf68a284 100644 --- a/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_swarmflow_stateful_outbound.py +++ b/tests/unit_tests/agent_teams/kv_cache/test_kv_cache_swarmflow_stateful_outbound.py @@ -20,10 +20,9 @@ KVCacheAffinityConfig, KVCacheIdentity, ) -from openjiuwen.core.foundation.llm.model_clients.ascend_affinity_model_client import ( - AscendAffinityModelClient, -) +from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, ModelClientConfig, ModelRequestConfig, ProviderType, @@ -35,6 +34,19 @@ from openjiuwen.core.single_agent.schema.agent_card import AgentCard +def _affinity_client() -> OpenAIModelClient: + return OpenAIModelClient( + model_config=ModelRequestConfig(model="test-model"), + model_client_config=ModelClientConfig( + client_provider=ProviderType.OpenAI, + auth_mode=LLMAuthMode.CustomHeaders, + api_base="https://example.test", + extensions={"kv_cache": {"mode": "affinity"}}, + verify_ssl=False, + ), + ) + + class _CapturingModel: def __init__(self, *, supports: bool, events: list[str]) -> None: self._supports = supports @@ -43,15 +55,7 @@ def __init__(self, *, supports: bool, events: list[str]) -> None: self.evict_calls: list[dict[str, Any]] = [] self.offload_calls: list[dict[str, Any]] = [] self.prefetch_calls: list[dict[str, Any]] = [] - self.client = AscendAffinityModelClient( - model_config=ModelRequestConfig(model="test-model"), - model_client_config=ModelClientConfig( - client_provider=ProviderType.AscendAffinity, - api_key="test-key", - api_base="https://example.test", - verify_ssl=False, - ), - ) + self.client = _affinity_client() def supports_kv_cache_release(self) -> bool: return False diff --git a/tests/unit_tests/core/context_engine/test_forked_inference_affinity_kv_cache_release_with_processors.py b/tests/unit_tests/core/context_engine/test_forked_inference_affinity_kv_cache_release_with_processors.py index 04bef7dab..eb2c58b02 100644 --- a/tests/unit_tests/core/context_engine/test_forked_inference_affinity_kv_cache_release_with_processors.py +++ b/tests/unit_tests/core/context_engine/test_forked_inference_affinity_kv_cache_release_with_processors.py @@ -6,7 +6,7 @@ **Scope**: Full flow from ContextEngine.create_context + MessageSummaryOffloader to get_context_window() + KVCacheModelCallHook and release() call/logs. Uses mocked -InferenceAffinityModel invoke and patched InferenceAffinityModelClient.release. Verifies: +InferenceAffinityModel invoke and patched OpenAIModelClient.release. Verifies: - When processors (e.g. MessageSummaryOffloader) modify context, release is triggered and session_id / block_released / cache_salt are correct in call and logs. - When no processors modify context, release is NOT called and no release logs. @@ -109,8 +109,7 @@ def _extract_metadata_from_record(record) -> dict: @pytest.mark.asyncio -@patch( - 'openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client.InferenceAffinityModelClient.release') +@patch('openjiuwen.core.foundation.llm.model_clients.openai_model_client.OpenAIModelClient.release') async def test_inference_affinity_kv_cache_release_with_message_offloader(mock_release, caplog): """ Test KV cache release when MessageSummaryOffloader modifies context messages. @@ -123,7 +122,7 @@ async def test_inference_affinity_kv_cache_release_with_message_offloader(mock_r 5. Release operation is properly logged with correct session_id and block count Args: - mock_release: Mocked release method from InferenceAffinityModelClient + mock_release: Mocked release method from OpenAIModelClient caplog: Pytest fixture for capturing log messages """ # Dictionary to track release call details for verification @@ -319,8 +318,7 @@ async def mock_release_impl(*args, **kwargs): @pytest.mark.asyncio -@patch( - 'openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client.InferenceAffinityModelClient.release') +@patch('openjiuwen.core.foundation.llm.model_clients.openai_model_client.OpenAIModelClient.release') async def test_inference_affinity_kv_cache_no_release_without_modification(mock_release, caplog): """ Test that KV cache is NOT released when context messages remain unchanged. @@ -335,7 +333,7 @@ async def test_inference_affinity_kv_cache_no_release_without_modification(mock_ releases cache when necessary, avoiding unnecessary performance overhead. Args: - mock_release: Mocked release method from InferenceAffinityModelClient + mock_release: Mocked release method from OpenAIModelClient caplog: Pytest fixture for capturing log messages """ caplog.set_level(logging.INFO) diff --git a/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py b/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py index e1536388e..7cf5410b0 100644 --- a/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py +++ b/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py @@ -14,6 +14,7 @@ UserMessage, ) from openjiuwen.core.foundation.llm.model_clients.anthropic_model_client import AnthropicModelClient +from openjiuwen.core.foundation.llm.schema.config import LLMAuthMode def _build_mock_anthropic_response(text: str = "ok") -> MagicMock: @@ -136,6 +137,21 @@ async def test_aclose_connections_unknown_config_is_noop(self): assert client.close.call_count == 0 assert AnthropicModelClient.connection_key(cfg) in AnthropicModelClient._client_cache + def test_connection_key_includes_auth_mode(self): + api_key_cfg = self._cfg("https://api.anthropic.com") + custom_headers_cfg = ModelClientConfig( + client_provider=ProviderType.Anthropic, + api_key="sk-ant", + api_base="https://api.anthropic.com", + auth_mode=LLMAuthMode.CustomHeaders, + verify_ssl=False, + ) + + assert ( + AnthropicModelClient.connection_key(api_key_cfg) + != AnthropicModelClient.connection_key(custom_headers_cfg) + ) + class TestFallbackClient: @pytest.mark.asyncio diff --git a/tests/unit_tests/core/foundation/llm/test_anthropic_model_client.py b/tests/unit_tests/core/foundation/llm/test_anthropic_model_client.py index 786d57ca4..b6b9d0b78 100644 --- a/tests/unit_tests/core/foundation/llm/test_anthropic_model_client.py +++ b/tests/unit_tests/core/foundation/llm/test_anthropic_model_client.py @@ -10,12 +10,19 @@ from unittest.mock import MagicMock +import pytest + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import BaseError from openjiuwen.core.foundation.llm import ( + AssistantMessage, ModelClientConfig, ModelRequestConfig, ProviderType, ) +from openjiuwen.core.foundation.llm.schema.config import LLMAuthMode from openjiuwen.core.foundation.llm.model_clients.anthropic_model_client import ( + _ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY, AnthropicModelClient, _apply_messages_cache_breakpoint, _apply_static_cache_breakpoints, @@ -143,6 +150,94 @@ def test_assistant_with_invalid_json_arguments_falls_back_to_raw(self): tool_use = [b for b in messages[0]["content"] if b["type"] == "tool_use"] assert tool_use[0]["input"] == {"_raw_arguments": "not json"} + def test_signed_thinking_blocks_are_replayed_unchanged(self): + thinking = { + "type": "thinking", + "thinking": "inspect the inputs", + "signature": "opaque-signature", + } + tool_use = { + "type": "tool_use", + "id": "tool-1", + "name": "search", + "input": {"q": "x"}, + } + _, messages = _convert_message_schemas([{ + "role": "assistant", + "content": "", + "reasoning_content": "display copy only", + "tool_calls": [{ + "id": "tool-1", + "type": "function", + "function": {"name": "search", "arguments": '{"q":"x"}'}, + }], + "metadata": { + _ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY: [thinking, tool_use], + }, + }]) + + assert messages == [{ + "role": "assistant", + "content": [thinking, tool_use], + }] + assert "reasoning_content" not in messages[0] + + def test_openai_base64_image_is_converted_to_anthropic_source(self): + _, messages = _convert_message_schemas([{ + "role": "user", + "content": [{ + "type": "image_url", + "image_url": {"url": "data:image/png;base64,abc", "detail": "low"}, + }], + }]) + + assert messages[0]["content"] == [{ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc", + }, + }] + + def test_openai_remote_image_is_converted_to_anthropic_source(self): + _, messages = _convert_message_schemas([{ + "role": "user", + "content": [{ + "type": "image_url", + "image_url": {"url": "https://example.com/image.webp"}, + }], + }]) + + assert messages[0]["content"] == [{ + "type": "image", + "source": { + "type": "url", + "url": "https://example.com/image.webp", + }, + }] + + def test_unsupported_image_data_url_fails_before_request(self): + with pytest.raises(ValueError, match="JPEG, PNG, GIF, or WebP"): + _convert_message_schemas([{ + "role": "user", + "content": [{ + "type": "image_url", + "image_url": {"url": "data:image/svg+xml;base64,abc"}, + }], + }]) + + @pytest.mark.parametrize("image_value", [{}, {"url": ""}, {"url": None}, None]) + def test_empty_openai_image_url_fails_before_request(self, image_value): + with pytest.raises(ValueError, match="non-empty image URL"): + _convert_message_schemas([{ + "role": "user", + "content": [{ + "type": "image_url", + "image_url": image_value, + }], + }]) + def test_tool_result_empty_content_padded(self): _, messages = _convert_message_schemas([ {"role": "tool", "tool_call_id": "t1", "content": ""}, @@ -245,6 +340,15 @@ def test_mark_cache_control_noop_on_empty(self): # Should not raise on empty list _mark_cache_control([], "1h") + def test_mark_cache_control_skips_signed_thinking_block(self): + blocks = [ + {"type": "text", "text": "answer"}, + {"type": "thinking", "thinking": "hidden", "signature": "sig"}, + ] + _mark_cache_control(blocks, "5m") + assert "cache_control" not in blocks[1] + assert _is_5m_ephemeral(blocks[0]["cache_control"]) + def test_apply_static_breakpoints_marks_last_tool_and_last_system(self): tools = [{"name": "a"}, {"name": "b"}] system_blocks = [{"type": "text", "text": "s"}] @@ -376,6 +480,12 @@ def test_passthrough_when_no_v1_suffix(self): == "https://api.anthropic.com" ) + def test_strips_full_messages_endpoint(self): + assert ( + AnthropicModelClient._normalize_base_url("https://proxy.example/anthropic/v1/messages") + == "https://proxy.example/anthropic" + ) + def test_empty_string_returns_none(self): assert AnthropicModelClient._normalize_base_url("") is None @@ -443,6 +553,105 @@ def test_client_name_is_anthropic(self): def test_get_client_name(self): assert _make_client()._get_client_name() == "Anthropic client" + def test_anthropic_client_requires_api_key_even_when_auth_mode_is_not_api_key(self): + client_config = ModelClientConfig( + client_provider=ProviderType.Anthropic, + api_base="https://api.anthropic.com", + auth_mode=LLMAuthMode.NoneAuth, + verify_ssl=False, + ) + + with pytest.raises(BaseError) as error: + AnthropicModelClient(ModelRequestConfig(model="claude-opus-4"), client_config) + + assert error.value.code == StatusCode.MODEL_SERVICE_CONFIG_ERROR.code + assert "api_key is required for Anthropic client" in str(error.value) + + +class TestResponseReasoning: + @staticmethod + def _block(**values): + block = MagicMock() + for key, value in values.items(): + setattr(block, key, value) + block.model_dump.return_value = dict(values) + return block + + @pytest.mark.asyncio + async def test_non_stream_response_normalizes_and_preserves_thinking(self): + response = MagicMock() + response.content = [ + self._block(type="thinking", thinking="plan", signature="sig"), + self._block(type="tool_use", id="t1", name="lookup", input={"x": 1}), + ] + response.usage = None + response.stop_reason = "tool_use" + + message = await _make_client()._parse_response(response) + + assert message.reasoning_content == "plan" + assert message.tool_calls and message.tool_calls[0].id == "t1" + assert message.metadata[_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY] == [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "tool_use", "id": "t1", "name": "lookup", "input": {"x": 1}}, + ] + + @pytest.mark.asyncio + async def test_gateway_top_level_reasoning_is_display_only(self): + response = MagicMock() + response.content = [self._block(type="text", text="answer")] + response.reasoning_content = "gateway plan" + response.usage = None + response.stop_reason = "end_turn" + + message = await _make_client()._parse_response(response) + + assert message.reasoning_content == "gateway plan" + assert message.metadata[_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY] == [ + {"type": "text", "text": "answer"}, + ] + + @pytest.mark.asyncio + async def test_redacted_thinking_is_preserved_but_not_exposed(self): + response = MagicMock() + response.content = [ + self._block(type="redacted_thinking", data="opaque-data"), + self._block(type="text", text="answer"), + ] + response.reasoning_content = None + response.usage = None + response.stop_reason = "end_turn" + + message = await _make_client()._parse_response(response) + + assert message.reasoning_content is None + assert message.metadata[_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY][0] == { + "type": "redacted_thinking", + "data": "opaque-data", + } + + def test_stream_thinking_and_signature_are_preserved(self): + client = _make_client() + state = {} + + start = MagicMock(type="content_block_start", index=0) + start.content_block = MagicMock(type="thinking", thinking="", signature="") + assert client._event_to_chunk(start, state) is None + + thinking_event = MagicMock(type="content_block_delta", index=0) + thinking_event.delta = MagicMock(type="thinking_delta", thinking="plan") + thinking_chunk = client._event_to_chunk(thinking_event, state) + assert thinking_chunk.reasoning_content == "plan" + + signature_event = MagicMock(type="content_block_delta", index=0) + signature_event.delta = MagicMock(type="signature_delta", signature="sig") + signature_chunk = client._event_to_chunk(signature_event, state) + assert signature_chunk.metadata[_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY] == [{ + "type": "thinking", + "thinking": "plan", + "signature": "sig", + }] + # --------------------------------------------------------------------------- # F. Sampling params: Anthropic forbids temperature + top_p together @@ -469,11 +678,11 @@ def test_temperature_wins_when_both_set(self): assert params["temperature"] == 0.6 assert "top_p" not in params - def test_config_defaults_send_temperature_only(self): - # ModelRequestConfig defaults both (temperature=0.95, top_p=0.1); the - # base builder backfills them, so a plain call must still drop top_p. + def test_config_defaults_do_not_override_anthropic_sampling(self): + # Common config defaults are OpenAI-oriented. Omitting both lets the + # Anthropic model choose its protocol default and avoids Claude 4.7+ 400s. params = self._params(_make_client()) - assert "temperature" in params + assert "temperature" not in params assert "top_p" not in params def test_top_p_forwarded_when_temperature_absent(self): @@ -491,3 +700,60 @@ def test_default_top_p_dropped_when_temperature_absent(self): params = self._params(client, top_p=1.0) assert "top_p" not in params assert "temperature" not in params + + def test_anthropic_native_reasoning_controls_are_forwarded(self): + client = _make_client() + client.model_config = ModelRequestConfig( + model="claude-opus-4-6", + thinking={"type": "adaptive"}, + output_config={"effort": "high"}, + ) + params = self._params(client, model="claude-opus-4-6") + assert params["thinking"] == {"type": "adaptive"} + assert params["output_config"] == {"effort": "high"} + assert "temperature" not in params + assert "top_p" not in params + + def test_compatible_endpoint_reasoning_effort_is_forwarded(self): + client = _make_client() + client.model_config = ModelRequestConfig(model="glm-5", reasoning_effort="high") + params = self._params(client, model="glm-5") + assert "reasoning_effort" not in params + assert params["extra_body"]["reasoning_effort"] == "high" + + def test_compatible_endpoint_extensions_merge_with_extra_body(self): + client = _make_client() + client.model_config = ModelRequestConfig( + model="glm-5", + reasoning_effort="high", + extra_body={"vendor_flag": True}, + ) + params = self._params(client, model="glm-5") + assert params["extra_body"] == { + "vendor_flag": True, + "reasoning_effort": "high", + } + + def test_current_claude_model_drops_explicit_custom_sampling(self): + params = self._params( + _make_client(), + model="claude-opus-4-8", + temperature=0.6, + top_p=0.8, + ) + assert "temperature" not in params + assert "top_p" not in params + + def test_base_message_metadata_survives_common_conversion(self): + client = _make_client() + thinking = {"type": "thinking", "thinking": "plan", "signature": "sig"} + tool_use = {"type": "tool_use", "id": "t1", "name": "lookup", "input": {}} + message = AssistantMessage( + content="", + tool_calls=[{"id": "t1", "type": "function", "name": "lookup", "arguments": "{}"}], + metadata={_ANTHROPIC_CONTENT_BLOCKS_METADATA_KEY: [thinking, tool_use]}, + ) + params = self._params(client, messages=[message]) + replayed = params["messages"][0]["content"] + assert replayed[0] == thinking + assert {key: replayed[1][key] for key in tool_use} == tool_use diff --git a/tests/unit_tests/core/foundation/llm/test_ascend_affinity_model_client.py b/tests/unit_tests/core/foundation/llm/test_ascend_affinity_model_client.py deleted file mode 100644 index d9ddbbc00..000000000 --- a/tests/unit_tests/core/foundation/llm/test_ascend_affinity_model_client.py +++ /dev/null @@ -1,305 +0,0 @@ -# coding: utf-8 - -import asyncio -import inspect -from unittest.mock import AsyncMock - -import pytest - -from openjiuwen.core.foundation.llm import Model -from openjiuwen.core.foundation.llm.model_clients import create_model_client -from openjiuwen.core.foundation.llm.model_clients.ascend_affinity_model_client import ( - AscendAffinityModelClient, -) -from openjiuwen.core.foundation.llm.schema.config import ( - ModelClientConfig, - ModelRequestConfig, - ProviderType, -) -from openjiuwen.core.foundation.kv_cache import ( - KVC_MANAGEMENT_MAX_ATTEMPTS, - KVC_SESSION_OFFLOAD_PREFETCH_TIMEOUT_SECONDS, -) - - -def _client() -> AscendAffinityModelClient: - return AscendAffinityModelClient( - model_config=ModelRequestConfig(model="test-model"), - model_client_config=ModelClientConfig( - client_provider=ProviderType.AscendAffinity, - api_key="test-key", - api_base="https://example.test", - verify_ssl=False, - ), - ) - - -def test_factory_creates_ascend_affinity_client(): - client = create_model_client( - client_config=ModelClientConfig( - client_provider="AscendAffinity", - api_key="test-key", - api_base="https://example.test", - verify_ssl=False, - ), - model_config=ModelRequestConfig(model="test-model"), - ) - - assert isinstance(client, AscendAffinityModelClient) - assert client.supports_kv_cache_affinity() is True - - -def test_normal_request_carries_agent_hint(): - params = _client()._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - session_id="sess-a", - parent_session_id="parent-a", - ) - - assert params["agent_hint"] == { - "session_id": "sess-a", - "parent_session_id": "parent-a", - } - - -def test_normal_request_without_session_omits_agent_hint(): - params = _client()._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert "agent_hint" not in params - - -def test_session_management_request_uses_empty_messages_and_context_management(): - client = _client() - params = client._build_request_params( - messages=[], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - session_id="sess-a", - parent_session_id="parent-a", - kv_action="evict", - target="session", - manage_request=True, - ) - - assert params["messages"] == [] - assert "tools" not in params - assert params["agent_hint"] == { - "session_id": "sess-a", - "parent_session_id": "parent-a", - "context_management": { - "manage_request": True, - "edits": [{"type": "evict", "target": "session"}], - }, - } - - -def test_management_request_defaults_to_session_target(): - params = _client()._build_request_params( - messages=[], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - session_id="sess-a", - kv_action="offload", - manage_request=True, - ) - - assert params["messages"] == [] - assert params["agent_hint"]["parent_session_id"] == "sess-a" - assert params["agent_hint"]["context_management"]["edits"] == [ - {"type": "offload", "target": "session"} - ] - - -def test_message_and_tools_management_builds_two_edits(): - params = _client()._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=[{"type": "function", "function": {"name": "x", "parameters": {}}}], - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - session_id="sess-a", - kv_action="evict", - target="messages", - manage_request=True, - msg_start=2, - msg_end=3, - include_tools=True, - tools_start=0, - tools_end=1, - ) - - edits = params["agent_hint"]["context_management"]["edits"] - assert edits == [ - {"type": "evict", "target": "messages", "start": 2, "end": 3}, - {"type": "evict", "target": "tools", "start": 0, "end": 1}, - ] - - -@pytest.mark.parametrize( - ("target", "msg_start", "msg_end", "tools_start", "tools_end"), - [ - ("messages", 1, None, None, None), - ("messages", None, 1, None, None), - ("tools", None, None, 0, None), - ("tools", None, None, None, 0), - ], -) -def test_range_target_requires_both_start_and_end( - target, msg_start, msg_end, tools_start, tools_end -): - with pytest.raises(Exception): - _client()._build_target_edits( - action="evict", - target=target, - msg_start=msg_start, - msg_end=msg_end, - tools_start=tools_start, - tools_end=tools_end, - ) - - -@pytest.mark.parametrize( - ("start", "end"), - [(-1, 0), (1, -1), (2, 1), (1, 1), (True, 1), (0, False)], -) -def test_range_target_rejects_invalid_half_open_range(start, end): - with pytest.raises(Exception): - _client()._build_target_edits( - action="evict", - target="messages", - msg_start=start, - msg_end=end, - ) - - -def test_session_management_rejects_ranges(): - with pytest.raises(Exception): - _client()._build_request_params( - messages=[], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - session_id="sess-a", - kv_action="evict", - target="session", - manage_request=True, - msg_start=1, - ) - - -def test_model_reports_affinity_support_and_builds_invoke_kwargs(): - model = Model( - model_client_config=ModelClientConfig( - client_provider=ProviderType.AscendAffinity, - api_key="test-key", - api_base="https://example.test", - verify_ssl=False, - ), - model_config=ModelRequestConfig(model="test-model"), - ) - - class Session: - @staticmethod - def get_session_id(): - return "sess-a" - - assert model.supports_kv_cache_affinity() is True - # The generic legacy wrapper must not enable AscendAffinity. New callers use - # the affinity-specific name below so release and affinity remain separate. - assert model.build_kv_cache_invoke_kwargs(session=Session()) == {} - assert model.build_kv_cache_affinity_invoke_kwargs(session=Session()) == {} - assert model.build_kv_cache_affinity_invoke_kwargs(session=Session(), enable_kv_cache_affinity=True) == { - "session_id": "sess-a", - "parent_session_id": "sess-a", - } - - -def test_kv_action_methods_have_explicit_parameters(): - expected = { - "self", - "session_id", - "parent_session_id", - "target", - "messages", - "tools", - "model", - "msg_start", - "msg_end", - "tools_start", - "tools_end", - "include_tools", - "timeout", - } - - for method_name in ("evict_kvc", "offload_kvc", "prefetch_kvc"): - params = inspect.signature(getattr(AscendAffinityModelClient, method_name)).parameters - assert set(params) == expected - assert not any(param.kind == inspect.Parameter.VAR_KEYWORD for param in params.values()) - - model_params = inspect.signature(getattr(Model, method_name)).parameters - assert set(model_params) == expected - assert not any(param.kind == inspect.Parameter.VAR_KEYWORD for param in model_params.values()) - - -@pytest.mark.asyncio -async def test_kv_management_uses_shared_total_timeout_and_single_attempt(): - client = _client() - request = AsyncMock(return_value={"choices": [{"message": {"content": ""}}]}) - client._make_ascend_affinity_request = request - - assert await client.offload_kvc(session_id="sess-a") is True - - assert request.await_args.kwargs["timeout"] == KVC_SESSION_OFFLOAD_PREFETCH_TIMEOUT_SECONDS - assert request.await_args.kwargs["max_attempts"] == KVC_MANAGEMENT_MAX_ATTEMPTS - - -@pytest.mark.asyncio -async def test_kv_management_total_timeout_cancels_request(): - client = _client() - cancelled = asyncio.Event() - - async def _slow_request(*_args, **_kwargs): - try: - await asyncio.Event().wait() - finally: - cancelled.set() - - client._make_ascend_affinity_request = _slow_request - - with pytest.raises(Exception): - await client.evict_kvc(session_id="sess-a", timeout=0.01) - - assert cancelled.is_set() diff --git a/tests/unit_tests/core/foundation/llm/test_message_chunk.py b/tests/unit_tests/core/foundation/llm/test_message_chunk.py index 7550d18fa..965f7441e 100644 --- a/tests/unit_tests/core/foundation/llm/test_message_chunk.py +++ b/tests/unit_tests/core/foundation/llm/test_message_chunk.py @@ -307,6 +307,22 @@ def test_assistant_add_handles_none_reasoning_content(): assert result.reasoning_content == "Some reasoning" +def test_assistant_add_keeps_latest_non_empty_metadata(): + chunk1 = AssistantMessageChunk( + content="", + metadata={"anthropic_content_blocks": [{"type": "thinking", "thinking": "a"}]}, + ) + chunk2 = AssistantMessageChunk( + content="", + metadata={"anthropic_content_blocks": [{"type": "thinking", "thinking": "ab"}]}, + ) + chunk3 = AssistantMessageChunk(content="") + + result = chunk1 + chunk2 + chunk3 + + assert result.metadata == chunk2.metadata + + def test_assistant_add_merges_finish_reason(): """Test that __add__ handles finish_reason.""" chunk1 = AssistantMessageChunk( diff --git a/tests/unit_tests/core/foundation/llm/test_model_client_config.py b/tests/unit_tests/core/foundation/llm/test_model_client_config.py index e695265d8..776947095 100644 --- a/tests/unit_tests/core/foundation/llm/test_model_client_config.py +++ b/tests/unit_tests/core/foundation/llm/test_model_client_config.py @@ -7,8 +7,13 @@ from openjiuwen.core.common.exception.codes import StatusCode from openjiuwen.core.common.exception.errors import BaseError -from openjiuwen.core.foundation.llm import BaseModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ProviderType +from openjiuwen.core.foundation.llm import AnthropicModelClient, BaseModelClient +from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, + LLMApiMode, + ModelClientConfig, + ProviderType, +) class _TempMockClient(BaseModelClient): @@ -95,6 +100,47 @@ def test_model_client_config_requires_api_key_for_non_openai_account_provider(): assert "api_key is required for provider OpenAI" in str(error.value) +def test_model_client_config_allows_openai_without_api_key_for_none_auth(): + cfg = ModelClientConfig( + client_provider=ProviderType.OpenAI, + api_base="http://localhost:11434/v1", + auth_mode=LLMAuthMode.NoneAuth, + endpoint_profile="ollama", + ) + + assert cfg.client_provider == ProviderType.OpenAI + assert cfg.auth_mode == LLMAuthMode.NoneAuth.value + assert cfg.api_key == "" + + +def test_model_client_config_fills_local_profile_default_api_base(): + cfg = ModelClientConfig( + client_provider=ProviderType.OpenAI, + auth_mode=LLMAuthMode.NoneAuth, + endpoint_profile="ollama", + ) + + assert cfg.api_base == "http://localhost:11434/v1" + + +def test_anthropic_model_client_is_exported_from_llm_package(): + assert AnthropicModelClient.__name__ == "AnthropicModelClient" + + +def test_model_client_config_allows_openai_responses_oauth_without_api_key(): + cfg = ModelClientConfig( + client_provider=ProviderType.OpenAI, + api_base="https://chatgpt.com/backend-api/codex", + api_mode=LLMApiMode.Responses, + auth_mode=LLMAuthMode.OpenAIAccountOAuth, + ) + + assert cfg.client_provider == ProviderType.OpenAI + assert cfg.api_mode == LLMApiMode.Responses.value + assert cfg.auth_mode == LLMAuthMode.OpenAIAccountOAuth.value + assert cfg.api_key == "" + + def test_model_client_config_requires_api_base_for_top_level_provider(): with pytest.raises(BaseError) as error: ModelClientConfig( diff --git a/tests/unit_tests/core/foundation/llm/test_model_client_profile_routing.py b/tests/unit_tests/core/foundation/llm/test_model_client_profile_routing.py new file mode 100644 index 000000000..81e086ea2 --- /dev/null +++ b/tests/unit_tests/core/foundation/llm/test_model_client_profile_routing.py @@ -0,0 +1,177 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +from openjiuwen.core.foundation.llm.model_clients import create_model_client +from openjiuwen.core.foundation.llm.model_clients.openai_account_model_client import ( + DEFAULT_OPENAI_ACCOUNT_BASE_URL, + OpenAIAccountModelClient, +) +from openjiuwen.core.foundation.llm.model_clients.anthropic_model_client import AnthropicModelClient +from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient +from openjiuwen.core.foundation.llm.schema.config import ( + LLMAuthMode, + LLMApiMode, + ModelClientConfig, + ModelRequestConfig, + ProviderType, +) +from openjiuwen.core.foundation.llm.utils.endpoint_profiles import normalize_model_client_config + + +def _request_config() -> ModelRequestConfig: + return ModelRequestConfig(model="test-model") + + +def test_legacy_deepseek_provider_routes_to_openai_client_with_profile(): + config = ModelClientConfig( + client_provider="DeepSeek", + api_key="sk-test", + api_base="https://api.deepseek.com/v1", + ) + + client = create_model_client(config, _request_config()) + + assert isinstance(client, OpenAIModelClient) + assert client.model_client_config.client_provider == ProviderType.OpenAI.value + assert client.model_client_config.endpoint_profile == "deepseek" + + +def test_legacy_deepseek_provider_normalizes_to_openai_protocol_metadata(): + config = ModelClientConfig( + client_provider="DeepSeek", + api_key="sk-test", + api_base="https://api.deepseek.com/v1", + ) + + normalized = normalize_model_client_config(config) + + assert normalized.client_provider == ProviderType.OpenAI.value + assert normalized.endpoint_profile == "deepseek" + assert normalized.legacy_client_provider == ProviderType.DeepSeek.value + + +def test_legacy_affinity_provider_normalizes_kv_extension_defaults(): + config = ModelClientConfig( + client_provider="InferenceAffinity", + api_key="sk-test", + api_base="https://example.test", + verify_ssl=False, + ) + + normalized = normalize_model_client_config(config) + + assert normalized.client_provider == ProviderType.OpenAI.value + assert normalized.endpoint_profile == "openai_compatible" + assert normalized.extensions.kv_cache.mode == "release" + assert normalized.legacy_client_provider == ProviderType.InferenceAffinity.value + + +def test_legacy_alias_does_not_override_explicit_auth_or_extensions(): + config = ModelClientConfig( + client_provider="AscendAffinity", + api_key="sk-test", + api_base="https://example.test", + auth_mode=LLMAuthMode.ApiKey, + extensions={"kv_cache": {"mode": "none"}}, + verify_ssl=False, + ) + + normalized = normalize_model_client_config(config) + + assert normalized.client_provider == ProviderType.OpenAI.value + assert normalized.auth_mode == LLMAuthMode.ApiKey.value + assert normalized.extensions.kv_cache.mode == "none" + + +def test_openai_deepseek_profile_routes_to_openai_client_with_profile(): + config = ModelClientConfig( + client_provider="OpenAI", + endpoint_profile="deepseek", + api_key="sk-test", + api_base="https://api.deepseek.com/v1", + ) + + client = create_model_client(config, _request_config()) + + assert isinstance(client, OpenAIModelClient) + assert client.model_client_config.endpoint_profile == "deepseek" + + +def test_openai_account_oauth_shape_routes_to_account_client(): + config = ModelClientConfig( + client_provider="OpenAI", + api_mode=LLMApiMode.Responses, + auth_mode=LLMAuthMode.OpenAIAccountOAuth, + api_base=DEFAULT_OPENAI_ACCOUNT_BASE_URL, + ) + + client = create_model_client(config, _request_config()) + + assert isinstance(client, OpenAIAccountModelClient) + + +def test_openai_responses_api_key_shape_stays_on_openai_client(): + config = ModelClientConfig( + client_provider="OpenAI", + api_mode=LLMApiMode.Responses, + auth_mode=LLMAuthMode.ApiKey, + api_key="sk-test", + api_base="https://api.openai.com/v1", + ) + + client = create_model_client(config, _request_config()) + + assert isinstance(client, OpenAIModelClient) + + +def test_anthropic_provider_routes_to_anthropic_client(): + config = ModelClientConfig( + client_provider="Anthropic", + api_key="sk-ant-test", + api_base="https://api.anthropic.com", + ) + + client = create_model_client(config, _request_config()) + + assert isinstance(client, AnthropicModelClient) + + +def test_openai_profiles_route_to_unified_openai_client(): + for profile in ("openrouter", "siliconflow", "dashscope"): + config = ModelClientConfig( + client_provider="OpenAI", + endpoint_profile=profile, + api_key="sk-test", + api_base="https://example.test/v1", + verify_ssl=False, + ) + + client = create_model_client(config, _request_config()) + + assert isinstance(client, OpenAIModelClient) + assert client.model_client_config.endpoint_profile == profile + + +def test_openai_kv_extensions_route_to_unified_openai_client(): + release_config = ModelClientConfig( + client_provider="OpenAI", + api_key="sk-test", + api_base="https://example.test", + extensions={"kv_cache": {"mode": "release"}}, + verify_ssl=False, + ) + affinity_config = ModelClientConfig( + client_provider="OpenAI", + api_base="https://example.test", + auth_mode=LLMAuthMode.CustomHeaders, + extensions={"kv_cache": {"mode": "affinity"}}, + verify_ssl=False, + ) + + release_client = create_model_client(release_config, _request_config()) + affinity_client = create_model_client(affinity_config, _request_config()) + + assert isinstance(release_client, OpenAIModelClient) + assert release_client.supports_kv_cache_release() + assert isinstance(affinity_client, OpenAIModelClient) + assert affinity_client.supports_kv_cache_affinity() diff --git a/tests/unit_tests/core/foundation/llm/test_model_client_tracer.py b/tests/unit_tests/core/foundation/llm/test_model_client_tracer.py deleted file mode 100644 index 627072253..000000000 --- a/tests/unit_tests/core/foundation/llm/test_model_client_tracer.py +++ /dev/null @@ -1,462 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - -from contextlib import asynccontextmanager -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client import InferenceAffinityModelClient -from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient -from openjiuwen.core.foundation.llm.model_clients.siliconflow_model_client import SiliconFlowModelClient -from openjiuwen.core.foundation.llm.schema.config import ModelClientConfig, ModelRequestConfig, ProviderType -from openjiuwen.core.foundation.llm.schema.message import UserMessage -from openjiuwen.core.runner.callback.events import LLMCallEvents - - -@pytest.fixture -def openai_client_config(): - """Create OpenAI client config for testing.""" - return ModelClientConfig( - client_provider=ProviderType.OpenAI, - api_key="sk-test", - api_base="https://api.openai.com/v1", - verify_ssl=False, - ) - - -@pytest.fixture -def siliconflow_client_config(): - """Create SiliconFlow client config for testing.""" - return ModelClientConfig( - client_provider=ProviderType.SiliconFlow, - api_key="sk-test", - api_base="https://api.siliconflow.cn/v1", - verify_ssl=False, - ) - - -@pytest.fixture -def inference_affinity_client_config(): - """Create InferenceAffinity client config for testing.""" - return ModelClientConfig( - client_provider=ProviderType.InferenceAffinity, - api_key="sk-test", - api_base="https://api.inference-affinity.test/v1", - verify_ssl=False, - ) - - -@pytest.fixture -def model_request_config(): - """Create model request config for testing.""" - return ModelRequestConfig( - model_name="gpt-3.5-turbo", - temperature=0.7, - ) - - -@pytest.fixture -def model_request_config_with_extra_params(): - """Create model request config with extra params for LLM_INPUT regression tests.""" - config = ModelRequestConfig( - model_name="gpt-3.5-turbo", - temperature=0.7, - ) - config.frequency_penalty = -1 - config.presence_penalty = 0.5 - config.stop = "END" - return config - - -class TestOpenAIModelClientTracer: - """Test OpenAIModelClient tracer_record_data functionality.""" - - @pytest.mark.asyncio - async def test_invoke_calls_tracer_record_data_with_result( - self, openai_client_config, model_request_config - ): - """Test that invoke calls tracer_record_data with llm_result parameter.""" - client = OpenAIModelClient(model_request_config, openai_client_config) - - # Mock response - mock_response = MagicMock() - mock_response.choices = [MagicMock()] - mock_response.choices[0].message = MagicMock() - mock_response.choices[0].message.content = "Test response" - mock_response.choices[0].message.tool_calls = None - mock_response.choices[0].message.reasoning_content = None - mock_response.choices[0].finish_reason = "stop" - mock_response.usage = MagicMock() - mock_response.usage.prompt_tokens = 10 - mock_response.usage.completion_tokens = 20 - mock_response.usage.total_tokens = 30 - mock_response.usage.prompt_tokens_details = None - - mock_async_client = AsyncMock() - mock_async_client.chat.completions.create = AsyncMock(return_value=mock_response) - - with patch.object( - client, "_create_async_openai_client", return_value=mock_async_client - ): - tracer_mock = AsyncMock() - - messages = [UserMessage(content="Hello")] - await client.invoke(messages, tracer_record_data=tracer_mock) - - # Verify tracer_record_data was called with llm_response - # It should be called twice: once with llm_params, once with llm_response - assert tracer_mock.call_count == 2 - call_kwargs = tracer_mock.call_args_list[1].kwargs - assert "llm_response" in call_kwargs - result = call_kwargs["llm_response"] - assert result.content == "Test response" - - @pytest.mark.asyncio - async def test_stream_accumulates_final_message_and_calls_tracer( - self, openai_client_config, model_request_config - ): - """Test that stream accumulates final_message and calls tracer_record_data.""" - client = OpenAIModelClient(model_request_config, openai_client_config) - - # Create mock streaming chunks - chunks = [] - for i, content in enumerate(["Hello", " ", "world", "!"]): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta = MagicMock() - chunk.choices[0].delta.content = content - chunk.choices[0].delta.reasoning_content = None - chunk.choices[0].delta.tool_calls = None - chunk.choices[0].finish_reason = None - chunk.usage = None - chunks.append(chunk) - - # Make last chunk have finish_reason - chunks[-1].choices[0].finish_reason = "stop" - - mock_async_client = AsyncMock() - - async def chunk_generator(): - for chunk in chunks: - yield chunk - - mock_async_client.chat.completions.create = AsyncMock(return_value=chunk_generator()) - - with patch.object( - client, "_create_async_openai_client", return_value=mock_async_client - ): - tracer_mock = AsyncMock() - - messages = [UserMessage(content="Hello")] - - collected_chunks = [] - async for chunk in client.stream(messages, tracer_record_data=tracer_mock): - collected_chunks.append(chunk) - - # Verify tracer_record_data was called - assert tracer_mock.call_count == 2 - call_kwargs = tracer_mock.call_args_list[1].kwargs - assert "llm_response" in call_kwargs - result = call_kwargs["llm_response"] - - # Verify final_message has accumulated content - assert result.content == "Hello world!" - - -class TestSiliconFlowModelClientTracer: - """Test SiliconFlowModelClient tracer_record_data functionality.""" - - @pytest.mark.asyncio - async def test_invoke_calls_tracer_record_data_with_result( - self, siliconflow_client_config, model_request_config - ): - """Test that invoke calls tracer_record_data with llm_result parameter.""" - client = SiliconFlowModelClient(model_request_config, siliconflow_client_config) - - # Mock response data - mock_response_data = { - "choices": [ - { - "message": { - "content": "Test response", - "tool_calls": None, - }, - "finish_reason": "stop", - } - ], - "usage": { - "prompt_tokens": 10, - "completion_tokens": 20, - "total_tokens": 30, - }, - } - - # Create proper async context manager mock - mock_response = AsyncMock() - mock_response.json = AsyncMock(return_value=mock_response_data) - - @asynccontextmanager - async def mock_post_gen(params, timeout=None): - yield mock_response - - with patch.object(client, "_apost", side_effect=mock_post_gen): - tracer_mock = AsyncMock() - - messages = [UserMessage(content="Hello")] - await client.invoke(messages, tracer_record_data=tracer_mock) - - # Verify tracer_record_data was called with llm_response - assert tracer_mock.call_count == 2 - call_kwargs = tracer_mock.call_args_list[1].kwargs - assert "llm_response" in call_kwargs - result = call_kwargs["llm_response"] - assert result.content == "Test response" - - @pytest.mark.asyncio - async def test_stream_accumulates_final_message_and_calls_tracer( - self, siliconflow_client_config, model_request_config - ): - """Test that stream accumulates final_message and calls tracer_record_data.""" - client = SiliconFlowModelClient(model_request_config, siliconflow_client_config) - - # Create mock SSE chunks - chunks = [ - b'data: {"choices": [{"delta": {"content": "Hello"}}]}\n', - b'data: {"choices": [{"delta": {"content": " "}}]}\n', - b'data: {"choices": [{"delta": {"content": "world"}}]}\n', - b'data: {"choices": [{"delta": {"content": "!"}, "finish_reason": "stop"}]}\n', - b'data: [DONE]\n', - ] - - mock_response = AsyncMock() - - async def content_gen(): - for chunk in chunks: - yield chunk - - mock_response.content = content_gen() - - @asynccontextmanager - async def mock_post_gen(params, timeout=None): - yield mock_response - - with patch.object(client, "_apost", side_effect=mock_post_gen): - tracer_mock = AsyncMock() - - messages = [UserMessage(content="Hello")] - - collected_chunks = [] - async for chunk in client.stream(messages, tracer_record_data=tracer_mock): - collected_chunks.append(chunk) - - # Verify tracer_record_data was called - assert tracer_mock.call_count == 2 - call_kwargs = tracer_mock.call_args_list[1].kwargs - assert "llm_response" in call_kwargs - result = call_kwargs["llm_response"] - - # Verify final_message has accumulated content - assert result.content == "Hello world!" - - @pytest.mark.asyncio - async def test_invoke_without_tracer_does_not_fail( - self, openai_client_config, model_request_config - ): - """Test that invoke works without tracer_record_data parameter.""" - client = OpenAIModelClient(model_request_config, openai_client_config) - - mock_response = MagicMock() - mock_response.choices = [MagicMock()] - mock_response.choices[0].message = MagicMock() - mock_response.choices[0].message.content = "Test response" - mock_response.choices[0].message.tool_calls = None - mock_response.choices[0].message.reasoning_content = None - mock_response.choices[0].finish_reason = "stop" - mock_response.usage = MagicMock() - mock_response.usage.prompt_tokens = 10 - mock_response.usage.completion_tokens = 20 - mock_response.usage.total_tokens = 30 - mock_response.usage.prompt_tokens_details = None - - mock_async_client = AsyncMock() - mock_async_client.chat.completions.create = AsyncMock(return_value=mock_response) - - with patch.object( - client, "_create_async_openai_client", return_value=mock_async_client - ): - messages = [UserMessage(content="Hello")] - result = await client.invoke(messages) - assert result.content == "Test response" - - @pytest.mark.asyncio - async def test_stream_without_tracer_does_not_fail( - self, openai_client_config, model_request_config - ): - """Test that stream works without tracer_record_data parameter.""" - client = OpenAIModelClient(model_request_config, openai_client_config) - - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta = MagicMock() - chunk.choices[0].delta.content = "Hello" - chunk.choices[0].delta.reasoning_content = None - chunk.choices[0].delta.tool_calls = None - chunk.choices[0].finish_reason = "stop" - chunk.usage = None - - mock_async_client = AsyncMock() - - async def chunk_generator(): - yield chunk - - mock_async_client.chat.completions.create = AsyncMock(return_value=chunk_generator()) - - with patch.object( - client, "_create_async_openai_client", return_value=mock_async_client - ): - messages = [UserMessage(content="Hello")] - collected = [] - async for c in client.stream(messages): - collected.append(c) - - assert len(collected) > 0 - assert collected[0].content == "Hello" - - -@pytest.mark.parametrize( - ( - "client_cls", - "client_config_fixture", - "trigger_patch_path", - "method_name", - "abort_method_name", - ), - [ - ( - OpenAIModelClient, - "openai_client_config", - "openjiuwen.core.foundation.llm.model_clients.openai_model_client.trigger", - "invoke", - "_create_async_openai_client", - ), - ( - OpenAIModelClient, - "openai_client_config", - "openjiuwen.core.foundation.llm.model_clients.openai_model_client.trigger", - "stream", - "_create_async_openai_client", - ), - ( - SiliconFlowModelClient, - "siliconflow_client_config", - "openjiuwen.core.foundation.llm.model_clients.siliconflow_model_client.trigger", - "invoke", - "_apost", - ), - ( - SiliconFlowModelClient, - "siliconflow_client_config", - "openjiuwen.core.foundation.llm.model_clients.siliconflow_model_client.trigger", - "stream", - "_apost", - ), - ( - InferenceAffinityModelClient, - "inference_affinity_client_config", - "openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client.trigger", - "invoke", - "_make_async_request", - ), - ( - InferenceAffinityModelClient, - "inference_affinity_client_config", - "openjiuwen.core.foundation.llm.model_clients.inference_affinity_model_client.trigger", - "stream", - "_stream_response", - ), - ], -) -@pytest.mark.asyncio -async def test_llm_input_includes_extra_request_params( - request, - client_cls, - client_config_fixture, - trigger_patch_path, - method_name, - abort_method_name, - model_request_config_with_extra_params, -): - client_config = request.getfixturevalue(client_config_fixture) - client = client_cls(model_request_config_with_extra_params, client_config) - trigger_mock = AsyncMock() - - with patch(trigger_patch_path, trigger_mock), patch.object( - client, abort_method_name, side_effect=RuntimeError("abort after LLM_INPUT") - ): - try: - if method_name == "stream": - async for _ in client.stream([UserMessage(content="Hello")]): - pass - else: - await client.invoke([UserMessage(content="Hello")]) - except Exception: - pass - - llm_input_call = next( - call for call in trigger_mock.call_args_list if call.args[0] == LLMCallEvents.LLM_INPUT - ) - assert llm_input_call.kwargs["frequency_penalty"] == -1 - assert llm_input_call.kwargs["presence_penalty"] == 0.5 - assert llm_input_call.kwargs["stop"] == "END" - - -@pytest.mark.asyncio -async def test_invoke_llm_output_trigger_forwards_reasoning_content( - openai_client_config, model_request_config -): - """Non-streaming invoke must forward reasoning_content to the LLM_OUTPUT trigger. - - Regression guard: the streaming path already passes - ``reasoning_content=final_message.reasoning_content``; the non-streaming - path used to omit it, so ``on_llm_output`` (which reads - ``kwargs.get("reasoning_content")``) got an empty string and never - created the reasoning sub-span for reasoning models (o1/o3/etc). - """ - client = OpenAIModelClient(model_request_config, openai_client_config) - - mock_response = MagicMock() - mock_response.choices = [MagicMock()] - mock_response.choices[0].message = MagicMock() - mock_response.choices[0].message.content = "Test response" - mock_response.choices[0].message.tool_calls = None - mock_response.choices[0].message.reasoning_content = "let me think" - mock_response.choices[0].finish_reason = "stop" - mock_response.usage = MagicMock() - mock_response.usage.prompt_tokens = 10 - mock_response.usage.completion_tokens = 20 - mock_response.usage.total_tokens = 30 - mock_response.usage.prompt_tokens_details = None - - mock_async_client = AsyncMock() - mock_async_client.chat.completions.create = AsyncMock(return_value=mock_response) - - trigger_mock = AsyncMock() - with patch.object( - client, "_create_async_openai_client", return_value=mock_async_client - ), patch( - "openjiuwen.core.foundation.llm.model_clients.openai_model_client.trigger", - trigger_mock, - ): - await client.invoke([UserMessage(content="Hello")]) - - llm_output_call = next( - call - for call in trigger_mock.call_args_list - if call.args[0] == LLMCallEvents.LLM_OUTPUT - ) - # reasoning_content must be forwarded, mirroring the streaming path. - assert llm_output_call.kwargs["reasoning_content"] == "let me think" - # content/usage/tool_calls contracts unchanged. - assert llm_output_call.kwargs["response"] == "Test response" - assert llm_output_call.kwargs["tool_calls"] is None diff --git a/tests/unit_tests/core/foundation/llm/test_openai_model_client.py b/tests/unit_tests/core/foundation/llm/test_openai_model_client.py index feff30195..f51045013 100644 --- a/tests/unit_tests/core/foundation/llm/test_openai_model_client.py +++ b/tests/unit_tests/core/foundation/llm/test_openai_model_client.py @@ -1,10 +1,15 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +import pytest + +from openjiuwen.core.common.exception.errors import BaseError from openjiuwen.core.foundation.llm import ( ModelClientConfig, ModelRequestConfig, + UserMessage, ) +from openjiuwen.core.foundation.llm.schema.config import LLMAuthMode from openjiuwen.core.foundation.llm.model_clients.openai_model_client import ( ModelParamRule, OpenAIModelClient, @@ -248,6 +253,201 @@ def test_default_minimax_predicate_is_case_sensitive(self): assert "extra_body" not in params +def test_deepseek_endpoint_profile_adds_reasoning_content_to_assistant_messages(): + client_config = ModelClientConfig( + client_provider="OpenAI", + endpoint_profile="deepseek", + api_key="sk-test-key", + api_base="https://api.deepseek.com/v1", + verify_ssl=False, + ) + client = OpenAIModelClient(ModelRequestConfig(model="deepseek-chat"), client_config) + + params = client._build_request_params( + messages=[ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ], + tools=None, + temperature=None, + top_p=None, + model=None, + stop=None, + max_tokens=None, + stream=False, + ) + + assert params["messages"][1]["reasoning_content"] == "" + + +def test_openai_none_auth_uses_placeholder_sdk_key(): + client_config = ModelClientConfig( + client_provider="OpenAI", + endpoint_profile="ollama", + api_base="http://localhost:11434/v1", + auth_mode=LLMAuthMode.NoneAuth, + verify_ssl=False, + ) + client = OpenAIModelClient(ModelRequestConfig(model="qwen2.5:7b"), client_config) + + assert client._resolved_api_key() == "EMPTY" + + +def test_dashscope_profile_converts_text_and_reference_images_for_generation(): + client_config = ModelClientConfig( + client_provider="OpenAI", + endpoint_profile="dashscope", + api_key="sk-test-key", + api_base="https://dashscope.aliyuncs.com", + verify_ssl=False, + ) + client = OpenAIModelClient(ModelRequestConfig(model="wan2.6-image"), client_config) + + content = client._dashscope_image_content([ + UserMessage(content=[ + {"text": "turn this into watercolor"}, + {"image": "https://example.test/source.png"}, + {"image_url": {"url": "https://example.test/ref.png"}}, + ]) + ]) + + assert content == [ + {"text": "turn this into watercolor"}, + {"image": "https://example.test/source.png"}, + {"image": "https://example.test/ref.png"}, + ] + + +@pytest.mark.parametrize( + "content", + [ + [{"text": "prompt", "extra": "ignored"}], + [{"text": "prompt"}, {"image": "https://example.test/a.png", "extra": "ignored"}], + [{"text": "prompt"}, {"image_url": {"url": ""}}], + [{"text": "prompt"}, {"type": "image", "image": "https://example.test/a.png"}], + ], +) +def test_dashscope_profile_rejects_invalid_image_generation_content(content): + with pytest.raises(BaseError): + OpenAIModelClient._dashscope_image_content([UserMessage(content=content)]) + + +@pytest.mark.parametrize( + ("voice", "language_type"), + [ + ("UnknownVoice", "Auto"), + ("Cherry", "UnknownLanguage"), + ], +) +def test_dashscope_profile_rejects_invalid_speech_params(voice, language_type): + with pytest.raises(BaseError): + OpenAIModelClient._validate_dashscope_speech_params( + voice=voice, + language_type=language_type, + ) + + +@pytest.mark.parametrize( + ("img_url", "size", "resolution"), + [ + ("https://example.test/a.png", "1280*720", None), + (None, None, "720P"), + ], +) +def test_dashscope_profile_rejects_mismatched_video_size_params(img_url, size, resolution): + with pytest.raises(BaseError): + OpenAIModelClient._validate_dashscope_video_params( + img_url=img_url, + size=size, + resolution=resolution, + ) + + +def test_openrouter_profile_adds_prompt_cache_markers_on_openai_client(): + client_config = ModelClientConfig( + client_provider="OpenAI", + endpoint_profile="openrouter", + api_key="sk-test-key", + api_base="https://openrouter.ai/api/v1", + verify_ssl=False, + ) + client = OpenAIModelClient(ModelRequestConfig(model="anthropic/claude-sonnet-4"), client_config) + + params = client._build_request_params( + messages=[{"role": "user", "content": "hello"}], + tools=[{"type": "function", "function": {"name": "search", "parameters": {}}}], + temperature=None, + top_p=None, + model=None, + stop=None, + max_tokens=None, + stream=False, + ) + + assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert params["tools"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_kv_release_fields_move_to_extra_body_for_openai_sdk(): + client_config = ModelClientConfig( + client_provider="OpenAI", + api_key="sk-test-key", + api_base="https://example.test/v1", + extensions={"kv_cache": {"mode": "release"}}, + verify_ssl=False, + ) + client = OpenAIModelClient(ModelRequestConfig(model="qwen"), client_config) + + params = client._build_request_params( + messages=[{"role": "user", "content": "hello"}], + tools=None, + temperature=None, + top_p=None, + model=None, + stop=None, + max_tokens=None, + stream=False, + session_id="session-1", + enable_cache_sharing=True, + ) + client._move_openai_extra_body_extensions(params) + + assert params["extra_body"]["cache_salt"] == "session-1" + assert params["extra_body"]["cache_sharing"] is True + assert "cache_salt" not in params + + +def test_kv_affinity_agent_hint_moves_to_extra_body_for_openai_sdk(): + client_config = ModelClientConfig( + client_provider="OpenAI", + api_base="https://example.test/v1", + auth_mode=LLMAuthMode.CustomHeaders, + extensions={"kv_cache": {"mode": "affinity"}}, + verify_ssl=False, + ) + client = OpenAIModelClient(ModelRequestConfig(model="qwen"), client_config) + + params = client._build_request_params( + messages=[{"role": "user", "content": "hello"}], + tools=None, + temperature=None, + top_p=None, + model=None, + stop=None, + max_tokens=None, + stream=False, + session_id="child", + parent_session_id="parent", + ) + client._move_openai_extra_body_extensions(params) + + assert params["extra_body"]["agent_hint"] == { + "session_id": "child", + "parent_session_id": "parent", + } + assert "agent_hint" not in params + + class _Delta: """Lightweight stand-in for an OpenAI SDK delta/message object.""" diff --git a/tests/unit_tests/core/foundation/llm/test_openrouter_model_client.py b/tests/unit_tests/core/foundation/llm/test_openrouter_model_client.py deleted file mode 100644 index 915211511..000000000 --- a/tests/unit_tests/core/foundation/llm/test_openrouter_model_client.py +++ /dev/null @@ -1,855 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from openjiuwen.core.foundation.llm import ( - Model, - ModelClientConfig, - ModelRequestConfig, - ProviderType, - UserMessage, -) - - -def _build_mock_response(content: str = "ok") -> MagicMock: - response = MagicMock() - response.choices = [MagicMock()] - response.choices[0].message = MagicMock() - response.choices[0].message.content = content - response.choices[0].message.tool_calls = None - response.choices[0].message.reasoning = None - response.choices[0].message.reasoning_content = None - response.choices[0].finish_reason = "stop" - response.usage = MagicMock() - response.usage.prompt_tokens = 5 - response.usage.completion_tokens = 3 - response.usage.total_tokens = 8 - response.usage.prompt_tokens_details = None - return response - - -def _build_stream_chunk(content: str = "ok") -> MagicMock: - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta = MagicMock() - chunk.choices[0].delta.content = content - chunk.choices[0].delta.reasoning = None - chunk.choices[0].delta.reasoning_content = None - chunk.choices[0].delta.tool_calls = None - chunk.choices[0].finish_reason = "stop" - chunk.usage = None - return chunk - - -class TestOpenRouterModelClient: - - def _make_configs(self, custom_headers=None, **client_config_kwargs): - client_config = ModelClientConfig( - client_provider="OpenRouter", - api_key="sk-or-test-key", - api_base="https://openrouter.ai/api/v1", - timeout=60.0, - verify_ssl=False, - custom_headers=custom_headers, - **client_config_kwargs, - ) - request_config = ModelRequestConfig(model="anthropic/claude-sonnet-4") - return client_config, request_config - - def test_no_default_attribution_headers(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - client = OpenRouterModelClient(request_config, client_config) - - assert "HTTP-Referer" not in client._base_headers - assert "X-OpenRouter-Title" not in client._base_headers - assert "X-OpenRouter-Categories" not in client._base_headers - - def test_configurable_headers_from_custom_headers(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - "X-OpenRouter-Title": "JiuwenSwarm", - "X-OpenRouter-Categories": "cli-agent,cloud-agent", - } - ) - client = OpenRouterModelClient(request_config, client_config) - - assert client._base_headers["HTTP-Referer"] == "https://openjiuwen.com/" - assert client._base_headers["X-OpenRouter-Title"] == "JiuwenSwarm" - assert client._base_headers["X-OpenRouter-Categories"] == "cli-agent,cloud-agent" - - def test_attribution_headers_protected_from_request_override(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - "X-OpenRouter-Title": "JiuwenSwarm", - } - ) - client = OpenRouterModelClient(request_config, client_config) - - effective = client._build_request_headers( - client._base_headers, - { - "HTTP-Referer": "https://evil.com", - "X-OpenRouter-Title": "EvilApp", - }, - ) - assert effective["HTTP-Referer"] == "https://openjiuwen.com/" - assert effective["X-OpenRouter-Title"] == "JiuwenSwarm" - - def test_custom_non_attribution_headers_preserved(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - custom_headers={"X-Custom-Header": "my-value"} - ) - client = OpenRouterModelClient(request_config, client_config) - - assert client._base_headers.get("X-Custom-Header") == "my-value" - - def test_non_attribution_headers_can_be_overridden(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - custom_headers={"X-Custom-Header": "original"} - ) - client = OpenRouterModelClient(request_config, client_config) - - effective = client._build_request_headers( - client._base_headers, - {"X-Custom-Header": "overridden"}, - ) - assert effective["X-Custom-Header"] == "overridden" - - def test_client_name_is_openrouter_only(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - assert "OpenRouter" in OpenRouterModelClient.__client_name__ - assert "OpenAI" not in OpenRouterModelClient.__client_name__ - - def test_attribution_protection_is_case_insensitive(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - "X-OpenRouter-Title": "JiuwenSwarm", - } - ) - client = OpenRouterModelClient(request_config, client_config) - - effective = client._build_request_headers( - client._base_headers, - { - "http-referer": "https://evil.com", - "x-openrouter-title": "EvilApp", - }, - ) - assert effective["HTTP-Referer"] == "https://openjiuwen.com/" - assert effective["X-OpenRouter-Title"] == "JiuwenSwarm" - - def test_prompt_cache_marks_last_tool_first_message_and_last_message_for_anthropic(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[ - {"role": "system", "content": "system prompt"}, - {"role": "user", "content": "message 1"}, - {"role": "assistant", "content": "message 2"}, - {"role": "user", "content": "message 3"}, - ], - tools=[ - {"type": "function", "function": {"name": "first", "parameters": {}}}, - {"type": "function", "function": {"name": "last", "parameters": {}}}, - ], - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert "cache_control" not in params["tools"][0] - assert params["tools"][1]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][3]["content"][0]["cache_control"] == {"type": "ephemeral"} - - def test_prompt_cache_adds_1h_ttl_for_anthropic_when_enabled(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_1h_prompt_cache_ttl=True, - ) - client = OpenRouterModelClient( - request_config, - client_config, - ) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=[{"type": "function", "function": {"name": "lookup", "parameters": {}}}], - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - expected_cache_control = {"type": "ephemeral", "ttl": "1h"} - assert params["tools"][0]["cache_control"] == expected_cache_control - assert params["messages"][0]["content"][0]["cache_control"] == expected_cache_control - - def test_prompt_cache_supports_openrouter_latest_model_tilde_prefix(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - request_config.model_name = "~anthropic/claude-opus-latest" - client = OpenRouterModelClient( - request_config, - client_config, - ) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=[{"type": "function", "function": {"name": "lookup", "parameters": {}}}], - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["model"] == "~anthropic/claude-opus-latest" - assert params["tools"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - - def test_prompt_cache_does_not_add_1h_ttl_for_qwen_when_enabled(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_1h_prompt_cache_ttl=True, - ) - request_config.model_name = "qwen/qwen3-max" - client = OpenRouterModelClient( - request_config, - client_config, - ) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=[{"type": "function", "function": {"name": "lookup", "parameters": {}}}], - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["tools"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - - def test_prompt_cache_warns_when_explicit_caching_unsupported(self): - from openjiuwen.core.foundation.llm.model_clients import openrouter_model_client - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - request_config.model_name = "openai/gpt-4o" - client = OpenRouterModelClient(request_config, client_config) - - with patch.object(openrouter_model_client.llm_logger, "warning") as warning: - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"] == "hello" - assert warning.call_count == 1 - assert "explicit prompt caching is enabled but unsupported" in warning.call_args.args[0] - assert warning.call_args.args[1] == "openai/gpt-4o" - - def test_prompt_cache_warns_when_1h_ttl_unsupported_but_explicit_cache_supported(self): - from openjiuwen.core.foundation.llm.model_clients import openrouter_model_client - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_1h_prompt_cache_ttl=True, - ) - request_config.model_name = "qwen/qwen3-max" - client = OpenRouterModelClient( - request_config, - client_config, - ) - - with patch.object(openrouter_model_client.llm_logger, "warning") as warning: - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - assert warning.call_count == 1 - assert "1h prompt-cache TTL is enabled but unsupported" in warning.call_args.args[0] - assert warning.call_args.args[1] == "qwen/qwen3-max" - - def test_prompt_cache_warns_for_both_unsupported_explicit_cache_and_1h_ttl(self): - from openjiuwen.core.foundation.llm.model_clients import openrouter_model_client - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_1h_prompt_cache_ttl=True, - ) - request_config.model_name = "openai/gpt-4o" - client = OpenRouterModelClient( - request_config, - client_config, - ) - - with patch.object(openrouter_model_client.llm_logger, "warning") as warning: - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"] == "hello" - assert warning.call_count == 2 - warning_messages = [call.args[0] for call in warning.call_args_list] - assert "explicit prompt caching is enabled but unsupported" in warning_messages[0] - assert "1h prompt-cache TTL is enabled but unsupported" in warning_messages[1] - - def test_prompt_cache_marks_longest_prefix_overlap_by_default(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - client = OpenRouterModelClient(request_config, client_config) - - first_messages = [ - {"role": "system", "content": "stable instructions"}, - {"role": "user", "content": "message 1"}, - {"role": "assistant", "content": "message 2"}, - {"role": "user", "content": "message 3"}, - ] - second_messages = [ - {"role": "system", "content": "stable instructions"}, - {"role": "user", "content": "message 1"}, - {"role": "assistant", "content": "message 2"}, - {"role": "user", "content": "different message 3"}, - ] - - client._build_request_params( - messages=first_messages, - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - params = client._build_request_params( - messages=second_messages, - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][2]["content"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][3]["content"][0]["cache_control"] == {"type": "ephemeral"} - - def test_prompt_cache_skips_prefix_overlap_when_disabled(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_prompt_cache_prefix_matching=False, - ) - client = OpenRouterModelClient(request_config, client_config) - - first_messages = [ - {"role": "system", "content": "stable instructions"}, - {"role": "user", "content": "message 1"}, - {"role": "assistant", "content": "message 2"}, - {"role": "user", "content": "message 3"}, - ] - second_messages = [ - {"role": "system", "content": "stable instructions"}, - {"role": "user", "content": "message 1"}, - {"role": "assistant", "content": "message 2"}, - {"role": "user", "content": "different message 3"}, - ] - - client._build_request_params( - messages=first_messages, - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - params = client._build_request_params( - messages=second_messages, - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert client._previous_prompt_cache_messages is None - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][3]["content"][0]["cache_control"] == {"type": "ephemeral"} - assert params["messages"][2]["content"] == "message 2" # No cache_control - - def test_prompt_cache_prefix_overlap_ignores_marker_text_block_shape(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - _longest_prefix_overlap_index, - ) - - previous_messages = [{ - "role": "system", - "content": [{ - "type": "text", - "text": "stable instructions", - "cache_control": {"type": "ephemeral"}, - }], - }] - current_messages = [{"role": "system", "content": "stable instructions"}] - - assert _longest_prefix_overlap_index(previous_messages, current_messages) == 0 - - def test_prompt_cache_respects_existing_message_cache_control(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[{ - "role": "user", - "content": [{ - "type": "text", - "text": "hello", - "cache_control": {"type": "ephemeral", "ttl": "1h"}, - }], - }], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"} - - def test_prompt_cache_applies_to_qwen_model_prefix(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - request_config.model_name = "qwen/qwen3-max" - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - - def test_prompt_cache_supported_providers_can_be_overridden_from_config(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_explicit_prompt_cache_providers={"openai"}, - ) - request_config.model_name = "openai/gpt-4o" - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"} - - def test_prompt_cache_1h_ttl_supported_providers_can_be_overridden_from_config(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_1h_prompt_cache_ttl=True, - openrouter_prompt_cache_1h_ttl_providers="anthropic,qwen", - ) - request_config.model_name = "qwen/qwen3-max" - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"][0]["cache_control"] == { - "type": "ephemeral", - "ttl": "1h", - } - - def test_prompt_cache_skips_unsupported_model_prefix(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - request_config.model_name = "openai/gpt-4o" - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"] == "hello" - - def test_prompt_cache_skips_when_explicit_caching_disabled(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs( - openrouter_enable_explicit_prompt_caching=False, - ) - client = OpenRouterModelClient(request_config, client_config) - - params = client._build_request_params( - messages=[{"role": "user", "content": "hello"}], - tools=[{"type": "function", "function": {"name": "lookup", "parameters": {}}}], - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"] == "hello" - assert "cache_control" not in params["tools"][0] - assert client._previous_prompt_cache_messages is None - - def test_prompt_cache_disabled_clears_previous_message_state(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - enabled_client_config, request_config = self._make_configs( - openrouter_enable_prompt_cache_prefix_matching=True, - ) - client = OpenRouterModelClient(request_config, enabled_client_config) - - client._build_request_params( - messages=[{"role": "system", "content": "stable"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - assert client._previous_prompt_cache_messages is not None - - disabled_client_config, _ = self._make_configs( - openrouter_enable_explicit_prompt_caching=False, - openrouter_enable_prompt_cache_prefix_matching=True, - ) - client = OpenRouterModelClient(request_config, disabled_client_config) - client._previous_prompt_cache_messages = [{"role": "system", "content": "stale"}] - params = client._build_request_params( - messages=[{"role": "system", "content": "stable"}], - tools=None, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert params["messages"][0]["content"] == "stable" - assert client._previous_prompt_cache_messages is None - - def test_prompt_cache_does_not_mutate_caller_messages_or_tools(self): - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config, request_config = self._make_configs() - client = OpenRouterModelClient(request_config, client_config) - messages = [{"role": "user", "content": "hello"}] - tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}] - - client._build_request_params( - messages=messages, - tools=tools, - temperature=None, - top_p=None, - model=None, - stop=None, - max_tokens=None, - stream=False, - ) - - assert messages == [{"role": "user", "content": "hello"}] - assert tools == [{"type": "function", "function": {"name": "lookup", "parameters": {}}}] - - -class TestOpenRouterFactoryRouting: - - def test_factory_routes_openrouter_to_dedicated_client(self): - from openjiuwen.core.foundation.llm.model_clients import create_model_client - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config = ModelClientConfig( - client_provider="OpenRouter", - api_key="sk-or-test-key", - api_base="https://openrouter.ai/api/v1", - timeout=60.0, - verify_ssl=False, - ) - request_config = ModelRequestConfig(model="anthropic/claude-sonnet-4") - client = create_model_client(client_config, request_config) - assert isinstance(client, OpenRouterModelClient) - - def test_factory_routes_openai_to_openai_client(self): - from openjiuwen.core.foundation.llm.model_clients import create_model_client - from openjiuwen.core.foundation.llm.model_clients.openai_model_client import ( - OpenAIModelClient, - ) - from openjiuwen.core.foundation.llm.model_clients.openrouter_model_client import ( - OpenRouterModelClient, - ) - client_config = ModelClientConfig( - client_provider="OpenAI", - api_key="sk-test-key", - api_base="https://api.openai.com/v1", - timeout=60.0, - verify_ssl=False, - ) - request_config = ModelRequestConfig(model="gpt-4o") - client = create_model_client(client_config, request_config) - assert isinstance(client, OpenAIModelClient) - assert not isinstance(client, OpenRouterModelClient) - - def test_openai_client_no_longer_registers_openrouter(self): - from openjiuwen.core.foundation.llm.model_clients.openai_model_client import ( - OpenAIModelClient, - ) - names = OpenAIModelClient.__client_name__ - if isinstance(names, list): - assert "OpenRouter" not in names - else: - assert names != "OpenRouter" - - -class TestOpenRouterModelIntegration: - - async def _invoke_and_get_sent_headers(self, model: Model, request_headers=None) -> dict: - mock_async_client = AsyncMock() - mock_async_client.chat.completions.create = AsyncMock(return_value=_build_mock_response()) - - invoke_kwargs = {} - if request_headers is not None: - invoke_kwargs["custom_headers"] = request_headers - - with patch.object(model._client, "_create_async_openai_client", return_value=mock_async_client): - await model.invoke(messages=[UserMessage(content="hello")], **invoke_kwargs) - - sent_params = mock_async_client.chat.completions.create.call_args.kwargs - return sent_params.get("extra_headers", {}) - - @pytest.mark.asyncio - async def test_openrouter_model_sends_attribution_headers(self): - model = Model( - model_client_config=ModelClientConfig( - client_provider=ProviderType.OpenRouter, - api_key="sk-or-test", - api_base="https://openrouter.ai/api/v1", - verify_ssl=False, - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - "X-OpenRouter-Title": "JiuwenSwarm", - "X-OpenRouter-Categories": "cli-agent,cloud-agent", - }, - ), - model_config=ModelRequestConfig(model="anthropic/claude-sonnet-4"), - ) - - sent_headers = await self._invoke_and_get_sent_headers(model) - - assert sent_headers["HTTP-Referer"] == "https://openjiuwen.com/" - assert sent_headers["X-OpenRouter-Title"] == "JiuwenSwarm" - assert sent_headers["X-OpenRouter-Categories"] == "cli-agent,cloud-agent" - - @pytest.mark.asyncio - async def test_openrouter_model_protects_attribution_from_request_headers(self): - model = Model( - model_client_config=ModelClientConfig( - client_provider=ProviderType.OpenRouter, - api_key="sk-or-test", - api_base="https://openrouter.ai/api/v1", - verify_ssl=False, - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - "X-OpenRouter-Title": "JiuwenSwarm", - }, - ), - model_config=ModelRequestConfig(model="anthropic/claude-sonnet-4"), - ) - - sent_headers = await self._invoke_and_get_sent_headers( - model, - request_headers={ - "HTTP-Referer": "https://evil.com", - "X-OpenRouter-Title": "EvilApp", - }, - ) - - assert sent_headers["HTTP-Referer"] == "https://openjiuwen.com/" - assert sent_headers["X-OpenRouter-Title"] == "JiuwenSwarm" - - @pytest.mark.asyncio - async def test_openrouter_model_allows_non_attribution_request_headers(self): - model = Model( - model_client_config=ModelClientConfig( - client_provider=ProviderType.OpenRouter, - api_key="sk-or-test", - api_base="https://openrouter.ai/api/v1", - verify_ssl=False, - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - }, - ), - model_config=ModelRequestConfig(model="anthropic/claude-sonnet-4"), - ) - - sent_headers = await self._invoke_and_get_sent_headers( - model, - request_headers={ - "X-Custom-Request": "request-value", - }, - ) - - assert sent_headers["HTTP-Referer"] == "https://openjiuwen.com/" - assert sent_headers["X-Custom-Request"] == "request-value" - - @pytest.mark.asyncio - async def test_openrouter_stream_sends_attribution_headers(self): - model = Model( - model_client_config=ModelClientConfig( - client_provider=ProviderType.OpenRouter, - api_key="sk-or-test", - api_base="https://openrouter.ai/api/v1", - verify_ssl=False, - custom_headers={ - "HTTP-Referer": "https://openjiuwen.com/", - "X-OpenRouter-Title": "JiuwenSwarm", - }, - ), - model_config=ModelRequestConfig(model="anthropic/claude-sonnet-4"), - ) - - async def chunk_generator(): - yield _build_stream_chunk("hello") - - mock_async_client = AsyncMock() - mock_async_client.chat.completions.create = AsyncMock(return_value=chunk_generator()) - - with patch.object(model._client, "_create_async_openai_client", return_value=mock_async_client): - async for _ in model.stream(messages=[UserMessage(content="hello")]): - pass - - sent_params = mock_async_client.chat.completions.create.call_args.kwargs - sent_headers = sent_params.get("extra_headers", {}) - - assert sent_headers["HTTP-Referer"] == "https://openjiuwen.com/" - assert sent_headers["X-OpenRouter-Title"] == "JiuwenSwarm"