From 71929afe34773e41f7fa63391a033fe0451e57e3 Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Fri, 10 Jul 2026 02:43:11 +0000 Subject: [PATCH 1/2] feat(advisor): add executor+advisor profile with tool_call and review_gate strategies Signed-off-by: zengyuanl --- .agents/skills/switchyard-lib-core/SKILL.md | 1 + AGENTS.md | 2 + .../src/backends/anthropic.rs | 48 +- .../tests/adversarial_native_backends.rs | 108 ++-- switchyard/__init__.py | 7 + switchyard/cli/route_bundle.py | 72 +++ .../lib/backends/advisor_loop_backend.py | 507 +++++++++++++++++ .../lib/backends/advisor_tool_call_backend.py | 516 ++++++++++++++++++ switchyard/lib/cost_estimator.py | 9 + switchyard/lib/profiles/__init__.py | 6 + switchyard/lib/profiles/advisor.py | 72 +++ switchyard/lib/profiles/advisor_config.py | 134 +++++ switchyard/lib/profiles/advisor_presets.py | 101 ++++ switchyard/lib/profiles/advisor_prompts.py | 102 ++++ tests/test_advisor_loop_backend.py | 310 +++++++++++ tests/test_advisor_profile.py | 161 ++++++ tests/test_advisor_tool_call_backend.py | 409 ++++++++++++++ tests/test_route_bundle.py | 75 +++ 18 files changed, 2580 insertions(+), 60 deletions(-) create mode 100644 switchyard/lib/backends/advisor_loop_backend.py create mode 100644 switchyard/lib/backends/advisor_tool_call_backend.py create mode 100644 switchyard/lib/profiles/advisor.py create mode 100644 switchyard/lib/profiles/advisor_config.py create mode 100644 switchyard/lib/profiles/advisor_presets.py create mode 100644 switchyard/lib/profiles/advisor_prompts.py create mode 100644 tests/test_advisor_loop_backend.py create mode 100644 tests/test_advisor_profile.py create mode 100644 tests/test_advisor_tool_call_backend.py diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index de8b3b05..5f6b10ec 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -48,6 +48,7 @@ right validation set. If the change is driven by a launcher need, also read | Per-event error log | Use `switchyard.lib.endpoints.upstream_error_log.log_upstream_attempt_failure(...)` on the failure path. Events belong in logs/traces, not Prometheus sample timestamps. | | CLI launcher integration | Build one profile-backed `SwitchyardApp` with `build_tier_passthrough_switchyard(...)` for single-target mode, or merge route YAML with `load_route_bundle_table(...)`. Hand the result to `build_switchyard_app`. | | A new preset | Put preset helpers beside the profile config they produce, under `switchyard/lib/profiles/`. Presets should return typed config objects, not runnable chains. | +| A backend that pairs the executor with a stronger advisor model | `AdvisorProfileConfig` (`switchyard/lib/profiles/advisor.py`) dispatches on `AdvisorConfig.strategy`. **`tool_call`** (default) builds `AdvisorToolCallBackend` (`switchyard/lib/backends/advisor_tool_call_backend.py`): the proxy-side re-creation of Anthropic's `advisor_20260301` server tool — a real, **parameterless** `advisor` tool is appended to the client's tools (plus doc-verbatim executor steering and a length line injected cache-stably: system prepend + **first** user message); each advisor `tool_use` is intercepted before it reaches the client, the advisor is consulted on the transcript (tools summary + tail-kept conversation + the executor's current-turn text), and the advice loops back as a `tool_result` until the turn is advisor-free (`max_uses` consults per request, then `max_uses exceeded` error results; hard cap 8 turns; mixed advisor+client-tool turns are regenerated with siblings dropped). **`review_gate`** builds `AdvisorLoopBackend` (`switchyard/lib/backends/advisor_loop_backend.py`): no tool injected; at the executor's first **no-tool-call** turn (a plan, or "done") the advisor reviews **once per session** (hash of the conversation prefix, in-process) → `APPROVE` (return as-is) or `REDO` (feed the plan back and re-invoke) — near-superset of solo; front-loaded advice was found to cause premature convergence on Opus executors (see its docstring). Compose via a `type: advisor` route (`cli/route_bundle.py`); preset `AdvisorPresets.opus47_exec_opus48_advisor(strategy=...)`. Both tiers are native Anthropic (`/v1/messages`, Bearer): the executor call is delegated **verbatim** to an `AnthropicNativeBackend` so the client's `cache_control` (prompt caching) survives — no OpenAI translation. Like `LatencyServiceLLMBackend`, both are Python multi-call backends doing their **own** stats accounting (`ctx.selected_model`, plus advisor consults — and, for tool_call, the intercepted executor turns — into the planner bucket) — do **not** wrap in `StatsLlmBackend`; `with_runtime_components` attaches the accumulator through the `_stats` hook. Prompts in `switchyard/lib/profiles/advisor_prompts.py`. | ## Profile Pattern diff --git a/AGENTS.md b/AGENTS.md index 4d474a4f..4c8e5677 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -167,6 +167,8 @@ switchyard/ │ │ ├── openai_llm_backend.py # OpenAiPassthroughBackend │ │ ├── openai_native_backend.py # OpenAiNativeBackend │ │ ├── anthropic_native_llm_backend.py # AnthropicNativeBackend +│ │ ├── advisor_tool_call_backend.py # AdvisorToolCallBackend (advisor tool_call strategy) +│ │ ├── advisor_loop_backend.py # AdvisorLoopBackend (advisor review_gate strategy) │ │ ├── latency_service_llm_backend.py # LatencyServiceLLMBackend │ │ ├── llm_target.py # LlmTarget, BackendFormat │ │ ├── multi_llm_backend.py # MultiLlmBackend helpers diff --git a/crates/switchyard-components/src/backends/anthropic.rs b/crates/switchyard-components/src/backends/anthropic.rs index be76fd01..681655c3 100644 --- a/crates/switchyard-components/src/backends/anthropic.rs +++ b/crates/switchyard-components/src/backends/anthropic.rs @@ -80,24 +80,36 @@ impl AnthropicNativeBackend { } fn outbound_body(&self, request: &ChatRequest) -> Result { - let mut body = match request.request_type() { - ChatRequestType::Anthropic => request.body().clone(), - source => { - self.translation - .translate_request( - request_wire_format(source), - WireFormat::AnthropicMessages, - request.body(), - &self.translation_policy, - ) - .map_err(|error| { - SwitchyardError::Backend(format!( - "failed to translate {source:?} request to Anthropic Messages: {error}" - )) - })? - .body - } - }; + // Native Anthropic requests are a TRUE passthrough: forward the client's + // body verbatim, rewriting only the model id (routing) and merging any + // operator-configured `extra_body`. The strips/normalization in the + // translated branch below exist for OpenAI/Responses -> Anthropic bodies, + // which can carry fields/shapes the Anthropic API rejects. Applying them + // to a real Anthropic client (e.g. Claude Code) silently drops valid + // Anthropic features — `context_management` (context auto-compaction), + // signed thinking blocks, mid-conversation system turns, client tool ids + // — which a passthrough must never do. + if matches!(request.request_type(), ChatRequestType::Anthropic) { + let mut body = request.body().clone(); + set_json_model(&mut body, self.target.model.as_str()); + merge_target_extra_body(&mut body, self.target.extra_body.as_ref()); + return Ok(body); + } + let mut body = self + .translation + .translate_request( + request_wire_format(request.request_type()), + WireFormat::AnthropicMessages, + request.body(), + &self.translation_policy, + ) + .map_err(|error| { + SwitchyardError::Backend(format!( + "failed to translate {:?} request to Anthropic Messages: {error}", + request.request_type() + )) + })? + .body; set_json_model(&mut body, self.target.model.as_str()); strip_anthropic_incompatible_fields(&mut body); normalize_anthropic_body(&mut body); diff --git a/crates/switchyard-components/tests/adversarial_native_backends.rs b/crates/switchyard-components/tests/adversarial_native_backends.rs index f3e3b7ce..20ec54a4 100644 --- a/crates/switchyard-components/tests/adversarial_native_backends.rs +++ b/crates/switchyard-components/tests/adversarial_native_backends.rs @@ -669,9 +669,9 @@ fn anthropic_backend_is_anthropic_only() -> Result<()> { Ok(()) } -// Non-streaming Anthropic calls should strip incompatible fields and stamp context. +// Non-streaming native Anthropic calls forward the body verbatim and stamp context. #[tokio::test] -async fn anthropic_non_streaming_strips_incompatible_fields_and_records_context() -> Result<()> { +async fn anthropic_native_passthrough_forwards_body_verbatim() -> Result<()> { let server = OneShotServer::json( 200, json!({ @@ -731,16 +731,22 @@ async fn anthropic_non_streaming_strips_incompatible_fields_and_records_context( ); assert_eq!(request.body["model"], "target-claude"); assert_eq!(request.body["messages"][0]["content"], "hello"); - assert!(request.body.get("reasoning_effort").is_none()); - assert!(request.body.get("context_management").is_none()); + // Native Anthropic is a verbatim passthrough: client fields are preserved, + // not stripped (only the model id is rewritten for routing). Stripping these + // belongs to the translated path, not a real /v1/messages request. + assert_eq!(request.body["reasoning_effort"], "high"); + assert_eq!( + request.body["context_management"], + json!({"strategy": "auto"}) + ); assert_eq!(request.body["made_up_beta_field"], json!({"kept": true})); assert_eq!(request.body["extra_body"], json!({"caller": "value"})); Ok(()) } -// Anthropic-native calls should downgrade Opus-4.8-style system turns for legacy targets. +// Native Anthropic passthrough must NOT relocate message-level system/developer turns. #[tokio::test] -async fn anthropic_lifts_message_level_system_roles_before_native_call() -> Result<()> { +async fn anthropic_native_passthrough_does_not_lift_system_messages() -> Result<()> { let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; let mut ctx = ProxyContext::new(); @@ -767,28 +773,24 @@ async fn anthropic_lifts_message_level_system_roles_before_native_call() -> Resu .await?; let request = server.captured()?; - assert_eq!(request.body["system"], "System rules.\n\nDeveloper rules."); - let messages = request.body["messages"] - .as_array() - .ok_or_else(|| SwitchyardError::Other("messages should be an array".to_string()))?; - let roles = messages - .iter() - .map(|message| { - message - .get("role") - .and_then(Value::as_str) - .unwrap_or("") - }) - .collect::>(); - assert_eq!(roles, vec!["user", "assistant"]); - assert_eq!(messages[0]["content"], "hello"); - assert_eq!(messages[1]["content"], "ready"); + // Verbatim: no top-level system is synthesized, and the system/developer + // turns stay exactly where the client put them. + assert!(request.body.get("system").is_none()); + assert_eq!( + request.body["messages"], + json!([ + {"role": "system", "content": "System rules."}, + {"role": "user", "content": "hello"}, + {"role": "developer", "content": [{"type": "text", "text": "Developer rules."}]}, + {"role": "assistant", "content": "ready"} + ]) + ); Ok(()) } -// Interleaved system turns should preserve encounter order after lifting. +// Interleaved system/developer turns are preserved in place (no lifting) on passthrough. #[tokio::test] -async fn anthropic_lifts_multiple_interleaved_system_messages_in_order() -> Result<()> { +async fn anthropic_native_passthrough_preserves_interleaved_system_messages() -> Result<()> { let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; let mut ctx = ProxyContext::new(); @@ -813,24 +815,25 @@ async fn anthropic_lifts_multiple_interleaved_system_messages_in_order() -> Resu .await?; let request = server.captured()?; - assert_eq!( - request.body["system"], - "Top-level rules.\n\nFirst lifted system.\n\nSecond lifted system.\n\nDeveloper lifted system." - ); + // Verbatim: top-level system unchanged, all message-level turns preserved in place. + assert_eq!(request.body["system"], "Top-level rules."); assert_eq!( request.body["messages"], json!([ + {"role": "system", "content": "First lifted system."}, {"role": "user", "content": "first user"}, + {"role": "system", "content": "Second lifted system."}, {"role": "assistant", "content": "assistant reply"}, + {"role": "developer", "content": "Developer lifted system."}, {"role": "user", "content": "second user"} ]) ); Ok(()) } -// Existing structured Anthropic system prompts should keep their shape when lifted text is added. +// Structured system prompt and message-level system (incl. non-text blocks) pass through untouched. #[tokio::test] -async fn anthropic_lifts_message_level_system_into_existing_system_blocks() -> Result<()> { +async fn anthropic_native_passthrough_preserves_structured_system_and_messages() -> Result<()> { let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; let mut ctx = ProxyContext::new(); @@ -858,16 +861,25 @@ async fn anthropic_lifts_message_level_system_into_existing_system_blocks() -> R .await?; let request = server.captured()?; + // Verbatim: structured system kept as-is; the message-level system turn + // (including its image block) is preserved, not downgraded into system. assert_eq!( request.body["system"], - json!([ - {"type": "text", "text": "Existing system."}, - {"type": "text", "text": "Lifted system.\n\nLifted input text."} - ]) + json!([{"type": "text", "text": "Existing system."}]) ); assert_eq!( request.body["messages"], - json!([{"role": "user", "content": "hello"}]) + json!([ + { + "role": "system", + "content": [ + {"type": "text", "text": "Lifted system."}, + {"type": "image", "source": {"type": "url", "url": "https://example.test/a.png"}}, + {"type": "input_text", "text": "Lifted input text."} + ] + }, + {"role": "user", "content": "hello"} + ]) ); Ok(()) } @@ -898,9 +910,9 @@ async fn anthropic_translates_responses_requests_with_default_max_tokens() -> Re Ok(()) } -// Invalid Anthropic tool-use IDs should be sanitized consistently with results. +// Native Anthropic passthrough preserves client tool-use IDs verbatim (no sanitization). #[tokio::test] -async fn anthropic_sanitizes_invalid_tool_use_ids_and_matching_results() -> Result<()> { +async fn anthropic_native_passthrough_preserves_tool_use_ids() -> Result<()> { let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; let mut ctx = ProxyContext::new(); @@ -936,8 +948,10 @@ async fn anthropic_sanitizes_invalid_tool_use_ids_and_matching_results() -> Resu .await?; let request = server.captured()?; + // Verbatim: the client's tool-use id is forwarded unchanged (sanitization + // belongs to the translated path; a real Anthropic client sends valid ids). let tool_use_id = &request.body["messages"][1]["content"][0]["id"]; - assert_eq!(tool_use_id, "toolu_01_bad_id"); + assert_eq!(tool_use_id, "toolu_01*bad:id"); assert_eq!( &request.body["messages"][2]["content"][0]["tool_use_id"], tool_use_id @@ -945,9 +959,9 @@ async fn anthropic_sanitizes_invalid_tool_use_ids_and_matching_results() -> Resu Ok(()) } -// Unsigned synthetic thinking blocks should be removed before Anthropic replay. +// Native Anthropic passthrough preserves thinking blocks verbatim (no stripping). #[tokio::test] -async fn anthropic_strips_unsigned_thinking_blocks_before_native_call() -> Result<()> { +async fn anthropic_native_passthrough_preserves_thinking_blocks() -> Result<()> { let server = OneShotServer::json(200, json!({"id": "msg-test", "content": []}))?; let backend = AnthropicNativeBackend::new(anthropic_target(server.base_url().to_string())?)?; let mut ctx = ProxyContext::new(); @@ -985,15 +999,22 @@ async fn anthropic_strips_unsigned_thinking_blocks_before_native_call() -> Resul .await?; let request = server.captured()?; + // Verbatim: thinking blocks (signed or not) are preserved; the upstream API + // decides what to accept. Stripping unsigned blocks belongs to the translated + // path, where they are synthetic translation artifacts. assert_eq!( request.body["messages"][0]["content"] .as_array() .ok_or_else(|| SwitchyardError::Other("content should be an array".to_string()))? .len(), - 1 + 2 ); assert_eq!( request.body["messages"][0]["content"][0]["type"], + "thinking" + ); + assert_eq!( + request.body["messages"][0]["content"][1]["type"], "tool_use" ); assert_eq!( @@ -1004,7 +1025,10 @@ async fn anthropic_strips_unsigned_thinking_blocks_before_native_call() -> Resul request.body["messages"][1]["content"][0]["thinking"], "real" ); - assert_eq!(request.body["messages"][2]["content"], ""); + assert_eq!( + request.body["messages"][2]["content"], + json!([{"type": "thinking", "thinking": "only synthetic"}]) + ); Ok(()) } diff --git a/switchyard/__init__.py b/switchyard/__init__.py index cb179043..ee899c91 100644 --- a/switchyard/__init__.py +++ b/switchyard/__init__.py @@ -55,6 +55,9 @@ RouteLLMRequestProcessor, ) from switchyard.lib.profiles import ( + AdvisorConfig, + AdvisorPresets, + AdvisorProfileConfig, ClassifierConfig, ContextAwareProfile, DeterministicRoutingConfig, @@ -156,6 +159,10 @@ def __getattr__(name: str) -> Any: # Chain infrastructure "Switchyard", "LLMBackend", + # Advisor (executor consults a stronger advisor model) + "AdvisorConfig", + "AdvisorPresets", + "AdvisorProfileConfig", "NoopProfile", "NoopProfileConfig", "StageRouterConfig", diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index c41a13a8..2e062ba1 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -36,6 +36,7 @@ from switchyard.lib.processors.llm_classifier import DEFAULT_MAX_REQUEST_CHARS from switchyard.lib.processors.llm_classifier.presets import PROFILE_FACTORIES from switchyard.lib.profiles import ( + AdvisorProfileConfig, DeterministicRoutingProfileConfig, LatencyServiceProfileConfig, PlanExecuteProfileConfig, @@ -43,6 +44,7 @@ RouteLLMProfileConfig, StageRouterProfileConfig, ) +from switchyard.lib.profiles.advisor_config import AdvisorConfig from switchyard.lib.profiles.deterministic_routing_config import DeterministicRoutingConfig from switchyard.lib.profiles.plan_execute_config import PlanExecuteConfig from switchyard.lib.profiles.plan_execute_presets import PlanExecutePresets @@ -264,6 +266,28 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "fallback_target_on_evict", }) ) +_ADVISOR_ROUTE_KEYS = ( + _ROUTE_METADATA_KEYS + | _TARGET_DEFAULT_ROUTE_KEYS + | frozenset({ + "executor", + "advisor", + "strategy", + "advisor_tool_name", + "max_uses", + "inject_steering", + "executor_steering", + "advisor_length_line", + "advisor_system_prompt", + "advisor_tool_description", + "reviewer_system_prompt", + "advisor_max_tokens", + "advisor_temperature", + "transcript_max_chars", + "fail_open", + "enable_stats", + }) +) _DETERMINISTIC_CLASSIFIER_KEYS = frozenset({ "model", "api_key", @@ -300,6 +324,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "deterministic": _DETERMINISTIC_ROUTE_KEYS, "stage_router": _STAGE_ROUTER_ROUTE_KEYS, "plan_execute": _PLAN_EXECUTE_ROUTE_KEYS, + "advisor": _ADVISOR_ROUTE_KEYS, } _DEFAULT_KEYS_BY_TYPE: Mapping[str, frozenset[str]] = { "model": _TARGET_DEFAULT_KEYS, @@ -311,6 +336,7 @@ def llm_target_to_route_dict(target: LlmTarget) -> dict[str, Any]: "deterministic": _TARGET_DEFAULT_KEYS, "stage_router": _TARGET_DEFAULT_KEYS, "plan_execute": _TARGET_DEFAULT_KEYS, + "advisor": _TARGET_DEFAULT_KEYS, } # Shipping planner/executor model ids for `type: plan_execute` routes, so an @@ -856,6 +882,16 @@ def _build_switchyard_for_route( extra_response_processors=extra_response_processors, ) + if route_type == "advisor": + return _advisor_switchyard( + model_id, + route, + target_defaults=target_defaults, + stats=stats, + pre_routing_request_processors=pre_routing_request_processors, + extra_response_processors=extra_response_processors, + ) + raise RouteBundleConfigError(f"unsupported route type {route_type!r}") @@ -1111,6 +1147,41 @@ def _plan_execute_switchyard( ) +def _advisor_switchyard( + model_id: str, # noqa: ARG001 - kept for dispatch-call symmetry with siblings + route: Mapping[str, object], + *, + target_defaults: Mapping[str, object], + stats: StatsAccumulator, + pre_routing_request_processors: Sequence[Any] = (), + extra_response_processors: Sequence[Any] = (), +) -> ChainRuntime: + """Build an executor + advisor chain from a ``type: advisor`` route. + + Mirrors :func:`_plan_execute_switchyard`: the ``executor`` / ``advisor`` + tiers and scalar fields go through the shared ``_route_config`` → + :meth:`AdvisorConfig.model_validate` path. Both tiers are required and both + are typically ``format: anthropic`` (native ``/v1/messages``, so the + executor passthrough preserves prompt caching). ``strategy`` selects the + advisor mode: ``tool_call`` (default) offers the executor a + proxy-intercepted ``advisor`` tool; ``review_gate`` gates the executor with + a once-per-session advisor review. + """ + config = AdvisorConfig.model_validate( + _route_config(route, target_defaults, ("executor", "advisor")) + ) + return ProfileSwitchyard( + AdvisorProfileConfig.from_config(config) + .build() + .with_runtime_components( + stats_accumulator=stats, + enable_stats=config.enable_stats, + pre_request_processors=pre_routing_request_processors, + response_processors=extra_response_processors, + ) + ) + + def _passthrough_target( model_id: str, route: Mapping[str, object], @@ -1358,6 +1429,7 @@ def _route_type(model_id: str, route: Mapping[str, object]) -> str: "stage_router_routing": "stage_router", "plan": "plan_execute", "plan_execute": "plan_execute", + "advisor": "advisor", } try: return aliases[normalized] diff --git a/switchyard/lib/backends/advisor_loop_backend.py b/switchyard/lib/backends/advisor_loop_backend.py new file mode 100644 index 00000000..74293e3a --- /dev/null +++ b/switchyard/lib/backends/advisor_loop_backend.py @@ -0,0 +1,507 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``LLMBackend`` that gates the executor with a once-per-session advisor review. + +This is the ``review_gate`` strategy of :class:`AdvisorConfig`; the +executor-triggered tool-call strategy lives in +``switchyard/lib/backends/advisor_tool_call_backend.py``. + +An earlier design offered the executor an ``advisor`` tool it could call +mid-generation. Trace analysis showed that front-loading the advisor's plan +*suppressed the executor's own test-and-iterate loop* — it trusted the plan, +one-shot it, and declared "done" prematurely (e.g. solving a concurrency task in +4 turns vs the 17 the unadvised baseline needed to catch the bug). Net effect +was within noise, with real losses on tasks the baseline solved by iterating. + +This backend instead uses the advisor as a **once-per-session review gate**: + +1. The executor works the task with its **own** tools (no advisor tool injected, + no upfront advice) — its iteration loop is untouched. +2. The first time the executor produces a turn with **no tool calls** — either a + plan it is about to execute, or a claim that the task is complete — the + backend consults the advisor **once** to review the full transcript: + - ``APPROVE`` → the executor's turn is returned unchanged (sound plan / done). + - ``REDO`` → the advisor's optimized plan is fed back as a user turn and the + executor is re-invoked to **keep working** (it produces tool calls again). +3. Subsequent turns in the same session pass through unreviewed + (once-per-session), so the gate can force at most one extra round of work. + +This is a near-superset of solo behavior — identical to the bare executor until +"done", plus one quality gate — so it is downside-protected (≈ baseline if the +advisor always approves) while catching premature convergence. + +Both tiers are **native Anthropic** (``/v1/messages``, Bearer auth) — no OpenAI +anywhere. The executor is called by delegating verbatim to an +:class:`AnthropicNativeBackend`, so the client's ``cache_control`` breakpoints +reach the upstream unchanged and prompt caching is honored. The gate reads the +executor's tool use from the Anthropic ``stop_reason``/``tool_use`` content +blocks. The advisor is consulted via the Anthropic Messages API. + +Chain integration:: + + [RequestProcessor*] → AdvisorLoopBackend → [ResponseProcessor*] → TranslationEngine + +Declares ``supported_request_types = [ANTHROPIC]`` and normalizes inbound +OpenAI / Responses via the TranslationEngine, mirroring +:class:`LatencyServiceLLMBackend`. The outer chain's ``StatsResponseProcessor`` +records executor token usage (including cache reads) from the returned response; +this backend additionally records the advisor review's usage into the planner +bucket and stamps ``ctx.selected_model``. + +Streaming is single-pass: until a session is reviewed, each executor turn is +streamed and buffered while detecting whether it has tool calls; a passed-through +/ approved turn's buffered events are replayed verbatim, so the turn is generated +once. After the review fires, the session is pure passthrough (the upstream +stream is returned directly — true streaming, full caching, zero overhead). +Once-per-session is tracked in-process by a hash of the conversation's stable +prefix (system + first user message); all of a task's turns hit the same per-run +switchyard pod. A pod restart mid-session could allow a second review (rare, +harmless). +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import sys +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol + +import httpx + +from switchyard.lib.backends.multi_llm_backend import build_native_backend +from switchyard.lib.chat_response.anthropic import AnthropicResponseStream +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.profiles.advisor_prompts import REDO_FEEDBACK_PREFIX +from switchyard.lib.roles import LLMBackend +from switchyard_rust.core import ( + ChatRequestType, + ChatResponse, + ChatResponseType, + request_type_matches, + request_with_type, +) +from switchyard_rust.translation import TranslationEngine + +if TYPE_CHECKING: + from switchyard.lib.proxy_context import ProxyContext + from switchyard.lib.stats_accumulator import StatsAccumulator + from switchyard_rust.core import ChatRequest + +log = logging.getLogger(__name__) + +_ANTHROPIC_VERSION = "2023-06-01" + + +class AdvisorCaller(Protocol): + """Consults the advisor model and returns ``(text, usage)``.""" + + async def advise(self, *, system: str, transcript: str) -> tuple[str, Any]: + ... + + +@dataclass +class _ExecTurn: + """One executor turn, normalized across the buffered streaming / completion paths.""" + + has_tool_use: bool + content: str | None + latency_ms: float + completion_body: Any | None = None + stream_events: list[Any] | None = None + + +class AdvisorLoopBackend(LLMBackend): + """Executor backend gated by a once-per-session advisor review (native Anthropic).""" + + def __init__( + self, + config: AdvisorConfig, + *, + stats_accumulator: StatsAccumulator | None = None, + executor_backend: LLMBackend | None = None, + advisor_caller: AdvisorCaller | None = None, + ) -> None: + self._config = config + self._stats = stats_accumulator if config.enable_stats else None + self._translation = TranslationEngine() + # Sessions already reviewed (once-per-session), keyed by conversation + # prefix hash. In-process; a task's turns share one switchyard pod. + self._reviewed: set[str] = set() + # The executor is delegated to verbatim so cache_control passes through. + self._executor_backend = executor_backend or build_native_backend(config.executor) + self._advisor_caller = advisor_caller or _build_advisor_caller(config) + + async def startup(self) -> None: + await self._executor_backend.startup() + + async def shutdown(self) -> None: + await self._executor_backend.shutdown() + + @property + def supported_request_types(self) -> list[ChatRequestType]: + """Executor + gate are native Anthropic Messages.""" + return [ChatRequestType.ANTHROPIC] + + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + normalized = self._translation.request_to_any_of( + request, self.supported_request_types, + ) + if not request_type_matches(normalized, ChatRequestType.ANTHROPIC): + raise TypeError( + "AdvisorLoopBackend expected an Anthropic request after translation" + ) + + body = dict(normalized.body) + messages: list[dict[str, Any]] = list(body.get("messages") or []) + session = _session_key(body.get("system"), messages) + + # After the gate has fired for this session, every turn is pure + # passthrough — return the upstream stream directly (true streaming, + # caching intact, no buffering). + if session in self._reviewed: + return await self._passthrough(ctx, normalized) + + # Not yet reviewed: run the executor and inspect its turn for tool use. + turn = await self._run_executor(ctx, normalized) + + # Tool calls → executor is working; never gate mid-work. + if turn.has_tool_use: + return await self._finish(ctx, turn) + + # No tool calls = a plan or a "done". Gate it (once per session). + self._reviewed.add(session) + verdict, plan = await self._review(messages, turn.content) + if verdict != "REDO": + return await self._finish(ctx, turn) + + # REDO: feed the optimized plan back and re-invoke so the executor keeps + # working instead of stopping. The session is now reviewed, so the redo + # turn (and everything after it) is plain passthrough. + redo_messages = [ + *messages, + {"role": "assistant", "content": turn.content or ""}, + {"role": "user", "content": REDO_FEEDBACK_PREFIX + plan}, + ] + redo_body = {**body, "messages": redo_messages} + redo_request = request_with_type("anthropic", redo_body) + return await self._passthrough(ctx, redo_request) + + # ------------------------------------------------------------------ + # Executor turn + # ------------------------------------------------------------------ + + async def _run_executor(self, ctx: ProxyContext, request: ChatRequest) -> _ExecTurn: + """Call the executor, buffering its response to detect tool use.""" + started = time.monotonic() + try: + response = await self._executor_backend.call(ctx, request) + except Exception: + # Includes ContextWindowExceeded (the chain uses it for evict-and-retry). + if self._stats is not None: + await self._stats.record_error(self._config.executor.model) + raise + + latency_ms = (time.monotonic() - started) * 1000.0 + if response.response_type == ChatResponseType.ANTHROPIC_STREAM: + events, has_tool_use, content = await _consume_anthropic_stream(response.stream) + return _ExecTurn( + has_tool_use=has_tool_use, + content=content, + latency_ms=latency_ms, + stream_events=events, + ) + body = response.to_body() + has_tool_use, content = _completion_tool_use(body) + return _ExecTurn( + has_tool_use=has_tool_use, + content=content, + latency_ms=latency_ms, + completion_body=body, + ) + + async def _passthrough(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + """Call the executor and return its response verbatim (no buffering).""" + started = time.monotonic() + try: + response = await self._executor_backend.call(ctx, request) + except Exception: + if self._stats is not None: + await self._stats.record_error(self._config.executor.model) + raise + await self._stamp(ctx, (time.monotonic() - started) * 1000.0) + return response + + async def _finish(self, ctx: ProxyContext, turn: _ExecTurn) -> ChatResponse: + """Record stats, stamp ctx, and rebuild the buffered turn as a response.""" + await self._stamp(ctx, turn.latency_ms) + if turn.stream_events is not None: + return ChatResponse.anthropic_stream( + AnthropicResponseStream(_replay_events(turn.stream_events)) + ) + return ChatResponse.anthropic_completion(turn.completion_body) + + async def _stamp(self, ctx: ProxyContext, latency_ms: float) -> None: + ctx.selected_model = self._config.executor.model + ctx.backend_call_latency_ms = latency_ms + if self._stats is not None: + await self._stats.record_success(self._config.executor.model, latency_ms) + + # ------------------------------------------------------------------ + # Advisor review + # ------------------------------------------------------------------ + + async def _review( + self, messages: list[dict[str, Any]], terminal_content: str | None, + ) -> tuple[str, str]: + """Consult the advisor once; return ``(verdict, plan)``. + + ``verdict`` is ``"APPROVE"`` or ``"REDO"``. On a fail-open advisor error + or an unparseable reply, defaults to ``APPROVE`` (do not disrupt a + possibly-correct turn). + """ + transcript = self._serialize_transcript(messages, terminal_content) + started = time.monotonic() + try: + text, usage = await self._advisor_caller.advise( + system=self._config.reviewer_system_prompt, transcript=transcript, + ) + except Exception as exc: + if not self._config.fail_open: + raise + log.warning("AdvisorLoopBackend: review failed; approving (fail-open): %s", exc) + if self._stats is not None: + await self._stats.record_planner_error(self._config.advisor.model) + _audit_review(verdict="APPROVE", error=str(exc), usage=None, + latency_ms=(time.monotonic() - started) * 1000.0) + return "APPROVE", "" + latency_ms = (time.monotonic() - started) * 1000.0 + verdict, plan = _parse_verdict(text) + # Record the advisor review's token usage so the run's own cost output + # (``routing_stats_final.json``) accounts for the advisor, not just the + # executor. Recorded into the planner bucket — the advisor review is a + # secondary-model consult, like a planner — so its Opus-4.8 cost rolls + # into ``cost_estimate.total_cost``. + if self._stats is not None: + prompt_tokens, completion_tokens = _usage_tokens(usage) + await self._stats.record_planner_usage( + model=self._config.advisor.model, + prompt_tokens=prompt_tokens or 0, + completion_tokens=completion_tokens or 0, + cached_tokens=0, + latency_ms=latency_ms, + ) + _audit_review(verdict=verdict, error=None, usage=usage, latency_ms=latency_ms) + return verdict, plan + + def _serialize_transcript( + self, messages: list[dict[str, Any]], terminal_content: str | None, + ) -> str: + """Serialize the conversation + the executor's terminal turn for review.""" + text = json.dumps(messages, default=str, ensure_ascii=False) + cap = self._config.transcript_max_chars + if len(text) > cap: + text = text[: cap - 16] + "..." + tail = terminal_content or "(no text)" + return ( + f"Conversation so far (JSON):\n\n{text}\n\n" + f"The executor's latest turn (a plan, or its claim the task is done):\n{tail}" + ) + + +# ---------------------------------------------------------------------- +# Advisor caller (native Anthropic Messages) +# ---------------------------------------------------------------------- + + +def _build_advisor_caller(config: AdvisorConfig) -> AdvisorCaller: + """Build the Anthropic Messages advisor caller for ``config.advisor``.""" + target = config.advisor + return _AnthropicAdvisorCaller( + api_key=target.endpoint.api_key, + base_url=target.endpoint.base_url, + model=target.model, + max_tokens=config.advisor_max_tokens, + temperature=config.advisor_temperature, + timeout=target.endpoint.timeout_secs, + ) + + +class _AnthropicAdvisorCaller: + """Reviews via an Anthropic-Messages advisor (``/v1/messages``, Bearer auth).""" + + def __init__( + self, *, api_key: str | None, base_url: str | None, model: str, + max_tokens: int, temperature: float | None, timeout: float | None, + ) -> None: + self._url = _messages_url(base_url) + self._api_key = api_key + self._model = model + self._max_tokens = max_tokens + self._temperature = temperature + self._timeout = timeout + + async def advise(self, *, system: str, transcript: str) -> tuple[str, Any]: + body: dict[str, Any] = { + "model": self._model, + "system": system, + "messages": [{"role": "user", "content": transcript}], + "max_tokens": self._max_tokens, + } + if self._temperature is not None: + body["temperature"] = self._temperature + headers = { + "Authorization": f"Bearer {self._api_key}", + "anthropic-version": _ANTHROPIC_VERSION, + "Content-Type": "application/json", + } + async with httpx.AsyncClient(timeout=self._timeout) as client: + response = await client.post(self._url, json=body, headers=headers) + response.raise_for_status() + data = response.json() + return _anthropic_text(data), data.get("usage") + + +# ---------------------------------------------------------------------- +# Module-level helpers +# ---------------------------------------------------------------------- + + +async def _consume_anthropic_stream(stream: Any) -> tuple[list[Any], bool, str | None]: + """Buffer an Anthropic stream; return (events, has_tool_use, assistant_text).""" + events: list[Any] = [] + has_tool_use = False + text_parts: list[str] = [] + async for event in stream: + events.append(event) + etype = _ev(event, "type") + if etype == "content_block_start": + if _ev(_ev(event, "content_block"), "type") == "tool_use": + has_tool_use = True + elif etype == "content_block_delta": + delta = _ev(event, "delta") + if _ev(delta, "type") == "text_delta": + piece = _ev(delta, "text") + if isinstance(piece, str): + text_parts.append(piece) + elif etype == "message_delta": + if _ev(_ev(event, "delta"), "stop_reason") == "tool_use": + has_tool_use = True + return events, has_tool_use, ("".join(text_parts) or None) + + +async def _replay_events(events: list[Any]) -> Any: + """Replay buffered stream events verbatim as a fresh async stream.""" + for event in events: + yield event + + +def _completion_tool_use(body: Any) -> tuple[bool, str | None]: + """Read (has_tool_use, assistant_text) from an Anthropic completion body.""" + if not isinstance(body, dict): + return False, None + content = body.get("content") or [] + has_tool_use = body.get("stop_reason") == "tool_use" or any( + isinstance(b, dict) and b.get("type") == "tool_use" for b in content + ) + return has_tool_use, (_blocks_text(content) or None) + + +def _ev(event: Any, key: str) -> Any: + """Read a field from a stream event (dict from Rust, or an SDK object).""" + if event is None: + return None + if isinstance(event, dict): + return event.get(key) + return getattr(event, key, None) + + +def _session_key(system: Any, messages: list[dict[str, Any]]) -> str: + """Stable per-session key: hash of system prompt + first user message.""" + parts: list[str] = ["S:" + _blocks_text(system)] + for m in messages: + if m.get("role") == "user": + parts.append("U:" + _blocks_text(m.get("content"))) + break + return hashlib.sha256("\n".join(parts).encode("utf-8", "ignore")).hexdigest() + + +def _blocks_text(content: Any) -> str: + """Flatten Anthropic content (string, or a list of blocks) to text.""" + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join( + b.get("text", "") for b in content + if isinstance(b, dict) and isinstance(b.get("text"), str) + ) + return "" + + +def _parse_verdict(text: str) -> tuple[str, str]: + """Parse the reviewer reply into (verdict, plan). Unclear → APPROVE.""" + stripped = (text or "").strip() + head = stripped[:16].upper() + if head.startswith("APPROVE"): + return "APPROVE", "" + if head.startswith("REDO"): + plan = stripped[4:].lstrip(" :\n-").strip() + return "REDO", plan or stripped + return "APPROVE", "" + + +def _messages_url(base_url: str | None) -> str: + """Resolve the Anthropic Messages URL from a target base URL.""" + base = (base_url or "https://api.anthropic.com").rstrip("/") + if base.endswith("/v1/messages"): + return base + if base.endswith("/v1"): + return f"{base}/messages" + return f"{base}/v1/messages" + + +def _anthropic_text(data: dict[str, Any]) -> str: + """Join the ``text`` content blocks of an Anthropic Messages response.""" + content = data.get("content") or [] + return "".join( + b.get("text", "") for b in content + if isinstance(b, dict) and b.get("type") == "text" + ).strip() + + +def _usage_tokens(usage: Any) -> tuple[int | None, int | None]: + """Read (input, output) token counts from Anthropic- or OpenAI-shaped usage.""" + if usage is None: + return None, None + + def get(*names: str) -> int | None: + for name in names: + value = usage.get(name) if isinstance(usage, dict) else getattr(usage, name, None) + if value is not None: + return int(value) + return None + + return get("input_tokens", "prompt_tokens"), get("output_tokens", "completion_tokens") + + +def _audit_review(*, verdict: str, error: str | None, usage: Any, latency_ms: float) -> None: + """Emit a one-line ``advisor_review=...`` audit record to stderr.""" + payload: dict[str, Any] = { + "advisor_review": True, + "verdict": verdict, + "error": error, + "latency_ms": round(latency_ms, 1), + } + prompt_tokens, completion_tokens = _usage_tokens(usage) + if prompt_tokens is not None: + payload["prompt_tokens"] = prompt_tokens + if completion_tokens is not None: + payload["completion_tokens"] = completion_tokens + sys.stderr.write(f"advisor_review={json.dumps(payload, sort_keys=True)}\n") + sys.stderr.flush() + + +__all__ = ["AdvisorCaller", "AdvisorLoopBackend"] diff --git a/switchyard/lib/backends/advisor_tool_call_backend.py b/switchyard/lib/backends/advisor_tool_call_backend.py new file mode 100644 index 00000000..26739da1 --- /dev/null +++ b/switchyard/lib/backends/advisor_tool_call_backend.py @@ -0,0 +1,516 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``LLMBackend`` that re-creates Anthropic's advisor tool proxy-side (native Anthropic). + +Anthropic's native ``advisor_20260301`` server tool runs the advisor +sub-inference server-side, so it is unavailable on gateways that only proxy +model traffic (e.g. NVIDIA Inference Hub). This backend reproduces the +*executor-triggered* behavior in the proxy: + +1. Offer the executor a real, parameterless ``advisor`` tool (empty + ``input_schema`` — like the native tool, the executor signals timing and the + proxy supplies context). +2. Call the executor. If its turn contains ``advisor`` ``tool_use`` blocks, + intercept them **before they reach the client**, consult the advisor model + on the transcript (including the text the executor produced so far in the + turn, mirroring the native tool's context), append the advice as a + ``tool_result``, and call the executor again. +3. Loop until the executor returns an advisor-free turn (a real tool call for + the client, or a final answer), then return that turn. + +Once ``max_uses`` consultations have happened in a request, further advisor +calls receive a ``max_uses exceeded`` tool result without a consult — the +executor sees the error and continues, mirroring the native tool's +``max_uses_exceeded`` error result. + +Both tiers are **native Anthropic** (``/v1/messages``, Bearer auth). The +executor is called by delegating to an :class:`AnthropicNativeBackend`, so the +client's ``cache_control`` breakpoints reach the upstream unchanged and prompt +caching is honored. Steering is injected cache-stably: the executor steering is +prepended to the system prompt and the advisor length line is appended to the +**first** user message (both constant across a session's turns; the native +doc suggests the latest user message, but re-injecting there would shift the +cached prefix on every turn because the client never sees the injection). + +A turn that mixes advisor and client tool calls is regenerated: the appended +assistant turn keeps only the advisor ``tool_use`` blocks (plus thinking/text), +and the sibling client calls are re-issued advice-informed on the next +iteration. The native API instead pauses with the client calls pending; a +proxy cannot, because it can neither execute the client's tools nor hand the +client a turn containing a tool it was never offered. + +Chain integration:: + + [RequestProcessor*] → AdvisorToolCallBackend → [ResponseProcessor*] → TranslationEngine + +Declares ``supported_request_types = [ANTHROPIC]`` and normalizes inbound +OpenAI / Responses via the TranslationEngine, mirroring +:class:`AdvisorLoopBackend`. The outer chain's ``StatsResponseProcessor`` +records the terminal turn's token usage; this backend additionally records the +advisor consults **and the intermediate executor turns** (which the client +never sees) into the planner bucket, so the run's cost output prices the full +loop, and stamps ``ctx.selected_model``. + +Streaming is single-pass: each executor turn is streamed and buffered while its +content blocks are reassembled to detect advisor calls; the terminal turn's +buffered events are replayed verbatim, so the turn the client pays for is +generated exactly once. Advice is not persisted across client turns (the +client cannot carry advisor exchanges back); its effect lives in the terminal +turn the client keeps. +""" + +from __future__ import annotations + +import json +import logging +import sys +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from switchyard.lib.backends.advisor_loop_backend import ( + AdvisorCaller, + _blocks_text, + _build_advisor_caller, + _ev, + _replay_events, + _usage_tokens, +) +from switchyard.lib.backends.multi_llm_backend import build_native_backend +from switchyard.lib.chat_response.anthropic import AnthropicResponseStream +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.roles import LLMBackend +from switchyard_rust.core import ( + ChatRequestType, + ChatResponse, + ChatResponseType, + request_type_matches, + request_with_type, +) +from switchyard_rust.translation import TranslationEngine + +if TYPE_CHECKING: + from switchyard.lib.proxy_context import ProxyContext + from switchyard.lib.stats_accumulator import StatsAccumulator + from switchyard_rust.core import ChatRequest + +log = logging.getLogger(__name__) + +#: Tool result handed to the executor for advisor calls past the ``max_uses`` +#: cap (mirrors the native tool's ``max_uses_exceeded`` error result). +_MAX_USES_RESULT = "[advisor unavailable: max_uses exceeded]" + +#: Per-tool cap on the description text included in the advisor's transcript. +_TOOL_SUMMARY_DESC_CHARS = 200 + + +@dataclass +class _ToolCallTurn: + """One executor turn, normalized across the streaming / completion paths. + + ``blocks`` are the turn's reassembled Anthropic content blocks (dicts). + Exactly one of ``completion_body`` / ``stream_events`` carries the payload + for verbatim replay if the turn is terminal. + """ + + blocks: list[dict[str, Any]] + latency_ms: float + input_tokens: int + output_tokens: int + cached_tokens: int + completion_body: Any | None = None + stream_events: list[Any] | None = None + + @property + def text(self) -> str: + """The turn's assistant text (joined ``text`` blocks).""" + return _blocks_text(self.blocks) + + +class AdvisorToolCallBackend(LLMBackend): + """Executor backend offering a proxy-intercepted ``advisor`` tool (native Anthropic).""" + + #: Absolute backstop on executor calls per request. ``max_uses`` already + #: bounds consults; this bounds an executor that keeps calling the advisor + #: through ``max_uses exceeded`` results. + _HARD_ITERATION_CAP = 8 + + def __init__( + self, + config: AdvisorConfig, + *, + stats_accumulator: StatsAccumulator | None = None, + executor_backend: LLMBackend | None = None, + advisor_caller: AdvisorCaller | None = None, + ) -> None: + self._config = config + self._stats = stats_accumulator if config.enable_stats else None + self._translation = TranslationEngine() + # The executor is delegated to verbatim so cache_control passes through. + self._executor_backend = executor_backend or build_native_backend(config.executor) + self._advisor_caller = advisor_caller or _build_advisor_caller(config) + + async def startup(self) -> None: + await self._executor_backend.startup() + + async def shutdown(self) -> None: + await self._executor_backend.shutdown() + + @property + def supported_request_types(self) -> list[ChatRequestType]: + """Executor + advisor are native Anthropic Messages.""" + return [ChatRequestType.ANTHROPIC] + + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + normalized = self._translation.request_to_any_of( + request, self.supported_request_types, + ) + if not request_type_matches(normalized, ChatRequestType.ANTHROPIC): + raise TypeError( + "AdvisorToolCallBackend expected an Anthropic request after translation" + ) + + body = dict(normalized.body) + messages: list[dict[str, Any]] = list(body.get("messages") or []) + base_tools: list[dict[str, Any]] = list(body.get("tools") or []) + if self._config.inject_steering: + body["system"] = _prepend_system(body.get("system"), self._config.executor_steering) + messages = _with_length_line(messages, self._config.advisor_length_line) + tools = [*base_tools, self._advisor_tool_def()] + + advisor_uses = 0 + turn: _ToolCallTurn | None = None + for _ in range(self._HARD_ITERATION_CAP): + turn_body = {**body, "messages": messages, "tools": tools} + turn = await self._run_executor(ctx, request_with_type("anthropic", turn_body)) + + advisor_calls = [ + b for b in turn.blocks + if b.get("type") == "tool_use" and b.get("name") == self._config.advisor_tool_name + ] + if not advisor_calls: + # Real tool call(s) for the client, or a final answer. + return await self._finish(ctx, turn) + + # This turn stays proxy-internal — price it into the planner bucket + # (under the executor model) so the run's cost output sees it. + await self._record_internal_turn(turn) + + if advisor_uses < self._config.max_uses: + advisor_uses += 1 + advice = await self._consult_advisor(messages, turn.text, base_tools) + else: + advice = _MAX_USES_RESULT + + # Rebuild the assistant turn keeping thinking/text and ONLY the + # advisor tool_use blocks; sibling client calls are re-issued + # (advice-informed) on the next iteration. Thinking blocks must + # round-trip verbatim for upstreams that verify signatures. + messages = [ + *messages, + {"role": "assistant", "content": _advisor_only_content(turn.blocks, advisor_calls)}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": call.get("id"), "content": advice} + for call in advisor_calls + ]}, + ] + + log.warning( + "AdvisorToolCallBackend: hit hard iteration cap (%d) without a terminal " + "executor turn; returning the last result.", + self._HARD_ITERATION_CAP, + ) + if turn is None: # unreachable: _HARD_ITERATION_CAP >= 1 + raise RuntimeError("AdvisorToolCallBackend produced no executor turn") + return await self._finish(ctx, turn) + + # ------------------------------------------------------------------ + # Executor turn + # ------------------------------------------------------------------ + + async def _run_executor(self, ctx: ProxyContext, request: ChatRequest) -> _ToolCallTurn: + """Call the executor, buffering its response and reassembling content blocks.""" + started = time.monotonic() + try: + response = await self._executor_backend.call(ctx, request) + except Exception: + # Includes ContextWindowExceeded (the chain uses it for evict-and-retry). + if self._stats is not None: + await self._stats.record_error(self._config.executor.model) + raise + + latency_ms = (time.monotonic() - started) * 1000.0 + if response.response_type == ChatResponseType.ANTHROPIC_STREAM: + events, blocks, usage = await _consume_stream(response.stream) + return _ToolCallTurn( + blocks=blocks, latency_ms=latency_ms, stream_events=events, **usage, + ) + completion: Any = response.to_body() + blocks = [b for b in (completion.get("content") or []) if isinstance(b, dict)] + prompt_tokens, completion_tokens = _usage_tokens(completion.get("usage")) + cached = (completion.get("usage") or {}).get("cache_read_input_tokens") or 0 + return _ToolCallTurn( + blocks=blocks, + latency_ms=latency_ms, + input_tokens=prompt_tokens or 0, + output_tokens=completion_tokens or 0, + cached_tokens=int(cached), + completion_body=completion, + ) + + async def _finish(self, ctx: ProxyContext, turn: _ToolCallTurn) -> ChatResponse: + """Record stats, stamp ctx, and rebuild the terminal turn as a response.""" + ctx.selected_model = self._config.executor.model + ctx.backend_call_latency_ms = turn.latency_ms + if self._stats is not None: + await self._stats.record_success(self._config.executor.model, turn.latency_ms) + if turn.stream_events is not None: + return ChatResponse.anthropic_stream( + AnthropicResponseStream(_replay_events(turn.stream_events)) + ) + return ChatResponse.anthropic_completion(turn.completion_body) + + async def _record_internal_turn(self, turn: _ToolCallTurn) -> None: + """Price a proxy-internal executor turn into the planner bucket.""" + if self._stats is None: + return + await self._stats.record_planner_usage( + model=self._config.executor.model, + prompt_tokens=turn.input_tokens, + completion_tokens=turn.output_tokens, + cached_tokens=turn.cached_tokens, + latency_ms=turn.latency_ms, + ) + + # ------------------------------------------------------------------ + # Advisor consultation + # ------------------------------------------------------------------ + + async def _consult_advisor( + self, + messages: list[dict[str, Any]], + current_turn_text: str, + tools: list[dict[str, Any]], + ) -> str: + """Consult the advisor on the transcript and return its guidance. + + On failure with ``fail_open`` set, returns a short "unavailable" marker + so the executor can proceed; the failed call still counts toward + ``max_uses`` at the call site, bounding retries against a down advisor. + """ + transcript = self._serialize_transcript(messages, current_turn_text, tools) + started = time.monotonic() + try: + advice, usage = await self._advisor_caller.advise( + system=self._config.advisor_system_prompt, transcript=transcript, + ) + except Exception as exc: + if not self._config.fail_open: + raise + log.warning( + "AdvisorToolCallBackend: advisor call failed; continuing unadvised: %s", exc, + ) + if self._stats is not None: + await self._stats.record_planner_error(self._config.advisor.model) + _audit_advisor(error=str(exc), usage=None, + latency_ms=(time.monotonic() - started) * 1000.0) + return f"[advisor unavailable: {type(exc).__name__}]" + + latency_ms = (time.monotonic() - started) * 1000.0 + if self._stats is not None: + prompt_tokens, completion_tokens = _usage_tokens(usage) + await self._stats.record_planner_usage( + model=self._config.advisor.model, + prompt_tokens=prompt_tokens or 0, + completion_tokens=completion_tokens or 0, + cached_tokens=0, + latency_ms=latency_ms, + ) + _audit_advisor(error=None, usage=usage, latency_ms=latency_ms) + return advice + + def _serialize_transcript( + self, + messages: list[dict[str, Any]], + current_turn_text: str, + tools: list[dict[str, Any]], + ) -> str: + """Serialize the conversation for the advisor, newest turns kept first. + + Mirrors the native tool's context: the executor's tools (as a compact + name — description summary; full schemas would swamp the char budget), + the conversation, and the text the executor has produced so far in the + turn that called the advisor. When over ``transcript_max_chars``, the + **oldest** messages are dropped — the newest turns are the ones the + consult is about. + """ + sections: list[str] = [] + if tools: + summary = "\n".join( + f"- {t.get('name', '?')}: {str(t.get('description', ''))[:_TOOL_SUMMARY_DESC_CHARS]}" + for t in tools + ) + sections.append(f"Tools available to the executor:\n{summary}") + + parts = [json.dumps(m, default=str, ensure_ascii=False) for m in messages] + cap = self._config.transcript_max_chars + kept: list[str] = [] + total = 0 + for part in reversed(parts): + if kept and total + len(part) > cap: + break + if not kept and len(part) > cap: + part = "..." + part[-(cap - 16):] + kept.append(part) + total += len(part) + kept.reverse() + omitted = len(parts) - len(kept) + header = "Conversation so far (JSON, oldest first" + if omitted: + header += f"; {omitted} earlier messages omitted" + sections.append(header + "):\n[" + ",\n".join(kept) + "]") + + sections.append( + "The executor's turn so far (it is consulting you now):\n" + + (current_turn_text or "(no text)") + ) + return "\n\n".join(sections) + + # ------------------------------------------------------------------ + # Request shaping + # ------------------------------------------------------------------ + + def _advisor_tool_def(self) -> dict[str, Any]: + """The synthetic, parameterless ``advisor`` tool (Anthropic shape).""" + return { + "name": self._config.advisor_tool_name, + "description": self._config.advisor_tool_description, + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + } + + +# ---------------------------------------------------------------------- +# Module-level helpers +# ---------------------------------------------------------------------- + + +async def _consume_stream( + stream: Any, +) -> tuple[list[Any], list[dict[str, Any]], dict[str, int]]: + """Buffer an Anthropic stream; reassemble its content blocks and usage. + + Events are the dicts the native backend's SSE parser yields. Returns + ``(events, blocks, usage)`` where ``usage`` carries ``input_tokens`` / + ``output_tokens`` / ``cached_tokens`` keyword-ready for ``_ToolCallTurn``. + """ + events: list[Any] = [] + blocks: dict[int, dict[str, Any]] = {} + json_parts: dict[int, list[str]] = {} + usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0} + async for event in stream: + events.append(event) + etype = _ev(event, "type") + if etype == "message_start": + start_usage = _ev(_ev(event, "message"), "usage") or {} + usage["input_tokens"] = int(start_usage.get("input_tokens") or 0) + usage["cached_tokens"] = int(start_usage.get("cache_read_input_tokens") or 0) + elif etype == "content_block_start": + index = int(_ev(event, "index") or 0) + block = _ev(event, "content_block") + blocks[index] = dict(block) if isinstance(block, dict) else {} + elif etype == "content_block_delta": + index = int(_ev(event, "index") or 0) + block = blocks.setdefault(index, {}) + delta = _ev(event, "delta") + dtype = _ev(delta, "type") + if dtype == "text_delta": + block["text"] = str(block.get("text") or "") + str(_ev(delta, "text") or "") + elif dtype == "input_json_delta": + json_parts.setdefault(index, []).append(str(_ev(delta, "partial_json") or "")) + elif dtype == "thinking_delta": + block["thinking"] = ( + str(block.get("thinking") or "") + str(_ev(delta, "thinking") or "") + ) + elif dtype == "signature_delta": + block["signature"] = _ev(delta, "signature") + elif etype == "message_delta": + delta_usage = _ev(event, "usage") or {} + usage["output_tokens"] = int(delta_usage.get("output_tokens") or 0) + for index, parts in json_parts.items(): + joined = "".join(parts).strip() + try: + blocks[index]["input"] = json.loads(joined) if joined else {} + except json.JSONDecodeError: + blocks[index]["input"] = {} + ordered = [blocks[i] for i in sorted(blocks)] + return events, ordered, usage + + +def _advisor_only_content( + blocks: list[dict[str, Any]], advisor_calls: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """The assistant turn's blocks with sibling (non-advisor) tool calls dropped.""" + kept_ids = {call.get("id") for call in advisor_calls} + return [ + b for b in blocks + if b.get("type") != "tool_use" or b.get("id") in kept_ids + ] + + +def _prepend_system(system: Any, prefix: str) -> Any: + """Prepend steering to an Anthropic ``system`` field (string or block list).""" + if system is None or system == "": + return prefix + if isinstance(system, str): + return f"{prefix}\n\n{system}" + if isinstance(system, list): + return [{"type": "text", "text": prefix}, *system] + return f"{prefix}\n\n{system}" + + +def _with_length_line( + messages: list[dict[str, Any]], line: str, +) -> list[dict[str, Any]]: + """Append the advisor length line to the **first** user message. + + The doc suggests the latest user message, but the client never sees this + injection, so re-injecting into each turn's newest message would shift the + upstream cache prefix every turn. The first user message is constant across + a session, keeping the prefix stable; the advisor still reads the line via + the forwarded transcript. + """ + msgs = [dict(m) for m in messages] + for msg in msgs: + if msg.get("role") != "user": + continue + content = msg.get("content") + if isinstance(content, list): + msg["content"] = [*content, {"type": "text", "text": line}] + elif isinstance(content, str) or content is None: + msg["content"] = f"{content or ''}\n\n{line}".lstrip() + break + return msgs + + +def _audit_advisor(*, error: str | None, usage: Any, latency_ms: float) -> None: + """Emit a one-line ``advisor_call=...`` audit record to stderr.""" + payload: dict[str, Any] = { + "advisor_call": True, + "error": error, + "latency_ms": round(latency_ms, 1), + } + prompt_tokens, completion_tokens = _usage_tokens(usage) + if prompt_tokens is not None: + payload["prompt_tokens"] = prompt_tokens + if completion_tokens is not None: + payload["completion_tokens"] = completion_tokens + sys.stderr.write(f"advisor_call={json.dumps(payload, sort_keys=True)}\n") + sys.stderr.flush() + + +__all__ = ["AdvisorToolCallBackend"] diff --git a/switchyard/lib/cost_estimator.py b/switchyard/lib/cost_estimator.py index 7afd8c86..9b982546 100644 --- a/switchyard/lib/cost_estimator.py +++ b/switchyard/lib/cost_estimator.py @@ -155,6 +155,9 @@ class ModelPriceData: ), # --- Anthropic Claude on AWS Bedrock (via NVIDIA Inference Hub) --- # 5-minute cache write = 1.25x input; cache read = 0.1x input. + "aws/anthropic/bedrock-claude-opus-4-8": ModelPriceData( + input=5.00, output=25.00, cached=0.50, cache_write=6.25, + ), "aws/anthropic/bedrock-claude-opus-4-7": ModelPriceData( input=5.00, output=25.00, cached=0.50, cache_write=6.25, ), @@ -177,6 +180,9 @@ class ModelPriceData: "azure/anthropic/claude-opus-4-7": ModelPriceData( input=5.00, output=25.00, cached=0.50, cache_write=6.25, ), + "azure/anthropic/claude-opus-4-8": ModelPriceData( + input=5.00, output=25.00, cached=0.50, cache_write=6.25, + ), "aws/anthropic/bedrock-claude-sonnet-4-6": ModelPriceData( input=3.00, output=15.00, cached=0.30, cache_write=3.75, ), @@ -187,6 +193,9 @@ class ModelPriceData: input=1.00, output=5.00, cached=0.10, cache_write=1.25, ), # --- Anthropic direct API aliases (no AWS prefix) --- + "claude-opus-4-8": ModelPriceData( + input=5.00, output=25.00, cached=0.50, cache_write=6.25, + ), "claude-opus-4-7": ModelPriceData( input=5.00, output=25.00, cached=0.50, cache_write=6.25, ), diff --git a/switchyard/lib/profiles/__init__.py b/switchyard/lib/profiles/__init__.py index e71f5dca..7ea42067 100644 --- a/switchyard/lib/profiles/__init__.py +++ b/switchyard/lib/profiles/__init__.py @@ -3,6 +3,9 @@ """Python profile abstractions matching the components-v2 design.""" +from switchyard.lib.profiles.advisor import AdvisorProfileConfig +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.profiles.advisor_presets import AdvisorPresets from switchyard.lib.profiles.deterministic_routing_config import ( DeterministicRoutingConfig, ) @@ -56,6 +59,9 @@ from switchyard.lib.profiles.translate_profile_config import TranslateProfileConfig __all__ = [ + "AdvisorConfig", + "AdvisorPresets", + "AdvisorProfileConfig", "StageRouterProfileConfig", "StageRouterConfig", "ClassifierConfig", diff --git a/switchyard/lib/profiles/advisor.py b/switchyard/lib/profiles/advisor.py new file mode 100644 index 00000000..dacc13c3 --- /dev/null +++ b/switchyard/lib/profiles/advisor.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Profile-owned advisor construction. + +Chain shape:: + + [ReasoningEffortNormalizer] → AdvisorToolCallBackend | AdvisorLoopBackend + +``config.strategy`` selects the backend: ``"tool_call"`` (default) offers the +executor a proxy-intercepted ``advisor`` tool +(:class:`~switchyard.lib.backends.advisor_tool_call_backend.AdvisorToolCallBackend`); +``"review_gate"`` consults the advisor once per session at the executor's first +no-tool-call turn +(:class:`~switchyard.lib.backends.advisor_loop_backend.AdvisorLoopBackend`). +Both delegate the executor call to a native Anthropic backend (caching intact), +stamp ``ctx.selected_model``, and record the advisor's usage themselves; they +cannot be wrapped by ``StatsLlmBackend``, so the runtime attaches the shared +accumulator through the ``_stats`` compatibility hook — mirroring +``LatencyServiceLLMBackend``. Serving-level stats processors arrive via +``with_runtime_components``, like every other profile. +""" + +from __future__ import annotations + +from typing import Any, Self + +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.profiles.chain import ComponentChainProfile +from switchyard.lib.profiles.table import profile_config + + +@profile_config("advisor") +class AdvisorProfileConfig: + """Profile config wrapper for executor + stronger-advisor chains.""" + + config: AdvisorConfig + + @classmethod + def from_config(cls, config: AdvisorConfig) -> Self: + """Create a profile config from the validated parsing model.""" + return cls(config=config) + + def build(self) -> ComponentChainProfile: + """Build the advisor profile runtime for the configured strategy.""" + from switchyard.lib.processors.reasoning_effort_normalizer import ( + ReasoningEffortNormalizer, + ) + + config = self.config + backend: Any + if config.strategy == "review_gate": + from switchyard.lib.backends.advisor_loop_backend import AdvisorLoopBackend + + backend = AdvisorLoopBackend(config) + else: + from switchyard.lib.backends.advisor_tool_call_backend import ( + AdvisorToolCallBackend, + ) + + backend = AdvisorToolCallBackend(config) + + # Normalize unsupported ``reasoning_effort`` values (Claude Code's + # ``/effort xhigh`` in particular) before they reach the executor. + # Same placement as the plan-execute profile. + return ComponentChainProfile( + request_processors=[ReasoningEffortNormalizer()], + backend=backend, + ) + + +__all__ = ["AdvisorProfileConfig"] diff --git a/switchyard/lib/profiles/advisor_config.py b/switchyard/lib/profiles/advisor_config.py new file mode 100644 index 00000000..017c5bb7 --- /dev/null +++ b/switchyard/lib/profiles/advisor_config.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Config model for the advisor profile. + +An advisor chain pairs an **executor** (the base model under test) with a +stronger **advisor**. ``strategy`` selects how the advisor participates: + +* ``"tool_call"`` (default, the shipping strategy) — the executor is offered a + real, parameterless ``advisor`` tool. When it calls, the backend consults the + advisor model on the full transcript and feeds the guidance back as the tool + result, looping until the executor stops asking. The proxy-side re-creation + of Anthropic's ``advisor_20260301`` server tool for gateways that cannot run + it server-side. See ``switchyard/lib/backends/advisor_tool_call_backend.py``. + +* ``"review_gate"`` — no advisor tool is injected. The executor works the task + with its own tools; when it first produces a no-tool-call turn — a plan, or + a claim of "done" — the backend consults the advisor once to APPROVE or send + it back (REDO) with an optimized plan. See + ``switchyard/lib/backends/advisor_loop_backend.py``. + +Both tiers are ordinary targets, served native Anthropic-Messages (Opus 4.7 +executor + Opus 4.8 advisor on NVIDIA Inference Hub) — no OpenAI translation, so +the client's prompt caching survives. +""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator + +from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target +from switchyard.lib.profiles.advisor_prompts import ( + ADVISOR_LENGTH_LINE, + ADVISOR_SYSTEM_PROMPT, + ADVISOR_TOOL_DESCRIPTION, + EXECUTOR_STEERING, + REVIEWER_SYSTEM_PROMPT, +) + + +class AdvisorConfig(BaseModel): + """Configuration for the advisor profile. + + Attributes: + executor: The base model under test. Runs the user-visible chat + completion with the client's own tools. + advisor: The stronger advisor model. Must be at least as capable. + strategy: How the advisor participates. ``"tool_call"`` offers the + executor a real ``advisor`` tool it calls mid-generation; + ``"review_gate"`` consults the advisor once per session at the + executor's first no-tool-call turn. + + advisor_tool_name: (tool_call) Tool name the executor calls to consult + the advisor. Must match what the steering prompt references + (``"advisor"``). + max_uses: (tool_call) Per-request cap on advisor consultations. Over-cap + calls receive a ``max_uses exceeded`` tool result without a consult + (mirroring the native tool's ``max_uses_exceeded`` error result) + and the executor continues. Failed (fail-open) consultations count + toward this cap, which bounds retry storms when the advisor is + unavailable. + inject_steering: (tool_call) Prepend ``executor_steering`` to the + executor's system prompt and append ``advisor_length_line`` to the + latest user turn. The doc notes executors under-call the advisor + without this; keep it a toggle so a benchmark can ablate steering. + executor_steering: (tool_call) Verbatim doc steering for the executor. + advisor_length_line: (tool_call) Verbatim doc length hint, injected into + the latest user turn (the advisor sees it via the transcript). + advisor_system_prompt: (tool_call) System prompt for the advisor's own + LLM call (authored — the doc publishes none). + advisor_tool_description: (tool_call) Description for the synthetic + ``advisor`` tool. + + reviewer_system_prompt: (review_gate) System prompt for the advisor's + review call; instructs the APPROVE / REDO contract. + + advisor_max_tokens: Cap on the advisor's output per call (the doc's + recommended starting point is 2048). + advisor_temperature: Sampling temperature for the advisor call. ``None`` + (default) omits the field — required for Anthropic targets that + reject ``temperature``. + transcript_max_chars: Cap on the serialized transcript handed to the + advisor, so a long agent conversation can't blow its context. + fail_open: When ``True`` (default), an advisor-call failure degrades + gracefully — the executor proceeds unadvised (tool_call) or the + turn passes through as APPROVE (review_gate). When ``False``, the + failure surfaces as 5xx. + enable_stats: Record executor success/error + latency into the shared + accumulator and stamp ``ctx.selected_model``. + preset: Optional name of the preset that produced this config. + """ + + model_config = ConfigDict(frozen=True, arbitrary_types_allowed=True) + + executor: LlmTarget + advisor: LlmTarget + strategy: Literal["tool_call", "review_gate"] = "tool_call" + + # tool_call strategy + advisor_tool_name: str = "advisor" + max_uses: int = Field(default=2, ge=1) + inject_steering: bool = True + executor_steering: str = EXECUTOR_STEERING + advisor_length_line: str = ADVISOR_LENGTH_LINE + advisor_system_prompt: str = ADVISOR_SYSTEM_PROMPT + advisor_tool_description: str = ADVISOR_TOOL_DESCRIPTION + + # review_gate strategy + reviewer_system_prompt: str = REVIEWER_SYSTEM_PROMPT + + # shared + advisor_max_tokens: int = Field(default=2048, ge=1) + advisor_temperature: float | None = None + transcript_max_chars: int = Field(default=24_000, ge=256) + fail_open: bool = True + enable_stats: bool = True + preset: str | None = None + + @field_validator("executor", "advisor", mode="before") + @classmethod + def _coerce_target(cls, value: object, info: ValidationInfo) -> LlmTarget: + return coerce_llm_target(value, default_id=info.field_name or "target") + + @field_validator("executor", "advisor") + @classmethod + def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget: + if not tier.model: + raise ValueError("target.model must be a non-empty string") + return tier + + +__all__ = ["AdvisorConfig"] diff --git a/switchyard/lib/profiles/advisor_presets.py b/switchyard/lib/profiles/advisor_presets.py new file mode 100644 index 00000000..aa92836d --- /dev/null +++ b/switchyard/lib/profiles/advisor_presets.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Named :class:`AdvisorConfig` presets keyed by shipping bundle. + +The shipping default :meth:`AdvisorPresets.opus47_exec_opus48_advisor` pairs an +Opus 4.7 executor with an Opus 4.8 advisor, both served native Anthropic from +NVIDIA Inference Hub. This is the proxy-side re-creation of Anthropic's +server-side ``advisor_20260301`` beta for a gateway that cannot run it. +``strategy`` selects how the advisor participates: ``"tool_call"`` (default) +offers the executor a real, parameterless ``advisor`` tool it calls +mid-generation; ``"review_gate"`` consults the advisor once per session at the +executor's first no-tool-call turn to APPROVE or send it back (REDO). + +Example:: + + from switchyard import AdvisorPresets, AdvisorProfileConfig, ProfileSwitchyard + + config = AdvisorPresets.opus47_exec_opus48_advisor(api_key=nvidia_api_key) + switchyard = ProfileSwitchyard( + AdvisorProfileConfig.from_config(config).build().with_runtime_components() + ) +""" + +from __future__ import annotations + +from typing import Literal + +from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget +from switchyard.lib.profiles.advisor_config import AdvisorConfig + +# All shipping presets route through NVIDIA Inference Hub's Anthropic Messages +# endpoint by default; callers override with ``base_url=`` for a different gateway. +_INFERENCE_HUB_BASE_URL = "https://inference-api.nvidia.com/v1" + +# Inference Hub model ids. Both tiers (Opus 4.7 executor, Opus 4.8 advisor) are +# served native Anthropic-Messages at ``/v1/messages`` — no OpenAI translation, +# so prompt caching survives. Both are overridable via the preset's +# ``executor_model`` / ``advisor_model``. +_MODEL_OPUS_4_7_EXECUTOR = "aws/anthropic/bedrock-claude-opus-4-7" +_MODEL_OPUS_4_8_ADVISOR = "aws/anthropic/bedrock-claude-opus-4-8" + + +class AdvisorPresets: + """Factory of pre-built :class:`AdvisorConfig` bundles.""" + + @staticmethod + def opus47_exec_opus48_advisor( + *, + api_key: str, + base_url: str = _INFERENCE_HUB_BASE_URL, + timeout_secs: float | None = 600.0, + executor_model: str = _MODEL_OPUS_4_7_EXECUTOR, + advisor_model: str = _MODEL_OPUS_4_8_ADVISOR, + strategy: Literal["tool_call", "review_gate"] = "tool_call", + ) -> AdvisorConfig: + """Opus 4.7 executor + Opus 4.8 advisor on NVIDIA Inference Hub. + + Args: + api_key: Inference Hub API key, used for both tiers (one tenancy). + base_url: OpenAI-compatible gateway base URL. + timeout_secs: Per-call timeout for both tiers. Generous by default + because the advisor consult adds an extra round-trip inside a + single client request. + executor_model: Override the executor model id if your tenancy + serves Opus 4.7 under a different string. + advisor_model: Override the advisor model id likewise. + strategy: Advisor strategy — ``"tool_call"`` (default) or + ``"review_gate"``. + """ + return AdvisorConfig( + executor=LlmTarget( + # Native Anthropic Messages (``/v1/messages``): the request passes + # through verbatim so the client's cache_control breakpoints reach + # the upstream and prompt caching is honored. Inference Hub wants + # Bearer auth (not Anthropic's x-api-key), so suppress x-api-key + # (api_key="") and carry the key in an Authorization header. + id="executor", + model=executor_model, + format=BackendFormat.ANTHROPIC, + api_key="", + base_url=base_url, + timeout_secs=timeout_secs, + extra_headers={"Authorization": f"Bearer {api_key}"}, + ), + advisor=LlmTarget( + # Anthropic Messages format: consulted via ``/v1/messages`` (the + # advisor caller sends Bearer auth directly from ``api_key``). + id="advisor", + model=advisor_model, + format=BackendFormat.ANTHROPIC, + api_key=api_key, + base_url=base_url, + timeout_secs=timeout_secs, + ), + strategy=strategy, + preset="opus47_exec_opus48_advisor", + ) + + +__all__ = ["AdvisorPresets"] diff --git a/switchyard/lib/profiles/advisor_prompts.py b/switchyard/lib/profiles/advisor_prompts.py new file mode 100644 index 00000000..6dd52abe --- /dev/null +++ b/switchyard/lib/profiles/advisor_prompts.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Default prompts for the advisor strategies. + +Two strategies share this module (selected by ``AdvisorConfig.strategy``): + +* ``tool_call`` — the executor is offered a real, parameterless ``advisor`` + tool it calls mid-generation (the proxy-side re-creation of Anthropic's + ``advisor_20260301`` server tool). ``EXECUTOR_STEERING`` and + ``ADVISOR_LENGTH_LINE`` are reproduced **verbatim** from Anthropic's + advisor-tool documentation + (https://platform.claude.com/docs/en/agents-and-tools/tool-use/advisor-tool, + sections "Suggested system prompt for coding tasks" and "Trimming advisor + output length"). ``ADVISOR_SYSTEM_PROMPT`` and ``ADVISOR_TOOL_DESCRIPTION`` + are authored here — the native tool runs the advisor server-side under + Anthropic-internal instructions the docs do not publish. + +* ``review_gate`` — the advisor is a once-per-session reviewer, not an + executor-triggered tool. It is consulted at the first point the executor + produces a no-tool-call turn — either a plan it is about to execute, or a + claim that the task is done — and returns ``APPROVE`` (let the executor + stop) or ``REDO`` + an optimized plan (send it back to keep working). This + preserves the executor's own test-and-iterate loop (which front-loaded + advice was found to suppress, causing premature convergence) and adds a + single quality gate on top. + +These are defaults; :class:`~switchyard.lib.profiles.advisor_config.AdvisorConfig` +exposes each as an overridable field for ablation. +""" + +from __future__ import annotations + +# Verbatim: the doc's "Timing guidance" block followed directly by the "How the +# executor should treat the advice" block (the doc instructs placing the latter +# "directly after the timing block"). Addresses the EXECUTOR model. +EXECUTOR_STEERING = """\ +You have access to an `advisor` tool backed by a stronger reviewer model. It takes NO parameters — when you call advisor(), your entire conversation history is automatically forwarded. They see the task, every tool call you've made, every result you've seen. + +Call advisor BEFORE substantive work — before writing, before committing to an interpretation, before building on an assumption. If the task requires orientation first (finding files, fetching a source, seeing what's there), do that, then call advisor. Orientation is not substantive work. Writing, editing, and declaring an answer are. + +Also call advisor: +- When you believe the task is complete. BEFORE this call, make your deliverable durable: write the file, save the result, commit the change. The advisor call takes time; if the session ends during it, a durable result persists and an unwritten one doesn't. +- When stuck — errors recurring, approach not converging, results that don't fit. +- When considering a change of approach. + +On tasks longer than a few steps, call advisor at least once before committing to an approach and once before declaring done. On short reactive tasks where the next action is dictated by tool output you just read, you don't need to keep calling — the advisor adds most of its value on the first call, before the approach crystallizes. + +Give the advice serious weight. If you follow a step and it fails empirically, or you have primary-source evidence that contradicts a specific claim (the file says X, the paper states Y), adapt. A passing self-test is not evidence the advice is wrong — it's evidence your test doesn't check what the advice is checking. + +If you've already retrieved data pointing one way and the advisor points another: don't silently switch. Surface the conflict in one more advisor call — "I found X, you suggest Y, which constraint breaks the tie?" The advisor saw your evidence but may have underweighted it; a reconcile call is cheaper than committing to the wrong branch.\ +""" + +# Verbatim: the doc's advisor-directed length line. The doc places it in the +# user message because the advisor follows instructions addressed to it directly +# far more reliably than third-person descriptions. +ADVISOR_LENGTH_LINE = ( + "(Advisor: please keep your guidance under 80 words — I need a focused " + "starting point, not a comprehensive plan.)" +) + +# Authored here (the doc publishes no advisor system prompt — it is server-side +# internal in the native tool). Tells the advisor model its role so it advises +# rather than attempting the task itself. +ADVISOR_SYSTEM_PROMPT = """\ +You are a higher-intelligence advisor model consulted mid-task by a faster executor model. You can see the full conversation: the task, every tool call, and every result. You do not act, write code, or call tools — you provide strategic guidance only: a focused plan or a course correction the executor will carry out. Be concrete and brief.\ +""" + +# Authored here (the native tool's "built-in description" is not published). +# Description for the synthetic ``advisor`` tool offered to the executor. +ADVISOR_TOOL_DESCRIPTION = ( + "Consult a stronger reviewer model for strategic guidance. Takes no " + "parameters; your full conversation history is forwarded automatically." +) + +REVIEWER_SYSTEM_PROMPT = """\ +You are a senior reviewer acting as a quality gate for a faster executor model working a coding/agent task. You are given the full transcript: the task, every action the executor took and every result it saw, and its latest message — in which it has either (a) proposed a plan before doing the work, or (b) concluded the task is complete. + +Decide whether to let the executor stop or send it back to keep working. Put your verdict as the FIRST word of your reply: + +- APPROVE — the proposed plan is sound, OR the work is genuinely complete and correct. Reply with exactly: APPROVE +- REDO — the plan has a real flaw, OR the work is incomplete/incorrect: an unhandled edge case, an untested assumption, a subtly wrong approach, missing verification, or a stated requirement not met. Reply: REDO, then a SHORT, concrete, actionable plan naming exactly what is wrong or missing and what to do about it. No generic advice — point at the specific gap. + +Bias toward APPROVE when the work looks correct and complete; the executor has already done its own iteration. Use REDO specifically to catch a premature "done" on a subtly incomplete solution, or a flawed plan before it is executed. A self-claim of success is not proof — check the actual task requirements against what was actually done. +""" + +#: Prepended to the advisor's REDO plan when it is injected back to the executor +#: as a user turn, instructing it to continue rather than stop. +REDO_FEEDBACK_PREFIX = ( + "A senior reviewer examined your work and determined the task is NOT yet " + "complete or correct. Do not stop here — address the following, then keep " + "working until it is genuinely done:\n\n" +) + +__all__ = [ + "ADVISOR_LENGTH_LINE", + "ADVISOR_SYSTEM_PROMPT", + "ADVISOR_TOOL_DESCRIPTION", + "EXECUTOR_STEERING", + "REDO_FEEDBACK_PREFIX", + "REVIEWER_SYSTEM_PROMPT", +] diff --git a/tests/test_advisor_loop_backend.py b/tests/test_advisor_loop_backend.py new file mode 100644 index 00000000..0e8b1485 --- /dev/null +++ b/tests/test_advisor_loop_backend.py @@ -0,0 +1,310 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the review-gate :class:`AdvisorLoopBackend` (native Anthropic). + +All loop tests inject a fake executor backend (returns ``ChatResponse``) and a +fake advisor caller — no network. They cover the gate contract: tool-use turns +pass through unreviewed; the first no-tool-use turn is reviewed once; APPROVE +returns it; REDO re-invokes the executor to continue; the review is +once-per-session and the session is pure passthrough afterward; fail-open +approves. Separate tests cover the Anthropic reviewer caller (respx) and the +pure helpers. Both streaming and completion executor responses are exercised. +""" + +import json +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +import respx + +from switchyard.lib.backends.advisor_loop_backend import ( + AdvisorLoopBackend, + _anthropic_text, + _AnthropicAdvisorCaller, + _build_advisor_caller, + _messages_url, + _parse_verdict, + _session_key, +) +from switchyard.lib.chat_response.anthropic import AnthropicResponseStream +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.proxy_context import ProxyContext +from switchyard_rust.core import ChatRequest, ChatResponse, ChatResponseType, response_type_matches + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +def _completion_resp(*, text=None, tool_use=False, model="exec-model") -> ChatResponse: + """An Anthropic completion ChatResponse (text and/or a tool_use block).""" + content: list[dict] = [] + if tool_use: + content.append({"type": "tool_use", "id": "t1", "name": "bash", "input": {}}) + if text is not None: + content.append({"type": "text", "text": text}) + body = { + "id": "msg-x", "type": "message", "role": "assistant", "model": model, + "content": content, + "stop_reason": "tool_use" if tool_use else "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 3}, + } + return ChatResponse.anthropic_completion(body) + + +async def _agen(events): + for event in events: + yield event + + +def _stream_resp(*, text=None, tool_use=False) -> ChatResponse: + """An Anthropic streaming ChatResponse (SSE event dicts).""" + events: list[dict] = [{"type": "message_start", "message": {"usage": {"input_tokens": 10}}}] + if tool_use: + events.append({"type": "content_block_start", "index": 0, + "content_block": {"type": "tool_use", "id": "t1", "name": "bash", "input": {}}}) + events.append({"type": "message_delta", "delta": {"stop_reason": "tool_use"}, + "usage": {"output_tokens": 1}}) + else: + events.append({"type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": ""}}) + if text: + events.append({"type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": text}}) + events.append({"type": "message_delta", "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 3}}) + events.append({"type": "message_stop"}) + return ChatResponse.anthropic_stream(AnthropicResponseStream(_agen(events))) + + +def _exec_backend(*responses) -> MagicMock: + b = MagicMock() + b.call = AsyncMock(side_effect=list(responses)) + b.startup = AsyncMock() + b.shutdown = AsyncMock() + return b + + +def _reviewer(*verdicts: str) -> MagicMock: + """Fake advisor reviewer: ``advise`` yields successive ``(verdict_text, usage)``.""" + c = MagicMock() + c.advise = AsyncMock(side_effect=[(v, None) for v in verdicts]) + return c + + +def _failing_reviewer(exc: Exception) -> MagicMock: + c = MagicMock() + c.advise = AsyncMock(side_effect=exc) + return c + + +def _config(**overrides) -> AdvisorConfig: + base: dict = { + "executor": {"model": "exec-model", "base_url": "http://exec.test", "api_key": "k", + "format": "anthropic"}, + "advisor": {"model": "adv-model", "base_url": "http://adv.test", "api_key": "k", + "format": "anthropic"}, + } + base.update(overrides) + return AdvisorConfig(**base) + + +def _backend(config, executor_backend, advisor_caller) -> AdvisorLoopBackend: + return AdvisorLoopBackend( + config, executor_backend=executor_backend, advisor_caller=advisor_caller, + ) + + +def _request(**overrides) -> ChatRequest: + body: dict = {"model": "incoming", "system": "sys", + "messages": [{"role": "user", "content": "build X"}]} + body.update(overrides) + return ChatRequest.anthropic(body) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Gate behavior +# --------------------------------------------------------------------------- + + +async def test_tool_use_turn_passes_through_unreviewed() -> None: + """A turn with a tool_use block means the executor is working — no review.""" + exec_b = _exec_backend(_completion_resp(text="reading", tool_use=True)) + adv = _reviewer() + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert resp.to_body()["stop_reason"] == "tool_use" + assert exec_b.call.await_count == 1 + assert adv.advise.await_count == 0 # advisor never consulted + + +async def test_terminal_turn_approved_returns_as_is() -> None: + """First no-tool-use turn is reviewed; APPROVE returns it unchanged.""" + exec_b = _exec_backend(_completion_resp(text="done, all good")) + adv = _reviewer("APPROVE") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert _anthropic_text(resp.to_body()) == "done, all good" + assert exec_b.call.await_count == 1 + assert adv.advise.await_count == 1 + + +async def test_review_records_advisor_cost_into_stats() -> None: + """The advisor review's tokens are recorded into the accumulator so the run's + own cost output (routing_stats) includes the advisor, not just the executor.""" + exec_b = _exec_backend(_completion_resp(text="done")) + adv = MagicMock() + adv.advise = AsyncMock(return_value=("APPROVE", {"input_tokens": 500, "output_tokens": 8})) + stats = MagicMock() + stats.record_planner_usage = AsyncMock() + stats.record_success = AsyncMock() + backend = AdvisorLoopBackend( + _config(), stats_accumulator=stats, executor_backend=exec_b, advisor_caller=adv, + ) + await backend.call(ProxyContext(), _request()) + stats.record_planner_usage.assert_awaited_once() + kw = stats.record_planner_usage.await_args.kwargs + assert kw["model"] == "adv-model" + assert kw["prompt_tokens"] == 500 + assert kw["completion_tokens"] == 8 + + +async def test_redo_reinvokes_executor_with_feedback() -> None: + """REDO feeds the plan back and re-invokes the executor to keep working.""" + exec_b = _exec_backend( + _completion_resp(text="I think I'm done"), # terminal → review + _completion_resp(text="continuing", tool_use=True), # redo continuation (passthrough) + ) + adv = _reviewer("REDO: you forgot the empty-input case; add a guard and test it") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + assert exec_b.call.await_count == 2 # original + redo re-invocation + redo_request = exec_b.call.await_args_list[1].args[1] + redo_msgs = redo_request.to_body()["messages"] + assert any(m.get("role") == "assistant" and m.get("content") == "I think I'm done" for m in redo_msgs) + assert any(m.get("role") == "user" and "empty-input case" in (m.get("content") or "") for m in redo_msgs) + # the returned response is the continuation (has the real tool call) + assert resp.to_body()["stop_reason"] == "tool_use" + + +async def test_review_is_once_per_session() -> None: + """Two terminal turns in the same session → advisor consulted only once.""" + exec_b = _exec_backend(_completion_resp(text="done1"), _completion_resp(text="done2")) + adv = _reviewer("APPROVE") # only one verdict provided on purpose + backend = _backend(_config(), exec_b, adv) + await backend.call(ProxyContext(), _request()) # review #1 + await backend.call(ProxyContext(), _request()) # same prefix → no review + assert adv.advise.await_count == 1 + assert exec_b.call.await_count == 2 + + +async def test_reviewed_session_passes_through_verbatim() -> None: + """After the gate fires, later turns are pure passthrough (no advisor, returned as-is).""" + exec_b = _exec_backend( + _completion_resp(text="plan"), # review fires here + _completion_resp(text="more", tool_use=True), # passthrough verbatim + ) + adv = _reviewer("APPROVE") + backend = _backend(_config(), exec_b, adv) + await backend.call(ProxyContext(), _request()) + resp = await backend.call(ProxyContext(), _request()) + assert adv.advise.await_count == 1 + assert resp.to_body()["stop_reason"] == "tool_use" + + +async def test_fail_open_approves_on_review_error() -> None: + exec_b = _exec_backend(_completion_resp(text="done")) + adv = _failing_reviewer(RuntimeError("advisor down")) + resp = await _backend(_config(fail_open=True), exec_b, adv).call(ProxyContext(), _request()) + assert _anthropic_text(resp.to_body()) == "done" + assert exec_b.call.await_count == 1 # no redo + + +async def test_fail_closed_propagates_review_error() -> None: + exec_b = _exec_backend(_completion_resp(text="done")) + adv = _failing_reviewer(RuntimeError("advisor down")) + with pytest.raises(RuntimeError, match="advisor down"): + await _backend(_config(fail_open=False), exec_b, adv).call(ProxyContext(), _request()) + + +async def test_streaming_terminal_approved_replays() -> None: + """Streaming no-tool-use turn → review → APPROVE → replay (one generation).""" + exec_b = _exec_backend(_stream_resp(text="done")) + adv = _reviewer("APPROVE") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request(stream=True)) + assert exec_b.call.await_count == 1 + assert adv.advise.await_count == 1 + assert response_type_matches(resp, ChatResponseType.ANTHROPIC_STREAM) + events = [e async for e in resp.stream] + assert any(isinstance(e, dict) and e.get("type") == "content_block_delta" for e in events) + + +async def test_streaming_tool_use_passes_through() -> None: + """Streaming turn with a tool_use block → pass through, no review.""" + exec_b = _exec_backend(_stream_resp(tool_use=True)) + adv = _reviewer() + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request(stream=True)) + assert adv.advise.await_count == 0 + assert response_type_matches(resp, ChatResponseType.ANTHROPIC_STREAM) + + +# --------------------------------------------------------------------------- +# Pure helpers + advisor caller +# --------------------------------------------------------------------------- + + +def test_parse_verdict() -> None: + assert _parse_verdict("APPROVE") == ("APPROVE", "") + assert _parse_verdict("approve, looks complete")[0] == "APPROVE" + v, plan = _parse_verdict("REDO: add a guard for empty input and re-run tests") + assert v == "REDO" and "add a guard" in plan + assert _parse_verdict("hmm not sure")[0] == "APPROVE" # unclear → approve + + +def test_session_key_stable_across_turns() -> None: + msgs = [{"role": "user", "content": "the task"}] + later = msgs + [{"role": "assistant", "content": "..."}, {"role": "user", "content": "tool result"}] + assert _session_key("sys", msgs) == _session_key("sys", later) + assert _session_key("sys", msgs) != _session_key("sys", [{"role": "user", "content": "DIFFERENT"}]) + assert _session_key("sys", msgs) != _session_key("OTHER sys", msgs) # system is part of the key + + +def test_build_advisor_caller_is_anthropic() -> None: + assert isinstance(_build_advisor_caller(_config()), _AnthropicAdvisorCaller) + + +@respx.mock +async def test_anthropic_reviewer_hits_messages_endpoint() -> None: + route = respx.post("https://inference-api.nvidia.com/v1/messages").mock( + return_value=httpx.Response(200, json={ + "content": [{"type": "text", "text": "APPROVE"}], + "usage": {"input_tokens": 10, "output_tokens": 2}, + }) + ) + caller = _AnthropicAdvisorCaller( + api_key="secret-key", base_url="https://inference-api.nvidia.com/v1", + model="aws/anthropic/bedrock-claude-opus-4-8", max_tokens=256, + temperature=None, timeout=5.0, + ) + text, usage = await caller.advise(system="review this", transcript="conversation") + assert text == "APPROVE" + assert usage["input_tokens"] == 10 + request = route.calls.last.request + assert request.headers["authorization"] == "Bearer secret-key" + body = json.loads(request.content) + assert body["system"] == "review this" + assert "temperature" not in body + + +def test_messages_url_resolution() -> None: + base = "https://inference-api.nvidia.com" + assert _messages_url(f"{base}/v1") == f"{base}/v1/messages" + assert _messages_url(f"{base}/v1/messages") == f"{base}/v1/messages" + assert _messages_url(base) == f"{base}/v1/messages" + + +def test_anthropic_text_joins_text_blocks() -> None: + data = {"content": [{"type": "text", "text": "RE"}, {"type": "thinking", "text": "x"}, + {"type": "text", "text": "DO: fix it"}]} + assert _anthropic_text(data) == "REDO: fix it" diff --git a/tests/test_advisor_profile.py b/tests/test_advisor_profile.py new file mode 100644 index 00000000..b12d85be --- /dev/null +++ b/tests/test_advisor_profile.py @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Wiring tests for the advisor profile, config, preset, and public exports.""" + +from __future__ import annotations + +import pydantic +import pytest + +import switchyard +from switchyard.lib.backends.advisor_loop_backend import AdvisorLoopBackend +from switchyard.lib.backends.advisor_tool_call_backend import AdvisorToolCallBackend +from switchyard.lib.backends.llm_target import BackendFormat +from switchyard.lib.processors.reasoning_effort_normalizer import ( + ReasoningEffortNormalizer, +) +from switchyard.lib.processors.stats_request_processor import StatsRequestProcessor +from switchyard.lib.processors.stats_response_processor_accumulator import ( + StatsResponseProcessor, +) +from switchyard.lib.profiles import ( + AdvisorConfig, + AdvisorPresets, + AdvisorProfileConfig, + ProfileSwitchyard, +) +from switchyard.lib.profiles.table import profile_config_type +from switchyard.lib.stats_accumulator import StatsAccumulator + + +def _config(**overrides) -> AdvisorConfig: + base: dict = { + "executor": {"model": "exec-model", "base_url": "http://exec.test", "api_key": "k"}, + "advisor": {"model": "adv-model", "base_url": "http://adv.test", "api_key": "k"}, + } + base.update(overrides) + return AdvisorConfig(**base) + + +def _advisor_switchyard( + config: AdvisorConfig, + *, + stats_accumulator: StatsAccumulator | None = None, +) -> ProfileSwitchyard: + """Build the profile-backed runtime used by these tests.""" + return ProfileSwitchyard( + AdvisorProfileConfig.from_config(config) + .build() + .with_runtime_components( + stats_accumulator=stats_accumulator, + enable_stats=config.enable_stats, + ) + ) + + +class TestProfileStructure: + def test_registered_profile_type_is_advisor(self) -> None: + assert profile_config_type(AdvisorProfileConfig) == "advisor" + + def test_returns_profile_backed_switchyard_adapter(self) -> None: + assert isinstance(_advisor_switchyard(_config()), ProfileSwitchyard) + + def test_backend_defaults_to_tool_call(self) -> None: + cfg = _config() + assert cfg.strategy == "tool_call" + components = list(_advisor_switchyard(cfg).iter_components()) + assert any(isinstance(c, AdvisorToolCallBackend) for c in components) + + def test_review_gate_strategy_builds_loop_backend(self) -> None: + components = list( + _advisor_switchyard(_config(strategy="review_gate")).iter_components() + ) + assert any(isinstance(c, AdvisorLoopBackend) for c in components) + + def test_reasoning_effort_normalizer_present(self) -> None: + components = list(_advisor_switchyard(_config()).iter_components()) + assert any(isinstance(c, ReasoningEffortNormalizer) for c in components) + + def test_stats_processors_wired_when_enabled(self) -> None: + components = list(_advisor_switchyard(_config()).iter_components()) + assert any(isinstance(c, StatsRequestProcessor) for c in components) + assert any(isinstance(c, StatsResponseProcessor) for c in components) + + def test_stats_processors_absent_when_disabled(self) -> None: + components = list( + _advisor_switchyard(_config(enable_stats=False)).iter_components() + ) + assert not any(isinstance(c, StatsRequestProcessor) for c in components) + assert not any(isinstance(c, StatsResponseProcessor) for c in components) + + def test_runtime_attaches_accumulator_to_backend(self) -> None: + # The advisor backends do their own stats accounting (like + # LatencyServiceLLMBackend) and cannot be wrapped by StatsLlmBackend; + # with_runtime_components must attach the shared accumulator through + # the ``_stats`` compatibility hook instead. + stats = StatsAccumulator() + switchyard_adapter = _advisor_switchyard(_config(), stats_accumulator=stats) + backend = next( + c for c in switchyard_adapter.iter_components() + if isinstance(c, AdvisorToolCallBackend) + ) + assert backend._stats is stats + + +class TestAdvisorConfig: + def test_coerces_dict_targets(self) -> None: + cfg = _config() + assert cfg.executor.model == "exec-model" + assert cfg.advisor.model == "adv-model" + + def test_rejects_empty_target_model(self) -> None: + # The Rust-backed LlmTarget rejects empty model ids during coercion, + # before the config-level non-empty validator can fire. + with pytest.raises(pydantic.ValidationError, match="must not be empty"): + _config(executor={"model": "", "base_url": "http://e", "api_key": "k"}) + + def test_rejects_unknown_strategy(self) -> None: + with pytest.raises(pydantic.ValidationError): + _config(strategy="both") + + +class TestOpusPairPreset: + """Pins the validated executor+advisor pairing on the shipping default.""" + + def test_preset_pairs_opus_47_and_48(self) -> None: + cfg = AdvisorPresets.opus47_exec_opus48_advisor(api_key="nvapi-test") + assert cfg.executor.model == "aws/anthropic/bedrock-claude-opus-4-7" + assert cfg.advisor.model == "aws/anthropic/bedrock-claude-opus-4-8" + assert cfg.preset == "opus47_exec_opus48_advisor" + assert cfg.executor.endpoint.base_url == "https://inference-api.nvidia.com/v1" + # Both tiers are native Anthropic-Messages (no OpenAI translation → + # caching survives). The executor suppresses x-api-key and + # authenticates via Bearer. + assert cfg.executor.format == BackendFormat.ANTHROPIC + assert cfg.advisor.format == BackendFormat.ANTHROPIC + assert cfg.executor.endpoint.api_key == "" + assert cfg.executor.extra_headers == {"Authorization": "Bearer nvapi-test"} + + def test_preset_defaults_to_tool_call_strategy(self) -> None: + cfg = AdvisorPresets.opus47_exec_opus48_advisor(api_key="k") + assert cfg.strategy == "tool_call" + gate = AdvisorPresets.opus47_exec_opus48_advisor( + api_key="k", strategy="review_gate", + ) + assert gate.strategy == "review_gate" + + def test_preset_model_overrides(self) -> None: + cfg = AdvisorPresets.opus47_exec_opus48_advisor( + api_key="k", executor_model="custom/exec", advisor_model="custom/adv", + ) + assert cfg.executor.model == "custom/exec" + assert cfg.advisor.model == "custom/adv" + + +def test_public_exports() -> None: + assert switchyard.AdvisorConfig is AdvisorConfig + assert switchyard.AdvisorPresets is AdvisorPresets + assert switchyard.AdvisorProfileConfig is AdvisorProfileConfig + for name in ("AdvisorConfig", "AdvisorPresets", "AdvisorProfileConfig"): + assert name in switchyard.__all__ diff --git a/tests/test_advisor_tool_call_backend.py b/tests/test_advisor_tool_call_backend.py new file mode 100644 index 00000000..78bcdd96 --- /dev/null +++ b/tests/test_advisor_tool_call_backend.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the tool-call :class:`AdvisorToolCallBackend` (native Anthropic). + +All loop tests inject a fake executor backend (returns ``ChatResponse``) and a +fake advisor caller — no network. They cover the loop contract: the advisor +tool and steering are injected; an advisor-free turn returns verbatim; an +advisor ``tool_use`` is intercepted, consulted, and fed back as a +``tool_result``; ``max_uses`` returns an error result without a consult; +fail-open marks the advisor unavailable; sibling client tool calls are dropped +from the appended turn; the hard cap bounds a runaway loop. Both streaming +(block reassembly + verbatim replay) and completion paths are exercised, plus +the transcript / injection helpers. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from switchyard.lib.backends.advisor_loop_backend import _anthropic_text +from switchyard.lib.backends.advisor_tool_call_backend import ( + AdvisorToolCallBackend, + _prepend_system, + _with_length_line, +) +from switchyard.lib.chat_response.anthropic import AnthropicResponseStream +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.proxy_context import ProxyContext +from switchyard_rust.core import ChatRequest, ChatResponse, ChatResponseType + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +def _completion_resp(content: list[dict], *, stop_reason: str = "end_turn") -> ChatResponse: + """An Anthropic completion ChatResponse with the given content blocks.""" + body = { + "id": "msg-x", "type": "message", "role": "assistant", "model": "exec-model", + "content": content, + "stop_reason": stop_reason, + "usage": {"input_tokens": 10, "output_tokens": 3, "cache_read_input_tokens": 4}, + } + return ChatResponse.anthropic_completion(body) + + +def _text_turn(text: str = "done") -> ChatResponse: + return _completion_resp([{"type": "text", "text": text}]) + + +def _advisor_turn(*, extra_blocks: list[dict] | None = None) -> ChatResponse: + """A completion turn calling the advisor (optionally with sibling blocks).""" + content = [ + {"type": "text", "text": "let me ask"}, + *(extra_blocks or []), + {"type": "tool_use", "id": "adv1", "name": "advisor", "input": {}}, + ] + return _completion_resp(content, stop_reason="tool_use") + + +async def _agen(events): + for event in events: + yield event + + +def _stream_resp(events: list[dict]) -> ChatResponse: + return ChatResponse.anthropic_stream(AnthropicResponseStream(_agen(events))) + + +def _advisor_stream_events() -> list[dict]: + """SSE events for a streamed turn that calls the advisor (tool input via deltas).""" + return [ + {"type": "message_start", + "message": {"usage": {"input_tokens": 12, "cache_read_input_tokens": 5}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": "consulting"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, + "content_block": {"type": "tool_use", "id": "adv-s", "name": "advisor", "input": {}}}, + {"type": "content_block_delta", "index": 1, + "delta": {"type": "input_json_delta", "partial_json": ""}}, + {"type": "content_block_stop", "index": 1}, + {"type": "message_delta", "delta": {"stop_reason": "tool_use"}, + "usage": {"output_tokens": 7}}, + {"type": "message_stop"}, + ] + + +def _terminal_stream_events() -> list[dict]: + return [ + {"type": "message_start", "message": {"usage": {"input_tokens": 20}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": "final answer"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 5}}, + {"type": "message_stop"}, + ] + + +def _exec_backend(*responses) -> MagicMock: + b = MagicMock() + b.call = AsyncMock(side_effect=list(responses)) + b.startup = AsyncMock() + b.shutdown = AsyncMock() + return b + + +def _advisor(*advice: str) -> MagicMock: + c = MagicMock() + c.advise = AsyncMock(side_effect=[(a, {"input_tokens": 100, "output_tokens": 9}) + for a in advice]) + return c + + +def _failing_advisor(exc: Exception) -> MagicMock: + c = MagicMock() + c.advise = AsyncMock(side_effect=exc) + return c + + +def _config(**overrides) -> AdvisorConfig: + base: dict = { + "executor": {"model": "exec-model", "base_url": "http://exec.test", "api_key": "k", + "format": "anthropic"}, + "advisor": {"model": "adv-model", "base_url": "http://adv.test", "api_key": "k", + "format": "anthropic"}, + "executor_steering": "STEER", + "advisor_length_line": "(LENGTH)", + } + base.update(overrides) + return AdvisorConfig(**base) + + +def _backend(config, executor_backend, advisor_caller) -> AdvisorToolCallBackend: + return AdvisorToolCallBackend( + config, executor_backend=executor_backend, advisor_caller=advisor_caller, + ) + + +def _request(**overrides) -> ChatRequest: + body: dict = { + "model": "incoming", "system": "sys", + "messages": [{"role": "user", "content": "build X"}], + "tools": [{"name": "bash", "description": "Run a shell command", + "input_schema": {"type": "object"}}], + } + body.update(overrides) + return ChatRequest.anthropic(body) # type: ignore[arg-type] + + +def _sent_body(exec_b: MagicMock, call_index: int) -> dict: + return exec_b.call.await_args_list[call_index].args[1].to_body() + + +# --------------------------------------------------------------------------- +# Request shaping: tool injection + steering +# --------------------------------------------------------------------------- + + +async def test_advisor_tool_appended_to_client_tools() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(), exec_b, _advisor()).call(ProxyContext(), _request()) + tools = _sent_body(exec_b, 0)["tools"] + assert [t["name"] for t in tools] == ["bash", "advisor"] + assert tools[-1]["input_schema"]["properties"] == {} # parameterless + + +async def test_steering_prepends_system_and_appends_length_line() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(), exec_b, _advisor()).call(ProxyContext(), _request()) + body = _sent_body(exec_b, 0) + assert body["system"] == "STEER\n\nsys" + assert body["messages"][0]["content"] == "build X\n\n(LENGTH)" + + +async def test_steering_disabled_leaves_request_untouched() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(inject_steering=False), exec_b, _advisor()).call( + ProxyContext(), _request(), + ) + body = _sent_body(exec_b, 0) + assert body["system"] == "sys" + assert body["messages"][0]["content"] == "build X" + assert [t["name"] for t in body["tools"]] == ["bash", "advisor"] # tool still offered + + +# --------------------------------------------------------------------------- +# Loop behavior +# --------------------------------------------------------------------------- + + +async def test_advisor_free_turn_returns_without_consult() -> None: + exec_b = _exec_backend(_text_turn("done, all good")) + adv = _advisor() + ctx = ProxyContext() + resp = await _backend(_config(), exec_b, adv).call(ctx, _request()) + assert _anthropic_text(resp.to_body()) == "done, all good" + assert exec_b.call.await_count == 1 + assert adv.advise.await_count == 0 + assert ctx.selected_model == "exec-model" + + +async def test_client_tool_call_turn_returns_without_consult() -> None: + """A turn calling only the client's own tools is terminal — hand it back.""" + exec_b = _exec_backend(_completion_resp( + [{"type": "tool_use", "id": "b1", "name": "bash", "input": {"command": "ls"}}], + stop_reason="tool_use", + )) + adv = _advisor() + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert resp.to_body()["stop_reason"] == "tool_use" + assert adv.advise.await_count == 0 + + +async def test_advisor_call_is_intercepted_and_fed_back() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn("informed answer")) + adv = _advisor("try the channel-based pattern") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + assert exec_b.call.await_count == 2 + messages = _sent_body(exec_b, 1)["messages"] + assistant_turn = messages[-2] + assert assistant_turn["role"] == "assistant" + assert any(b.get("type") == "tool_use" and b.get("name") == "advisor" + for b in assistant_turn["content"]) + result_turn = messages[-1] + assert result_turn["role"] == "user" + assert result_turn["content"] == [{ + "type": "tool_result", "tool_use_id": "adv1", + "content": "try the channel-based pattern", + }] + assert _anthropic_text(resp.to_body()) == "informed answer" + + +async def test_transcript_carries_tools_conversation_and_current_turn() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn()) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + transcript = adv.advise.await_args.kwargs["transcript"] + assert "bash" in transcript # tool summary + assert "build X" in transcript # conversation + assert "it is consulting you now" in transcript # current-turn section + assert "let me ask" in transcript # current-turn text + assert adv.advise.await_args.kwargs["system"] == _config().advisor_system_prompt + + +async def test_max_uses_returns_error_result_without_consult() -> None: + exec_b = _exec_backend(_advisor_turn(), _advisor_turn(), _text_turn()) + adv = _advisor("first advice") # only one consult available on purpose + resp = await _backend(_config(max_uses=1), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + assert exec_b.call.await_count == 3 + second_result = _sent_body(exec_b, 2)["messages"][-1]["content"][0] + assert second_result["content"] == "[advisor unavailable: max_uses exceeded]" + assert _anthropic_text(resp.to_body()) == "done" + + +async def test_fail_open_marks_advisor_unavailable_and_continues() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn("proceeded unadvised")) + adv = _failing_advisor(RuntimeError("advisor down")) + resp = await _backend(_config(fail_open=True), exec_b, adv).call( + ProxyContext(), _request(), + ) + result = _sent_body(exec_b, 1)["messages"][-1]["content"][0] + assert result["content"] == "[advisor unavailable: RuntimeError]" + assert _anthropic_text(resp.to_body()) == "proceeded unadvised" + + +async def test_fail_closed_raises() -> None: + exec_b = _exec_backend(_advisor_turn()) + adv = _failing_advisor(RuntimeError("advisor down")) + with pytest.raises(RuntimeError, match="advisor down"): + await _backend(_config(fail_open=False), exec_b, adv).call( + ProxyContext(), _request(), + ) + + +async def test_sibling_tool_calls_dropped_from_appended_turn() -> None: + """Mixed advisor + client calls: the appended turn keeps only the advisor call.""" + sibling = {"type": "tool_use", "id": "b9", "name": "bash", "input": {"command": "ls"}} + exec_b = _exec_backend(_advisor_turn(extra_blocks=[sibling]), _text_turn()) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assistant_turn = _sent_body(exec_b, 1)["messages"][-2] + tool_uses = [b for b in assistant_turn["content"] if b.get("type") == "tool_use"] + assert [b["id"] for b in tool_uses] == ["adv1"] + # Non-tool blocks (the text) survive. + assert any(b.get("type") == "text" for b in assistant_turn["content"]) + + +async def test_hard_cap_returns_last_round() -> None: + exec_b = _exec_backend(*[_advisor_turn() for _ in range(8)]) + adv = _advisor(*["advice"] * 8) + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert exec_b.call.await_count == 8 + assert adv.advise.await_count == 2 # default max_uses=2; the rest get error results + assert resp.to_body()["stop_reason"] == "tool_use" + + +# --------------------------------------------------------------------------- +# Streaming +# --------------------------------------------------------------------------- + + +async def test_streaming_advisor_call_reassembled_then_terminal_replayed() -> None: + terminal_events = _terminal_stream_events() + exec_b = _exec_backend( + _stream_resp(_advisor_stream_events()), _stream_resp(terminal_events), + ) + adv = _advisor("advice") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + # The advisor call was reassembled from deltas: the fed-back tool_result + # targets the streamed tool_use id, and the current-turn text made it into + # the transcript. + result_turn = _sent_body(exec_b, 1)["messages"][-1] + assert result_turn["content"][0]["tool_use_id"] == "adv-s" + assert "consulting" in adv.advise.await_args.kwargs["transcript"] + # The terminal turn's buffered events are replayed verbatim. + assert resp.response_type == ChatResponseType.ANTHROPIC_STREAM + replayed = [event async for event in resp.stream] + assert replayed == terminal_events + + +# --------------------------------------------------------------------------- +# Stats accounting +# --------------------------------------------------------------------------- + + +async def test_planner_bucket_prices_internal_turns_and_consults() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn()) + adv = _advisor("advice") + stats = MagicMock() + stats.record_planner_usage = AsyncMock() + stats.record_success = AsyncMock() + backend = AdvisorToolCallBackend( + _config(), stats_accumulator=stats, executor_backend=exec_b, advisor_caller=adv, + ) + await backend.call(ProxyContext(), _request()) + + assert stats.record_planner_usage.await_count == 2 + by_model = {c.kwargs["model"]: c.kwargs for c in stats.record_planner_usage.await_args_list} + # The intercepted executor turn (client never sees it) is priced under the + # executor model, cache reads included. + assert by_model["exec-model"]["prompt_tokens"] == 10 + assert by_model["exec-model"]["cached_tokens"] == 4 + # The consult is priced under the advisor model. + assert by_model["adv-model"]["prompt_tokens"] == 100 + assert by_model["adv-model"]["completion_tokens"] == 9 + stats.record_success.assert_awaited_once() + assert stats.record_success.await_args.args[0] == "exec-model" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def test_serialize_transcript_drops_oldest_when_over_cap() -> None: + backend = _backend(_config(transcript_max_chars=300), _exec_backend(), _advisor()) + messages = [ + {"role": "user", "content": f"turn-{i}: " + "x" * 80} for i in range(10) + ] + transcript = backend._serialize_transcript(messages, "current", []) + assert "turn-9" in transcript # newest kept + assert "turn-0" not in transcript # oldest dropped + assert "earlier messages omitted" in transcript + assert transcript.endswith("current") + + +def test_serialize_transcript_no_truncation_under_cap() -> None: + backend = _backend(_config(), _exec_backend(), _advisor()) + messages = [{"role": "user", "content": "hello"}] + transcript = backend._serialize_transcript(messages, "", []) + assert "hello" in transcript + assert "omitted" not in transcript + assert transcript.endswith("(no text)") + + +def test_prepend_system_shapes() -> None: + assert _prepend_system(None, "S") == "S" + assert _prepend_system("base", "S") == "S\n\nbase" + assert _prepend_system([{"type": "text", "text": "base"}], "S") == [ + {"type": "text", "text": "S"}, {"type": "text", "text": "base"}, + ] + + +def test_with_length_line_targets_first_user_message() -> None: + messages = [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "a"}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t", "content": "r"}]}, + ] + out = _with_length_line(messages, "(L)") + assert out[0]["content"] == "first\n\n(L)" + assert out[2]["content"][-1]["type"] == "tool_result" # later user turns untouched + + +def test_with_length_line_block_content() -> None: + messages = [{"role": "user", "content": [{"type": "text", "text": "first"}]}] + out = _with_length_line(messages, "(L)") + assert out[0]["content"][-1] == {"type": "text", "text": "(L)"} diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 5544e97b..425a45d2 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -1388,6 +1388,81 @@ def test_enable_stats_false_disables_stats_processors(self): assert not any(isinstance(c, StatsResponseProcessor) for c in components) +class TestAdvisorRouteType: + """`type: advisor` wires the executor + advisor chain via YAML (strategy-selected).""" + + def _bundle(self) -> dict: + return { + "routes": { + "myrouter/advisor": { + "type": "advisor", + "executor": { + "model": "aws/anthropic/bedrock-claude-opus-4-7", + "api_key": "", + "base_url": "https://exec.invalid/v1", + "format": "anthropic", + "extra_headers": {"Authorization": "Bearer sk-exec"}, + }, + "advisor": { + "model": "aws/anthropic/bedrock-claude-opus-4-8", + "api_key": "sk-adv", + "base_url": "https://adv.invalid/v1", + "format": "anthropic", + }, + }, + }, + } + + def test_registers_under_route_key(self): + table = build_route_bundle_table(self._bundle()) + assert table.registered_models() == ["myrouter/advisor"] + + def test_metadata_records_advisor_profile(self): + table = build_route_bundle_table(self._bundle()) + _, _, metadata = next(iter(table.items())) + assert metadata["switchyard"]["profile"] == "advisor" + + def test_backend_defaults_to_tool_call(self): + from switchyard.lib.backends.advisor_tool_call_backend import ( + AdvisorToolCallBackend, + ) + + table = build_route_bundle_table(self._bundle()) + components = list(table.iter_components()) + assert any(isinstance(c, AdvisorToolCallBackend) for c in components) + + def test_review_gate_strategy_selects_loop_backend(self): + from switchyard.lib.backends.advisor_loop_backend import AdvisorLoopBackend + + bundle = self._bundle() + bundle["routes"]["myrouter/advisor"]["strategy"] = "review_gate" + table = build_route_bundle_table(bundle) + components = list(table.iter_components()) + assert any(isinstance(c, AdvisorLoopBackend) for c in components) + + def test_rejects_unknown_route_key(self): + bundle = self._bundle() + bundle["routes"]["myrouter/advisor"]["bogus_field"] = 1 + with pytest.raises(RouteBundleConfigError) as exc: + build_route_bundle_table(bundle) + assert "bogus_field" in str(exc.value) + + def test_enable_stats_false_disables_stats_processors(self): + from switchyard.lib.processors.stats_request_processor import ( + StatsRequestProcessor, + ) + from switchyard.lib.processors.stats_response_processor_accumulator import ( + StatsResponseProcessor, + ) + + bundle = self._bundle() + bundle["routes"]["myrouter/advisor"]["enable_stats"] = False + table = build_route_bundle_table(bundle) + components = list(table.iter_components()) + assert not any(isinstance(c, StatsRequestProcessor) for c in components) + assert not any(isinstance(c, StatsResponseProcessor) for c in components) + + def test_stage_router_route_hydrates_tier_catalogs( monkeypatch: pytest.MonkeyPatch, ) -> None: From 2ac06dd5522662cb4c0c6e8d1106b7eeafee6b07 Mon Sep 17 00:00:00 2001 From: zengyuanl Date: Sat, 11 Jul 2026 09:38:10 +0000 Subject: [PATCH 2/2] feat(advisor): dispatch tool_call strategy on executor and advisor wire formats Signed-off-by: zengyuanl --- .agents/skills/switchyard-lib-core/SKILL.md | 2 +- switchyard/cli/route_bundle.py | 15 +- .../lib/backends/advisor_loop_backend.py | 99 ++- .../lib/backends/advisor_tool_call_backend.py | 661 ++++++++++++++---- switchyard/lib/profiles/advisor.py | 15 +- switchyard/lib/profiles/advisor_config.py | 48 +- tests/test_advisor_loop_backend.py | 43 ++ tests/test_advisor_profile.py | 51 +- .../test_advisor_tool_call_backend_openai.py | 523 ++++++++++++++ tests/test_route_bundle.py | 39 ++ 10 files changed, 1339 insertions(+), 157 deletions(-) create mode 100644 tests/test_advisor_tool_call_backend_openai.py diff --git a/.agents/skills/switchyard-lib-core/SKILL.md b/.agents/skills/switchyard-lib-core/SKILL.md index 5f6b10ec..336b91cb 100644 --- a/.agents/skills/switchyard-lib-core/SKILL.md +++ b/.agents/skills/switchyard-lib-core/SKILL.md @@ -48,7 +48,7 @@ right validation set. If the change is driven by a launcher need, also read | Per-event error log | Use `switchyard.lib.endpoints.upstream_error_log.log_upstream_attempt_failure(...)` on the failure path. Events belong in logs/traces, not Prometheus sample timestamps. | | CLI launcher integration | Build one profile-backed `SwitchyardApp` with `build_tier_passthrough_switchyard(...)` for single-target mode, or merge route YAML with `load_route_bundle_table(...)`. Hand the result to `build_switchyard_app`. | | A new preset | Put preset helpers beside the profile config they produce, under `switchyard/lib/profiles/`. Presets should return typed config objects, not runnable chains. | -| A backend that pairs the executor with a stronger advisor model | `AdvisorProfileConfig` (`switchyard/lib/profiles/advisor.py`) dispatches on `AdvisorConfig.strategy`. **`tool_call`** (default) builds `AdvisorToolCallBackend` (`switchyard/lib/backends/advisor_tool_call_backend.py`): the proxy-side re-creation of Anthropic's `advisor_20260301` server tool — a real, **parameterless** `advisor` tool is appended to the client's tools (plus doc-verbatim executor steering and a length line injected cache-stably: system prepend + **first** user message); each advisor `tool_use` is intercepted before it reaches the client, the advisor is consulted on the transcript (tools summary + tail-kept conversation + the executor's current-turn text), and the advice loops back as a `tool_result` until the turn is advisor-free (`max_uses` consults per request, then `max_uses exceeded` error results; hard cap 8 turns; mixed advisor+client-tool turns are regenerated with siblings dropped). **`review_gate`** builds `AdvisorLoopBackend` (`switchyard/lib/backends/advisor_loop_backend.py`): no tool injected; at the executor's first **no-tool-call** turn (a plan, or "done") the advisor reviews **once per session** (hash of the conversation prefix, in-process) → `APPROVE` (return as-is) or `REDO` (feed the plan back and re-invoke) — near-superset of solo; front-loaded advice was found to cause premature convergence on Opus executors (see its docstring). Compose via a `type: advisor` route (`cli/route_bundle.py`); preset `AdvisorPresets.opus47_exec_opus48_advisor(strategy=...)`. Both tiers are native Anthropic (`/v1/messages`, Bearer): the executor call is delegated **verbatim** to an `AnthropicNativeBackend` so the client's `cache_control` (prompt caching) survives — no OpenAI translation. Like `LatencyServiceLLMBackend`, both are Python multi-call backends doing their **own** stats accounting (`ctx.selected_model`, plus advisor consults — and, for tool_call, the intercepted executor turns — into the planner bucket) — do **not** wrap in `StatsLlmBackend`; `with_runtime_components` attaches the accumulator through the `_stats` hook. Prompts in `switchyard/lib/profiles/advisor_prompts.py`. | +| A backend that pairs the executor with a stronger advisor model | `AdvisorProfileConfig` (`switchyard/lib/profiles/advisor.py`) dispatches on `AdvisorConfig.strategy`. **`tool_call`** (default) builds `AdvisorToolCallBackend` (`switchyard/lib/backends/advisor_tool_call_backend.py`): the proxy-side re-creation of Anthropic's `advisor_20260301` server tool — a real, **parameterless** `advisor` tool is appended to the client's tools (plus doc-verbatim executor steering and a length line injected cache-stably: system prepend + **first** user message); each advisor `tool_use` is intercepted before it reaches the client, the advisor is consulted on the transcript (tools summary + tail-kept conversation + the executor's current-turn text), and the advice loops back as a `tool_result` until the turn is advisor-free (`max_uses` consults per request, then `max_uses exceeded` error results; hard cap 8 turns; mixed advisor+client-tool turns are regenerated with siblings dropped). **`review_gate`** builds `AdvisorLoopBackend` (`switchyard/lib/backends/advisor_loop_backend.py`): no tool injected; at the executor's first **no-tool-call** turn (a plan, or "done") the advisor reviews **once per session** (hash of the conversation prefix, in-process) → `APPROVE` (return as-is) or `REDO` (feed the plan back and re-invoke) — near-superset of solo; front-loaded advice was found to cause premature convergence on Opus executors (see its docstring). Compose via a `type: advisor` route (`cli/route_bundle.py`); preset `AdvisorPresets.opus47_exec_opus48_advisor(strategy=...)`. Both tiers are **format-dispatched on `LlmTarget.format` and mix freely**: `anthropic` executors delegate **verbatim** to `AnthropicNativeBackend` (`/v1/messages`, Bearer — the client's `cache_control` prompt caching survives), `openai` executors (Qwen/DeepSeek/vLLM/NIM/OpenAI) delegate verbatim to `OpenAiNativeBackend` (`/chat/completions`) with the loop run on OpenAI-Chat wire via the private `_AnthropicDialect`/`_OpenAiDialect` objects in `advisor_tool_call_backend.py`; the advisor caller likewise dispatches on `advisor.format` (`_AnthropicAdvisorCaller` httpx `/v1/messages`, or `_OpenAiAdvisorCaller` via `OpenAILLMClient` `/chat/completions`). `responses` targets are rejected at `AdvisorConfig` validation (the loop is Chat-shaped), and `review_gate` is validated Anthropic-executor-only. Like `LatencyServiceLLMBackend`, both are Python multi-call backends doing their **own** stats accounting (`ctx.selected_model`, plus advisor consults — and, for tool_call, the intercepted executor turns — into the planner bucket) — do **not** wrap in `StatsLlmBackend`; `with_runtime_components` attaches the accumulator through the `_stats` hook. Prompts in `switchyard/lib/profiles/advisor_prompts.py`. | ## Profile Pattern diff --git a/switchyard/cli/route_bundle.py b/switchyard/cli/route_bundle.py index 2e062ba1..37ba9fd1 100644 --- a/switchyard/cli/route_bundle.py +++ b/switchyard/cli/route_bundle.py @@ -1160,12 +1160,15 @@ def _advisor_switchyard( Mirrors :func:`_plan_execute_switchyard`: the ``executor`` / ``advisor`` tiers and scalar fields go through the shared ``_route_config`` → - :meth:`AdvisorConfig.model_validate` path. Both tiers are required and both - are typically ``format: anthropic`` (native ``/v1/messages``, so the - executor passthrough preserves prompt caching). ``strategy`` selects the - advisor mode: ``tool_call`` (default) offers the executor a - proxy-intercepted ``advisor`` tool; ``review_gate`` gates the executor with - a once-per-session advisor review. + :meth:`AdvisorConfig.model_validate` path. Both tiers are required, and + each tier's ``format`` selects its wire independently — ``anthropic`` + (native ``/v1/messages``; the executor passthrough preserves prompt + caching) or ``openai`` (``/chat/completions``; Qwen/DeepSeek/vLLM/NIM/ + OpenAI endpoints). ``responses`` targets are rejected at validation. + ``strategy`` selects the advisor mode: ``tool_call`` (default) offers the + executor a proxy-intercepted ``advisor`` tool; ``review_gate`` + (Anthropic-executor-only) gates the executor with a once-per-session + advisor review. """ config = AdvisorConfig.model_validate( _route_config(route, target_defaults, ("executor", "advisor")) diff --git a/switchyard/lib/backends/advisor_loop_backend.py b/switchyard/lib/backends/advisor_loop_backend.py index 74293e3a..90089afc 100644 --- a/switchyard/lib/backends/advisor_loop_backend.py +++ b/switchyard/lib/backends/advisor_loop_backend.py @@ -31,12 +31,14 @@ "done", plus one quality gate — so it is downside-protected (≈ baseline if the advisor always approves) while catching premature convergence. -Both tiers are **native Anthropic** (``/v1/messages``, Bearer auth) — no OpenAI -anywhere. The executor is called by delegating verbatim to an -:class:`AnthropicNativeBackend`, so the client's ``cache_control`` breakpoints -reach the upstream unchanged and prompt caching is honored. The gate reads the -executor's tool use from the Anthropic ``stop_reason``/``tool_use`` content -blocks. The advisor is consulted via the Anthropic Messages API. +The executor is **native Anthropic** (``/v1/messages``, Bearer auth): it is +called by delegating verbatim to an :class:`AnthropicNativeBackend`, so the +client's ``cache_control`` breakpoints reach the upstream unchanged and prompt +caching is honored (``AdvisorConfig`` validation rejects non-Anthropic +executors for this strategy). The gate reads the executor's tool use from the +Anthropic ``stop_reason``/``tool_use`` content blocks. The advisor tier is +format-dispatched (``_build_advisor_caller``): Anthropic Messages or OpenAI +Chat Completions. Chain integration:: @@ -87,6 +89,7 @@ from switchyard_rust.translation import TranslationEngine if TYPE_CHECKING: + from switchyard.lib.backends.llm_target import LlmTarget from switchyard.lib.proxy_context import ProxyContext from switchyard.lib.stats_accumulator import StatsAccumulator from switchyard_rust.core import ChatRequest @@ -313,20 +316,34 @@ def _serialize_transcript( # ---------------------------------------------------------------------- -# Advisor caller (native Anthropic Messages) +# Advisor callers # ---------------------------------------------------------------------- def _build_advisor_caller(config: AdvisorConfig) -> AdvisorCaller: - """Build the Anthropic Messages advisor caller for ``config.advisor``.""" - target = config.advisor - return _AnthropicAdvisorCaller( - api_key=target.endpoint.api_key, - base_url=target.endpoint.base_url, - model=target.model, - max_tokens=config.advisor_max_tokens, - temperature=config.advisor_temperature, - timeout=target.endpoint.timeout_secs, + """Build the advisor caller for ``config.advisor``, dispatched on its format.""" + from switchyard.lib.backends.llm_target import BackendFormat + from switchyard.lib.backends.multi_llm_backend import resolve_llm_target + + target = resolve_llm_target(config.advisor) + if target.format == BackendFormat.ANTHROPIC: + return _AnthropicAdvisorCaller( + api_key=target.endpoint.api_key, + base_url=target.endpoint.base_url, + model=target.model, + max_tokens=config.advisor_max_tokens, + temperature=config.advisor_temperature, + timeout=target.endpoint.timeout_secs, + ) + if target.format == BackendFormat.OPENAI: + return _OpenAiAdvisorCaller( + target=target, + max_tokens=config.advisor_max_tokens, + temperature=config.advisor_temperature, + ) + raise ValueError( + f"advisor tier does not support format {target.format!r}; " + "use 'openai' or 'anthropic'" ) @@ -365,6 +382,56 @@ async def advise(self, *, system: str, transcript: str) -> tuple[str, Any]: return _anthropic_text(data), data.get("usage") +class _OpenAiAdvisorCaller: + """Consults an OpenAI-Chat advisor (``/chat/completions`` via the SDK). + + Covers OSS advisors (DeepSeek, Qwen on vLLM/NIM) and OpenAI. Built with + ``max_retries=0`` so a slow or down advisor falls through to the backend's + own ``fail_open`` handling at the configured timeout instead of + compounding via SDK exponential backoff (same rationale as the LLM + classifier's client). + """ + + def __init__( + self, *, target: LlmTarget, max_tokens: int, temperature: float | None, + ) -> None: + from switchyard.lib.llm_client import OpenAILLMClient + + self._client = OpenAILLMClient( + api_key=target.endpoint.api_key, + base_url=target.endpoint.base_url, + timeout=target.endpoint.timeout_secs, + max_retries=0, + ) + self._model = target.model + self._max_tokens = max_tokens + self._temperature = temperature + # Forward target-level overrides so gateway auth headers and vLLM + # chat-template hints configured on the route work here too. + self._extra_body = dict(target.extra_body) if target.extra_body else None + self._extra_headers = dict(target.extra_headers) if target.extra_headers else None + + async def advise(self, *, system: str, transcript: str) -> tuple[str, Any]: + kwargs: dict[str, Any] = { + "model": self._model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": transcript}, + ], + "max_tokens": self._max_tokens, + } + if self._temperature is not None: + kwargs["temperature"] = self._temperature + if self._extra_body is not None: + kwargs["extra_body"] = self._extra_body + if self._extra_headers is not None: + kwargs["extra_headers"] = self._extra_headers + result = await self._client.acompletion(**kwargs) + choices = getattr(result, "choices", None) or [] + content = getattr(getattr(choices[0], "message", None), "content", None) if choices else None + return (content or "").strip(), getattr(result, "usage", None) + + # ---------------------------------------------------------------------- # Module-level helpers # ---------------------------------------------------------------------- diff --git a/switchyard/lib/backends/advisor_tool_call_backend.py b/switchyard/lib/backends/advisor_tool_call_backend.py index 26739da1..1f5aa63d 100644 --- a/switchyard/lib/backends/advisor_tool_call_backend.py +++ b/switchyard/lib/backends/advisor_tool_call_backend.py @@ -1,21 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""``LLMBackend`` that re-creates Anthropic's advisor tool proxy-side (native Anthropic). +"""``LLMBackend`` that re-creates Anthropic's advisor tool proxy-side, on any Chat wire. Anthropic's native ``advisor_20260301`` server tool runs the advisor sub-inference server-side, so it is unavailable on gateways that only proxy -model traffic (e.g. NVIDIA Inference Hub). This backend reproduces the -*executor-triggered* behavior in the proxy: - -1. Offer the executor a real, parameterless ``advisor`` tool (empty - ``input_schema`` — like the native tool, the executor signals timing and the - proxy supplies context). -2. Call the executor. If its turn contains ``advisor`` ``tool_use`` blocks, - intercept them **before they reach the client**, consult the advisor model - on the transcript (including the text the executor produced so far in the - turn, mirroring the native tool's context), append the advice as a - ``tool_result``, and call the executor again. +model traffic (e.g. NVIDIA Inference Hub) — and it only exists for Anthropic +models at all. This backend reproduces the *executor-triggered* behavior in +the proxy, for **any** Chat-shaped executor: + +1. Offer the executor a real, parameterless ``advisor`` tool (empty schema — + like the native tool, the executor signals timing and the proxy supplies + context). +2. Call the executor. If its turn contains ``advisor`` tool calls, intercept + them **before they reach the client**, consult the advisor model on the + transcript (including the text the executor produced so far in the turn, + mirroring the native tool's context), append the advice as the tool + result, and call the executor again. 3. Loop until the executor returns an advisor-free turn (a real tool call for the client, or a final answer), then return that turn. @@ -24,36 +25,52 @@ executor sees the error and continues, mirroring the native tool's ``max_uses_exceeded`` error result. -Both tiers are **native Anthropic** (``/v1/messages``, Bearer auth). The -executor is called by delegating to an :class:`AnthropicNativeBackend`, so the -client's ``cache_control`` breakpoints reach the upstream unchanged and prompt -caching is honored. Steering is injected cache-stably: the executor steering is -prepended to the system prompt and the advisor length line is appended to the -**first** user message (both constant across a session's turns; the native -doc suggests the latest user message, but re-injecting there would shift the -cached prefix on every turn because the client never sees the injection). +The executor's wire format is selected by ``config.executor.format`` and +handled by a private dialect object: + +* ``anthropic`` — the executor call is delegated verbatim to an + :class:`AnthropicNativeBackend` (``/v1/messages``), so the client's + ``cache_control`` breakpoints reach the upstream unchanged and prompt + caching is honored. Thinking blocks round-trip verbatim for upstreams that + verify signatures. +* ``openai`` — the executor call is delegated verbatim to an + :class:`OpenAiNativeBackend` (``/chat/completions``), covering OSS models + (Qwen, DeepSeek on vLLM/NIM) and OpenAI. Vendor-specific assistant fields + (e.g. ``reasoning_content``) are dropped from the rebuilt advisor turns so + strict endpoints accept the replayed history. + +The advisor tier is likewise format-dispatched (see +``advisor_loop_backend._build_advisor_caller``); tiers mix freely. +``responses`` targets are rejected — the loop is Chat-shaped. + +Steering is injected cache-stably: the executor steering is prepended to the +system prompt and the advisor length line is appended to the **first** user +message (both constant across a session's turns; the native doc suggests the +latest user message, but re-injecting there would shift the cached prefix on +every turn because the client never sees the injection). The same placement +holds on the OpenAI wire, where vLLM/OpenAI automatic prefix caching also +wants a stable prefix. A turn that mixes advisor and client tool calls is regenerated: the appended -assistant turn keeps only the advisor ``tool_use`` blocks (plus thinking/text), -and the sibling client calls are re-issued advice-informed on the next -iteration. The native API instead pauses with the client calls pending; a -proxy cannot, because it can neither execute the client's tools nor hand the -client a turn containing a tool it was never offered. +assistant turn keeps only the advisor calls (plus text/thinking), and the +sibling client calls are re-issued advice-informed on the next iteration. The +native API instead pauses with the client calls pending; a proxy cannot, +because it can neither execute the client's tools nor hand the client a turn +containing a tool it was never offered. Chain integration:: [RequestProcessor*] → AdvisorToolCallBackend → [ResponseProcessor*] → TranslationEngine -Declares ``supported_request_types = [ANTHROPIC]`` and normalizes inbound -OpenAI / Responses via the TranslationEngine, mirroring -:class:`AdvisorLoopBackend`. The outer chain's ``StatsResponseProcessor`` -records the terminal turn's token usage; this backend additionally records the -advisor consults **and the intermediate executor turns** (which the client -never sees) into the planner bucket, so the run's cost output prices the full -loop, and stamps ``ctx.selected_model``. +Declares ``supported_request_types`` for the executor's wire so the +TranslationEngine normalizes any inbound format to it once. The outer chain's +``StatsResponseProcessor`` records the terminal turn's token usage; this +backend additionally records the advisor consults **and the intermediate +executor turns** (which the client never sees) into the planner bucket, so +the run's cost output prices the full loop, and stamps ``ctx.selected_model``. -Streaming is single-pass: each executor turn is streamed and buffered while its -content blocks are reassembled to detect advisor calls; the terminal turn's +Streaming is single-pass: each executor turn is streamed and buffered while +its content is reassembled to detect advisor calls; the terminal turn's buffered events are replayed verbatim, so the turn the client pays for is generated exactly once. Advice is not persisted across client turns (the client cannot carry advisor exchanges back); its effect lives in the terminal @@ -67,7 +84,7 @@ import sys import time from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol, cast from switchyard.lib.backends.advisor_loop_backend import ( AdvisorCaller, @@ -77,14 +94,20 @@ _replay_events, _usage_tokens, ) -from switchyard.lib.backends.multi_llm_backend import build_native_backend +from switchyard.lib.backends.llm_target import BackendFormat +from switchyard.lib.backends.multi_llm_backend import ( + build_native_backend, + resolve_llm_target, +) from switchyard.lib.chat_response.anthropic import AnthropicResponseStream +from switchyard.lib.chat_response.openai_chat import ResponseStream from switchyard.lib.profiles.advisor_config import AdvisorConfig from switchyard.lib.roles import LLMBackend from switchyard_rust.core import ( ChatRequestType, ChatResponse, ChatResponseType, + request_type_enum, request_type_matches, request_with_type, ) @@ -107,14 +130,20 @@ @dataclass class _ToolCallTurn: - """One executor turn, normalized across the streaming / completion paths. - - ``blocks`` are the turn's reassembled Anthropic content blocks (dicts). - Exactly one of ``completion_body`` / ``stream_events`` carries the payload - for verbatim replay if the turn is terminal. + """One executor turn, normalized across wire formats and delivery modes. + + ``advisor_calls`` are the turn's advisor invocations in the executor's + native shape (Anthropic ``tool_use`` blocks / OpenAI ``tool_calls`` + entries); ``message`` is the format-native assistant payload the dialect + rebuilds the feedback turn from (Anthropic content-block list / OpenAI + assistant message dict). Exactly one of ``completion_body`` / + ``stream_events`` carries the payload for verbatim replay if the turn is + terminal. """ - blocks: list[dict[str, Any]] + advisor_calls: list[dict[str, Any]] + text: str + message: Any latency_ms: float input_tokens: int output_tokens: int @@ -122,14 +151,294 @@ class _ToolCallTurn: completion_body: Any | None = None stream_events: list[Any] | None = None - @property - def text(self) -> str: - """The turn's assistant text (joined ``text`` blocks).""" - return _blocks_text(self.blocks) + +class _WireDialect(Protocol): + """Wire-format-specific operations of the advisor tool-call loop.""" + + #: ``request_with_type`` discriminator for the executor's wire + #: (``"anthropic"`` or ``"openai_chat"``). + request_type: str + + def advisor_tool_def(self, *, name: str, description: str) -> dict[str, Any]: + """Build the synthetic, parameterless ``advisor`` tool definition.""" + ... + + def inject_steering( + self, + body: dict[str, Any], + messages: list[dict[str, Any]], + *, + steering: str, + length_line: str, + ) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Return (body, messages) with executor steering injected cache-stably.""" + ... + + def tool_summaries(self, tools: list[dict[str, Any]]) -> list[tuple[str, str]]: + """Return (name, description) pairs for the client's tools.""" + ... + + async def parse_turn( + self, + response: ChatResponse, + *, + advisor_tool_name: str, + latency_ms: float, + ) -> _ToolCallTurn: + """Buffer and reassemble an executor response into a ``_ToolCallTurn``.""" + ... + + def feedback_messages( + self, turn: _ToolCallTurn, advice: str, + ) -> list[dict[str, Any]]: + """Messages appending the advisor turn + its tool result(s).""" + ... + + def replay(self, turn: _ToolCallTurn) -> ChatResponse: + """Rebuild the buffered terminal turn as a verbatim response.""" + ... + + +class _AnthropicDialect: + """Anthropic Messages wire (``tool_use`` blocks, ``tool_result`` turns).""" + + request_type = "anthropic" + + def advisor_tool_def(self, *, name: str, description: str) -> dict[str, Any]: + return { + "name": name, + "description": description, + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + } + + def inject_steering( + self, + body: dict[str, Any], + messages: list[dict[str, Any]], + *, + steering: str, + length_line: str, + ) -> tuple[dict[str, Any], list[dict[str, Any]]]: + body = {**body, "system": _prepend_system(body.get("system"), steering)} + return body, _with_length_line(messages, length_line) + + def tool_summaries(self, tools: list[dict[str, Any]]) -> list[tuple[str, str]]: + return [(str(t.get("name", "?")), str(t.get("description", ""))) for t in tools] + + async def parse_turn( + self, + response: ChatResponse, + *, + advisor_tool_name: str, + latency_ms: float, + ) -> _ToolCallTurn: + if response.response_type == ChatResponseType.ANTHROPIC_STREAM: + events, blocks, usage = await _consume_stream(response.stream) + return self._turn( + blocks, advisor_tool_name, latency_ms, stream_events=events, **usage, + ) + completion: Any = response.to_body() + blocks = [b for b in (completion.get("content") or []) if isinstance(b, dict)] + prompt_tokens, completion_tokens = _usage_tokens(completion.get("usage")) + cached = (completion.get("usage") or {}).get("cache_read_input_tokens") or 0 + return self._turn( + blocks, + advisor_tool_name, + latency_ms, + input_tokens=prompt_tokens or 0, + output_tokens=completion_tokens or 0, + cached_tokens=int(cached), + completion_body=completion, + ) + + def _turn( + self, + blocks: list[dict[str, Any]], + advisor_tool_name: str, + latency_ms: float, + **kwargs: Any, + ) -> _ToolCallTurn: + advisor_calls = [ + b for b in blocks + if b.get("type") == "tool_use" and b.get("name") == advisor_tool_name + ] + return _ToolCallTurn( + advisor_calls=advisor_calls, + text=_blocks_text(blocks), + message=blocks, + latency_ms=latency_ms, + **kwargs, + ) + + def feedback_messages( + self, turn: _ToolCallTurn, advice: str, + ) -> list[dict[str, Any]]: + # Thinking blocks must round-trip verbatim for upstreams that verify + # signatures; sibling client calls are dropped (regenerated next turn). + return [ + { + "role": "assistant", + "content": _advisor_only_content(turn.message, turn.advisor_calls), + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": call.get("id"), "content": advice} + for call in turn.advisor_calls + ], + }, + ] + + def replay(self, turn: _ToolCallTurn) -> ChatResponse: + if turn.stream_events is not None: + return ChatResponse.anthropic_stream( + AnthropicResponseStream(_replay_events(turn.stream_events)) + ) + return ChatResponse.anthropic_completion(turn.completion_body) + + +class _OpenAiDialect: + """OpenAI Chat Completions wire (``tool_calls``, ``role: tool`` results).""" + + request_type = "openai_chat" + + def advisor_tool_def(self, *, name: str, description: str) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + }, + } + + def inject_steering( + self, + body: dict[str, Any], + messages: list[dict[str, Any]], + *, + steering: str, + length_line: str, + ) -> tuple[dict[str, Any], list[dict[str, Any]]]: + # OpenAI carries the system prompt as a leading message, not a body + # field. _with_length_line's list branch already emits the OpenAI + # content-part shape, so it is shared verbatim. + messages = _prepend_system_message(messages, steering) + return body, _with_length_line(messages, length_line) + + def tool_summaries(self, tools: list[dict[str, Any]]) -> list[tuple[str, str]]: + summaries: list[tuple[str, str]] = [] + for tool in tools: + raw_function = tool.get("function") + function = raw_function if isinstance(raw_function, dict) else {} + summaries.append( + (str(function.get("name", "?")), str(function.get("description", ""))) + ) + return summaries + + async def parse_turn( + self, + response: ChatResponse, + *, + advisor_tool_name: str, + latency_ms: float, + ) -> _ToolCallTurn: + if response.response_type == ChatResponseType.OPENAI_STREAM: + events, message, usage = await _consume_openai_stream(response.stream) + return self._turn( + message, advisor_tool_name, latency_ms, stream_events=events, **usage, + ) + completion: Any = response.to_body() + choices = completion.get("choices") or [{}] + message = dict(choices[0].get("message") or {}) + usage_value = completion.get("usage") + prompt_tokens, completion_tokens = _usage_tokens(usage_value) + cached = ((usage_value or {}).get("prompt_tokens_details") or {}).get( + "cached_tokens" + ) or 0 + return self._turn( + message, + advisor_tool_name, + latency_ms, + input_tokens=prompt_tokens or 0, + output_tokens=completion_tokens or 0, + cached_tokens=int(cached), + completion_body=completion, + ) + + def _turn( + self, + message: dict[str, Any], + advisor_tool_name: str, + latency_ms: float, + **kwargs: Any, + ) -> _ToolCallTurn: + return _ToolCallTurn( + advisor_calls=_openai_advisor_calls(message, advisor_tool_name), + # reasoning_content is intentionally excluded, matching the + # Anthropic dialect excluding thinking blocks from the text. + text=str(message.get("content") or ""), + message=message, + latency_ms=latency_ms, + **kwargs, + ) + + def feedback_messages( + self, turn: _ToolCallTurn, advice: str, + ) -> list[dict[str, Any]]: + kept_ids = {call.get("id") for call in turn.advisor_calls} + # OpenAI requires exactly one role:"tool" message per tool_call id in + # the preceding assistant message; the same advice is fanned out. + return [ + _advisor_only_message(turn.message, kept_ids), + *( + { + "role": "tool", + "tool_call_id": call.get("id"), + "content": advice, + } + for call in turn.advisor_calls + ), + ] + + def replay(self, turn: _ToolCallTurn) -> ChatResponse: + if turn.stream_events is not None: + return ChatResponse.openai_stream( + ResponseStream(_replay_events(turn.stream_events)) + ) + return ChatResponse.openai_completion(turn.completion_body) + + +def _dialect_for_format(fmt: BackendFormat) -> _WireDialect: + """Return the wire dialect for a resolved executor format.""" + if fmt == BackendFormat.ANTHROPIC: + return _AnthropicDialect() + if fmt == BackendFormat.OPENAI: + return _OpenAiDialect() + if fmt == BackendFormat.RESPONSES: + # Backstop for format: auto resolving to a Responses endpoint; the + # config validator rejects an explicit responses format earlier. + raise ValueError( + "the advisor tool-call loop is Chat-shaped and does not support " + "Responses executors; use format 'openai' or 'anthropic'" + ) + raise ValueError( + f"advisor executor format {fmt!r} must be resolved before constructing " + "AdvisorToolCallBackend (pin format: 'openai' or 'anthropic' when " + "supplying executor_backend)" + ) class AdvisorToolCallBackend(LLMBackend): - """Executor backend offering a proxy-intercepted ``advisor`` tool (native Anthropic).""" + """Executor backend offering a proxy-intercepted ``advisor`` tool (any Chat wire).""" #: Absolute backstop on executor calls per request. ``max_uses`` already #: bounds consults; this bounds an executor that keeps calling the advisor @@ -147,8 +456,19 @@ def __init__( self._config = config self._stats = stats_accumulator if config.enable_stats else None self._translation = TranslationEngine() - # The executor is delegated to verbatim so cache_control passes through. - self._executor_backend = executor_backend or build_native_backend(config.executor) + # Resolve format: auto before dialect selection; injected fakes must + # pin a concrete format (probing a fake's endpoint makes no sense). + executor_target = ( + config.executor if executor_backend is not None + else resolve_llm_target(config.executor) + ) + self._dialect = _dialect_for_format(executor_target.format) + self._request_type = cast( + "ChatRequestType", request_type_enum(self._dialect.request_type), + ) + # The executor is delegated to verbatim so caching survives + # (cache_control breakpoints on Anthropic; prefix stability on OpenAI). + self._executor_backend = executor_backend or build_native_backend(executor_target) self._advisor_caller = advisor_caller or _build_advisor_caller(config) async def startup(self) -> None: @@ -159,37 +479,47 @@ async def shutdown(self) -> None: @property def supported_request_types(self) -> list[ChatRequestType]: - """Executor + advisor are native Anthropic Messages.""" - return [ChatRequestType.ANTHROPIC] + """The executor's native wire; inbound formats are normalized to it.""" + return [self._request_type] async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: normalized = self._translation.request_to_any_of( request, self.supported_request_types, ) - if not request_type_matches(normalized, ChatRequestType.ANTHROPIC): + if not request_type_matches(normalized, self._request_type): raise TypeError( - "AdvisorToolCallBackend expected an Anthropic request after translation" + "AdvisorToolCallBackend expected a " + f"{self._dialect.request_type} request after translation" ) body = dict(normalized.body) messages: list[dict[str, Any]] = list(body.get("messages") or []) base_tools: list[dict[str, Any]] = list(body.get("tools") or []) if self._config.inject_steering: - body["system"] = _prepend_system(body.get("system"), self._config.executor_steering) - messages = _with_length_line(messages, self._config.advisor_length_line) - tools = [*base_tools, self._advisor_tool_def()] + body, messages = self._dialect.inject_steering( + body, + messages, + steering=self._config.executor_steering, + length_line=self._config.advisor_length_line, + ) + tools = [ + *base_tools, + self._dialect.advisor_tool_def( + name=self._config.advisor_tool_name, + description=self._config.advisor_tool_description, + ), + ] + tool_summaries = self._dialect.tool_summaries(base_tools) advisor_uses = 0 turn: _ToolCallTurn | None = None for _ in range(self._HARD_ITERATION_CAP): turn_body = {**body, "messages": messages, "tools": tools} - turn = await self._run_executor(ctx, request_with_type("anthropic", turn_body)) + turn = await self._run_executor( + ctx, request_with_type(self._dialect.request_type, turn_body), + ) - advisor_calls = [ - b for b in turn.blocks - if b.get("type") == "tool_use" and b.get("name") == self._config.advisor_tool_name - ] - if not advisor_calls: + if not turn.advisor_calls: # Real tool call(s) for the client, or a final answer. return await self._finish(ctx, turn) @@ -199,22 +529,11 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: if advisor_uses < self._config.max_uses: advisor_uses += 1 - advice = await self._consult_advisor(messages, turn.text, base_tools) + advice = await self._consult_advisor(messages, turn.text, tool_summaries) else: advice = _MAX_USES_RESULT - # Rebuild the assistant turn keeping thinking/text and ONLY the - # advisor tool_use blocks; sibling client calls are re-issued - # (advice-informed) on the next iteration. Thinking blocks must - # round-trip verbatim for upstreams that verify signatures. - messages = [ - *messages, - {"role": "assistant", "content": _advisor_only_content(turn.blocks, advisor_calls)}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": call.get("id"), "content": advice} - for call in advisor_calls - ]}, - ] + messages = [*messages, *self._dialect.feedback_messages(turn, advice)] log.warning( "AdvisorToolCallBackend: hit hard iteration cap (%d) without a terminal " @@ -230,7 +549,7 @@ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: # ------------------------------------------------------------------ async def _run_executor(self, ctx: ProxyContext, request: ChatRequest) -> _ToolCallTurn: - """Call the executor, buffering its response and reassembling content blocks.""" + """Call the executor, buffering its response and reassembling the turn.""" started = time.monotonic() try: response = await self._executor_backend.call(ctx, request) @@ -241,22 +560,10 @@ async def _run_executor(self, ctx: ProxyContext, request: ChatRequest) -> _ToolC raise latency_ms = (time.monotonic() - started) * 1000.0 - if response.response_type == ChatResponseType.ANTHROPIC_STREAM: - events, blocks, usage = await _consume_stream(response.stream) - return _ToolCallTurn( - blocks=blocks, latency_ms=latency_ms, stream_events=events, **usage, - ) - completion: Any = response.to_body() - blocks = [b for b in (completion.get("content") or []) if isinstance(b, dict)] - prompt_tokens, completion_tokens = _usage_tokens(completion.get("usage")) - cached = (completion.get("usage") or {}).get("cache_read_input_tokens") or 0 - return _ToolCallTurn( - blocks=blocks, + return await self._dialect.parse_turn( + response, + advisor_tool_name=self._config.advisor_tool_name, latency_ms=latency_ms, - input_tokens=prompt_tokens or 0, - output_tokens=completion_tokens or 0, - cached_tokens=int(cached), - completion_body=completion, ) async def _finish(self, ctx: ProxyContext, turn: _ToolCallTurn) -> ChatResponse: @@ -265,11 +572,7 @@ async def _finish(self, ctx: ProxyContext, turn: _ToolCallTurn) -> ChatResponse: ctx.backend_call_latency_ms = turn.latency_ms if self._stats is not None: await self._stats.record_success(self._config.executor.model, turn.latency_ms) - if turn.stream_events is not None: - return ChatResponse.anthropic_stream( - AnthropicResponseStream(_replay_events(turn.stream_events)) - ) - return ChatResponse.anthropic_completion(turn.completion_body) + return self._dialect.replay(turn) async def _record_internal_turn(self, turn: _ToolCallTurn) -> None: """Price a proxy-internal executor turn into the planner bucket.""" @@ -291,7 +594,7 @@ async def _consult_advisor( self, messages: list[dict[str, Any]], current_turn_text: str, - tools: list[dict[str, Any]], + tool_summaries: list[tuple[str, str]], ) -> str: """Consult the advisor on the transcript and return its guidance. @@ -299,7 +602,7 @@ async def _consult_advisor( so the executor can proceed; the failed call still counts toward ``max_uses`` at the call site, bounding retries against a down advisor. """ - transcript = self._serialize_transcript(messages, current_turn_text, tools) + transcript = self._serialize_transcript(messages, current_turn_text, tool_summaries) started = time.monotonic() try: advice, usage = await self._advisor_caller.advise( @@ -334,7 +637,7 @@ def _serialize_transcript( self, messages: list[dict[str, Any]], current_turn_text: str, - tools: list[dict[str, Any]], + tool_summaries: list[tuple[str, str]], ) -> str: """Serialize the conversation for the advisor, newest turns kept first. @@ -346,10 +649,10 @@ def _serialize_transcript( consult is about. """ sections: list[str] = [] - if tools: + if tool_summaries: summary = "\n".join( - f"- {t.get('name', '?')}: {str(t.get('description', ''))[:_TOOL_SUMMARY_DESC_CHARS]}" - for t in tools + f"- {name}: {desc[:_TOOL_SUMMARY_DESC_CHARS]}" + for name, desc in tool_summaries ) sections.append(f"Tools available to the executor:\n{summary}") @@ -377,25 +680,9 @@ def _serialize_transcript( ) return "\n\n".join(sections) - # ------------------------------------------------------------------ - # Request shaping - # ------------------------------------------------------------------ - - def _advisor_tool_def(self) -> dict[str, Any]: - """The synthetic, parameterless ``advisor`` tool (Anthropic shape).""" - return { - "name": self._config.advisor_tool_name, - "description": self._config.advisor_tool_description, - "input_schema": { - "type": "object", - "properties": {}, - "additionalProperties": False, - }, - } - # ---------------------------------------------------------------------- -# Module-level helpers +# Anthropic-wire helpers # ---------------------------------------------------------------------- @@ -482,7 +769,8 @@ def _with_length_line( injection, so re-injecting into each turn's newest message would shift the upstream cache prefix every turn. The first user message is constant across a session, keeping the prefix stable; the advisor still reads the line via - the forwarded transcript. + the forwarded transcript. The list branch emits ``{"type": "text", ...}`` + parts, valid on both the Anthropic and OpenAI wires. """ msgs = [dict(m) for m in messages] for msg in msgs: @@ -497,6 +785,139 @@ def _with_length_line( return msgs +# ---------------------------------------------------------------------- +# OpenAI-wire helpers +# ---------------------------------------------------------------------- + + +async def _consume_openai_stream( + stream: Any, +) -> tuple[list[Any], dict[str, Any], dict[str, int]]: + """Buffer an OpenAI Chat stream; reassemble the assistant message and usage. + + Events are the ``chat.completion.chunk`` dicts the native backend's SSE + parser yields (``[DONE]`` is consumed upstream and never appears; the + backend force-injects ``stream_options.include_usage`` so a final usage + chunk normally arrives). ``delta.tool_calls`` fragments merge by ``index``: + non-empty ``id``/``name`` replace, ``arguments`` fragments concatenate. + """ + events: list[Any] = [] + text_parts: list[str] = [] + slots: dict[int, dict[str, str]] = {} + usage = {"input_tokens": 0, "output_tokens": 0, "cached_tokens": 0} + async for event in stream: + events.append(event) + chunk_usage = _ev(event, "usage") + if isinstance(chunk_usage, dict): + usage["input_tokens"] = int(chunk_usage.get("prompt_tokens") or 0) + usage["output_tokens"] = int(chunk_usage.get("completion_tokens") or 0) + details = chunk_usage.get("prompt_tokens_details") or {} + usage["cached_tokens"] = int(details.get("cached_tokens") or 0) + choices = _ev(event, "choices") or [] + delta = _ev(choices[0], "delta") if choices else None + if delta is None: + continue + piece = _ev(delta, "content") + if isinstance(piece, str): + text_parts.append(piece) + for fragment in _ev(delta, "tool_calls") or []: + index = int(_ev(fragment, "index") or 0) + slot = slots.setdefault(index, {"id": "", "name": "", "arguments": ""}) + fragment_id = _ev(fragment, "id") + if isinstance(fragment_id, str) and fragment_id: + slot["id"] = fragment_id + function = _ev(fragment, "function") or {} + name = _ev(function, "name") + if isinstance(name, str) and name: + slot["name"] = name + arguments = _ev(function, "arguments") + if isinstance(arguments, str): + slot["arguments"] += arguments + + message: dict[str, Any] = { + "role": "assistant", + "content": "".join(text_parts) or None, + } + if slots: + message["tool_calls"] = [ + { + # A missing id (some OSS servers omit it in deltas) gets a + # synthesized one so the tool result can reference it. + "id": slot["id"] or f"call_switchyard_{index}", + "type": "function", + "function": { + "name": slot["name"], + # Empty arguments become "{}" so strict endpoints accept + # the replayed history. + "arguments": slot["arguments"] or "{}", + }, + } + for index, slot in sorted(slots.items()) + ] + return events, message, usage + + +def _openai_advisor_calls( + message: dict[str, Any], tool_name: str, +) -> list[dict[str, Any]]: + """The message's advisor ``tool_calls``, detected by function name. + + Detection is by tool-call presence, never ``finish_reason`` — some OSS + servers mislabel tool-call turns as ``stop``. + """ + return [ + tc for tc in (message.get("tool_calls") or []) + if isinstance(tc, dict) + and (tc.get("function") or {}).get("name") == tool_name + ] + + +def _advisor_only_message( + message: dict[str, Any], kept_ids: set[Any], +) -> dict[str, Any]: + """The assistant message with sibling (non-advisor) tool calls dropped. + + Key-whitelisted on purpose: vendor fields such as ``reasoning_content`` + are dropped so strict endpoints (OpenAI proper) accept the round-trip; + the advisor already saw the turn's text via the transcript. + """ + return { + "role": "assistant", + "content": message.get("content"), + "tool_calls": [ + tc for tc in (message.get("tool_calls") or []) + if isinstance(tc, dict) and tc.get("id") in kept_ids + ], + } + + +def _prepend_system_message( + messages: list[dict[str, Any]], prefix: str, +) -> list[dict[str, Any]]: + """Prepend steering to the first ``system``/``developer`` message. + + When the request carries no system-role message at all, one is inserted at + index 0 so the steering still leads the prompt (and the prefix cache stays + stable across the session's turns). + """ + msgs = [dict(m) for m in messages] + for msg in msgs: + if msg.get("role") not in ("system", "developer"): + continue + content = msg.get("content") + if isinstance(content, list): + msg["content"] = [{"type": "text", "text": prefix}, *content] + else: + msg["content"] = f"{prefix}\n\n{content or ''}".rstrip() + return msgs + return [{"role": "system", "content": prefix}, *msgs] + + +# ---------------------------------------------------------------------- +# Audit +# ---------------------------------------------------------------------- + + def _audit_advisor(*, error: str | None, usage: Any, latency_ms: float) -> None: """Emit a one-line ``advisor_call=...`` audit record to stderr.""" payload: dict[str, Any] = { diff --git a/switchyard/lib/profiles/advisor.py b/switchyard/lib/profiles/advisor.py index dacc13c3..7aad4139 100644 --- a/switchyard/lib/profiles/advisor.py +++ b/switchyard/lib/profiles/advisor.py @@ -13,12 +13,15 @@ ``"review_gate"`` consults the advisor once per session at the executor's first no-tool-call turn (:class:`~switchyard.lib.backends.advisor_loop_backend.AdvisorLoopBackend`). -Both delegate the executor call to a native Anthropic backend (caching intact), -stamp ``ctx.selected_model``, and record the advisor's usage themselves; they -cannot be wrapped by ``StatsLlmBackend``, so the runtime attaches the shared -accumulator through the ``_stats`` compatibility hook — mirroring -``LatencyServiceLLMBackend``. Serving-level stats processors arrive via -``with_runtime_components``, like every other profile. +The executor call is delegated verbatim to the native backend for +``executor.format`` — Anthropic keeps ``cache_control`` intact; OpenAI keeps +the Chat body intact — and the advisor caller likewise dispatches on +``advisor.format``. Both backends stamp ``ctx.selected_model`` and record the +advisor's usage themselves; they cannot be wrapped by ``StatsLlmBackend``, so +the runtime attaches the shared accumulator through the ``_stats`` +compatibility hook — mirroring ``LatencyServiceLLMBackend``. Serving-level +stats processors arrive via ``with_runtime_components``, like every other +profile. """ from __future__ import annotations diff --git a/switchyard/lib/profiles/advisor_config.py b/switchyard/lib/profiles/advisor_config.py index 017c5bb7..46733f1f 100644 --- a/switchyard/lib/profiles/advisor_config.py +++ b/switchyard/lib/profiles/advisor_config.py @@ -19,18 +19,29 @@ it back (REDO) with an optimized plan. See ``switchyard/lib/backends/advisor_loop_backend.py``. -Both tiers are ordinary targets, served native Anthropic-Messages (Opus 4.7 -executor + Opus 4.8 advisor on NVIDIA Inference Hub) — no OpenAI translation, so -the client's prompt caching survives. +Both tiers are ordinary targets; each tier's ``format`` selects its wire +independently. ``anthropic`` targets are served native Anthropic-Messages with +the body passed through verbatim (the client's prompt caching survives); +``openai`` targets (Qwen, DeepSeek, vLLM/NIM, OpenAI) are served OpenAI Chat +Completions, likewise verbatim. Tiers mix freely under ``tool_call``; +``review_gate`` supports native-Anthropic executors only, and ``responses`` +targets are rejected (the advisor loop is Chat-shaped). """ from __future__ import annotations -from typing import Literal +from typing import Literal, Self -from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) -from switchyard.lib.backends.llm_target import LlmTarget, coerce_llm_target +from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget, coerce_llm_target from switchyard.lib.profiles.advisor_prompts import ( ADVISOR_LENGTH_LINE, ADVISOR_SYSTEM_PROMPT, @@ -130,5 +141,30 @@ def _target_model_non_empty(cls, tier: LlmTarget) -> LlmTarget: raise ValueError("target.model must be a non-empty string") return tier + @field_validator("executor", "advisor") + @classmethod + def _target_format_supported(cls, tier: LlmTarget, info: ValidationInfo) -> LlmTarget: + if tier.format == BackendFormat.RESPONSES: + raise ValueError( + f"{info.field_name}.format 'responses' is not supported by the advisor " + "profile (the loop is Chat-shaped); use 'openai' or 'anthropic'" + ) + return tier + + @model_validator(mode="after") + def _review_gate_requires_anthropic_executor(self) -> Self: + # AUTO is allowed here: it resolves to a concrete format at build time, + # where AdvisorLoopBackend's Anthropic-only wiring is the backstop. + if self.strategy == "review_gate" and self.executor.format not in ( + BackendFormat.ANTHROPIC, + BackendFormat.AUTO, + ): + raise ValueError( + "strategy 'review_gate' supports only native-Anthropic executors " + "(set executor.format: anthropic); use strategy 'tool_call' for " + "OpenAI-compatible executors" + ) + return self + __all__ = ["AdvisorConfig"] diff --git a/tests/test_advisor_loop_backend.py b/tests/test_advisor_loop_backend.py index 0e8b1485..0fb128e8 100644 --- a/tests/test_advisor_loop_backend.py +++ b/tests/test_advisor_loop_backend.py @@ -25,9 +25,12 @@ _AnthropicAdvisorCaller, _build_advisor_caller, _messages_url, + _OpenAiAdvisorCaller, _parse_verdict, _session_key, + _usage_tokens, ) +from switchyard.lib.backends.llm_target import coerce_llm_target from switchyard.lib.chat_response.anthropic import AnthropicResponseStream from switchyard.lib.profiles.advisor_config import AdvisorConfig from switchyard.lib.proxy_context import ProxyContext @@ -274,6 +277,46 @@ def test_build_advisor_caller_is_anthropic() -> None: assert isinstance(_build_advisor_caller(_config()), _AnthropicAdvisorCaller) +def test_build_advisor_caller_dispatches_openai() -> None: + config = _config(advisor={"model": "deepseek/deepseek-r2", "base_url": "http://adv.test", + "api_key": "k", "format": "openai"}) + assert isinstance(_build_advisor_caller(config), _OpenAiAdvisorCaller) + + +@respx.mock +async def test_openai_advisor_hits_chat_completions_endpoint() -> None: + route = respx.post("https://adv.example/v1/chat/completions").mock( + return_value=httpx.Response(200, json={ + "id": "chatcmpl-x", "object": "chat.completion", + "choices": [{"index": 0, "finish_reason": "stop", + "message": {"role": "assistant", "content": " use a heap "}}], + "usage": {"prompt_tokens": 42, "completion_tokens": 6}, + }) + ) + target = coerce_llm_target({ + "model": "deepseek/deepseek-r2", "base_url": "https://adv.example/v1", + "api_key": "secret-key", "format": "openai", + "extra_headers": {"X-Gateway": "test"}, + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + }, default_id="advisor") + caller = _OpenAiAdvisorCaller(target=target, max_tokens=256, temperature=None) + + text, usage = await caller.advise(system="advise this", transcript="conversation") + + assert text == "use a heap" + prompt_tokens, completion_tokens = _usage_tokens(usage) + assert (prompt_tokens, completion_tokens) == (42, 6) + request = route.calls.last.request + assert request.headers["authorization"] == "Bearer secret-key" + assert request.headers["x-gateway"] == "test" + body = json.loads(request.content) + assert body["messages"][0] == {"role": "system", "content": "advise this"} + assert body["messages"][1] == {"role": "user", "content": "conversation"} + assert body["max_tokens"] == 256 + assert "temperature" not in body + assert body["chat_template_kwargs"] == {"enable_thinking": False} + + @respx.mock async def test_anthropic_reviewer_hits_messages_endpoint() -> None: route = respx.post("https://inference-api.nvidia.com/v1/messages").mock( diff --git a/tests/test_advisor_profile.py b/tests/test_advisor_profile.py index b12d85be..fcdbd18c 100644 --- a/tests/test_advisor_profile.py +++ b/tests/test_advisor_profile.py @@ -27,12 +27,28 @@ ) from switchyard.lib.profiles.table import profile_config_type from switchyard.lib.stats_accumulator import StatsAccumulator +from switchyard_rust.core import ChatRequestType def _config(**overrides) -> AdvisorConfig: + # Formats pinned: an omitted format coerces to openai, which the + # review_gate validator rejects for the executor tier. base: dict = { - "executor": {"model": "exec-model", "base_url": "http://exec.test", "api_key": "k"}, - "advisor": {"model": "adv-model", "base_url": "http://adv.test", "api_key": "k"}, + "executor": {"model": "exec-model", "base_url": "http://exec.test", "api_key": "k", + "format": "anthropic"}, + "advisor": {"model": "adv-model", "base_url": "http://adv.test", "api_key": "k", + "format": "anthropic"}, + } + base.update(overrides) + return AdvisorConfig(**base) + + +def _openai_config(**overrides) -> AdvisorConfig: + base: dict = { + "executor": {"model": "qwen/qwen3-max", "base_url": "http://exec.test", + "api_key": "k", "format": "openai"}, + "advisor": {"model": "deepseek/deepseek-r2", "base_url": "http://adv.test", + "api_key": "k", "format": "openai"}, } base.update(overrides) return AdvisorConfig(**base) @@ -102,6 +118,20 @@ def test_runtime_attaches_accumulator_to_backend(self) -> None: ) assert backend._stats is stats + def test_openai_tiers_build_tool_call_backend_on_openai_wire(self) -> None: + backend = next( + c for c in _advisor_switchyard(_openai_config()).iter_components() + if isinstance(c, AdvisorToolCallBackend) + ) + assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT] + + def test_anthropic_executor_advertises_anthropic_wire(self) -> None: + backend = next( + c for c in _advisor_switchyard(_config()).iter_components() + if isinstance(c, AdvisorToolCallBackend) + ) + assert backend.supported_request_types == [ChatRequestType.ANTHROPIC] + class TestAdvisorConfig: def test_coerces_dict_targets(self) -> None: @@ -119,6 +149,23 @@ def test_rejects_unknown_strategy(self) -> None: with pytest.raises(pydantic.ValidationError): _config(strategy="both") + def test_rejects_responses_format_on_either_tier(self) -> None: + for tier in ("executor", "advisor"): + with pytest.raises(pydantic.ValidationError, match="responses"): + _config(**{tier: {"model": "m", "base_url": "http://t", "api_key": "k", + "format": "responses"}}) + + def test_review_gate_rejects_openai_executor(self) -> None: + with pytest.raises(pydantic.ValidationError, match="review_gate"): + _openai_config(strategy="review_gate") + + def test_tool_call_accepts_mixed_and_openai_tiers(self) -> None: + assert _openai_config().strategy == "tool_call" + mixed = _config(advisor={"model": "deepseek/deepseek-r2", "base_url": "http://adv.test", + "api_key": "k", "format": "openai"}) + assert mixed.advisor.format == BackendFormat.OPENAI + assert mixed.executor.format == BackendFormat.ANTHROPIC + class TestOpusPairPreset: """Pins the validated executor+advisor pairing on the shipping default.""" diff --git a/tests/test_advisor_tool_call_backend_openai.py b/tests/test_advisor_tool_call_backend_openai.py new file mode 100644 index 00000000..0ec1cd5a --- /dev/null +++ b/tests/test_advisor_tool_call_backend_openai.py @@ -0,0 +1,523 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for :class:`AdvisorToolCallBackend` on the OpenAI-Chat wire. + +Mirror of ``test_advisor_tool_call_backend.py`` (the Anthropic suite) with +OpenAI-format executors: fixtures are plain ``chat.completion`` bodies and +``chat.completion.chunk`` dicts, matching what ``OpenAiNativeBackend``'s SSE +parser yields. Covers the OpenAI-specific shapes — function-tool injection, +system-message steering, ``tool_calls`` interception, ``role:"tool"`` +feedback with 1:1 ids, delta reassembly + final-usage chunk, sibling drop +with vendor-field whitelisting — plus the format-dispatch seams (mixed tiers, +rejected formats). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from switchyard.lib.backends.advisor_tool_call_backend import ( + AdvisorToolCallBackend, + _prepend_system_message, +) +from switchyard.lib.chat_response.openai_chat import ResponseStream +from switchyard.lib.profiles.advisor_config import AdvisorConfig +from switchyard.lib.proxy_context import ProxyContext +from switchyard_rust.core import ChatRequest, ChatRequestType, ChatResponse, ChatResponseType + +# --------------------------------------------------------------------------- +# Fakes +# --------------------------------------------------------------------------- + + +def _completion_resp( + message: dict, *, finish_reason: str = "stop", usage: dict | None = None, +) -> ChatResponse: + """An OpenAI chat.completion ChatResponse with the given assistant message.""" + body = { + "id": "chatcmpl-x", "object": "chat.completion", "model": "exec-model", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": usage if usage is not None else { + "prompt_tokens": 10, + "completion_tokens": 3, + "prompt_tokens_details": {"cached_tokens": 4}, + }, + } + return ChatResponse.openai_completion(body) + + +def _text_turn(text: str = "done") -> ChatResponse: + return _completion_resp({"role": "assistant", "content": text}) + + +def _advisor_call(call_id: str = "adv1", *, arguments: str = "{}") -> dict: + return {"id": call_id, "type": "function", + "function": {"name": "advisor", "arguments": arguments}} + + +def _advisor_turn( + *, extra_calls: list[dict] | None = None, finish_reason: str = "tool_calls", + arguments: str = "{}", +) -> ChatResponse: + """A completion turn calling the advisor (optionally with sibling calls).""" + message = { + "role": "assistant", + "content": "let me ask", + "reasoning_content": "hidden chain of thought", + "tool_calls": [*(extra_calls or []), _advisor_call(arguments=arguments)], + } + return _completion_resp(message, finish_reason=finish_reason) + + +async def _agen(events): + for event in events: + yield event + + +def _stream_resp(events: list[dict]) -> ChatResponse: + return ChatResponse.openai_stream(ResponseStream(_agen(events))) + + +def _chunk(delta: dict, *, finish_reason: str | None = None) -> dict: + return { + "id": "chatcmpl-x", "object": "chat.completion.chunk", "model": "exec-model", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +def _usage_chunk(*, prompt: int = 12, completion: int = 7, cached: int = 5) -> dict: + """The final usage-only chunk stream_options.include_usage produces.""" + return { + "id": "chatcmpl-x", "object": "chat.completion.chunk", "model": "exec-model", + "choices": [], + "usage": {"prompt_tokens": prompt, "completion_tokens": completion, + "prompt_tokens_details": {"cached_tokens": cached}}, + } + + +def _advisor_stream_events() -> list[dict]: + """Chunks for a streamed advisor call: id/name first, arguments in fragments.""" + return [ + _chunk({"role": "assistant", "content": "consul"}), + _chunk({"content": "ting"}), + _chunk({"tool_calls": [{"index": 0, "id": "adv-s", "type": "function", + "function": {"name": "advisor", "arguments": ""}}]}), + _chunk({"tool_calls": [{"index": 0, "function": {"arguments": "{"}}]}), + _chunk({"tool_calls": [{"index": 0, "function": {"arguments": "}"}}]}), + _chunk({}, finish_reason="tool_calls"), + _usage_chunk(), + ] + + +def _terminal_stream_events() -> list[dict]: + return [ + _chunk({"role": "assistant", "content": "final"}), + _chunk({"content": " answer"}), + _chunk({}, finish_reason="stop"), + _usage_chunk(prompt=20, completion=5, cached=0), + ] + + +def _exec_backend(*responses) -> MagicMock: + b = MagicMock() + b.call = AsyncMock(side_effect=list(responses)) + b.startup = AsyncMock() + b.shutdown = AsyncMock() + return b + + +def _advisor(*advice: str) -> MagicMock: + c = MagicMock() + c.advise = AsyncMock(side_effect=[(a, {"prompt_tokens": 100, "completion_tokens": 9}) + for a in advice]) + return c + + +def _failing_advisor(exc: Exception) -> MagicMock: + c = MagicMock() + c.advise = AsyncMock(side_effect=exc) + return c + + +def _config(**overrides) -> AdvisorConfig: + base: dict = { + "executor": {"model": "exec-model", "base_url": "http://exec.test", "api_key": "k", + "format": "openai"}, + "advisor": {"model": "adv-model", "base_url": "http://adv.test", "api_key": "k", + "format": "openai"}, + "executor_steering": "STEER", + "advisor_length_line": "(LENGTH)", + } + base.update(overrides) + return AdvisorConfig(**base) + + +def _backend(config, executor_backend, advisor_caller) -> AdvisorToolCallBackend: + return AdvisorToolCallBackend( + config, executor_backend=executor_backend, advisor_caller=advisor_caller, + ) + + +def _request(**overrides) -> ChatRequest: + body: dict = { + "model": "incoming", + "messages": [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "build X"}, + ], + "tools": [{"type": "function", + "function": {"name": "bash", "description": "Run a shell command", + "parameters": {"type": "object"}}}], + } + body.update(overrides) + return ChatRequest.openai_chat(body) # type: ignore[arg-type] + + +def _sent_body(exec_b: MagicMock, call_index: int) -> dict: + return exec_b.call.await_args_list[call_index].args[1].to_body() + + +def _content_text(body: dict) -> str: + return str(body["choices"][0]["message"].get("content") or "") + + +# --------------------------------------------------------------------------- +# Request shaping: tool injection + steering +# --------------------------------------------------------------------------- + + +async def test_advisor_tool_appended_in_function_shape() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(), exec_b, _advisor()).call(ProxyContext(), _request()) + tools = _sent_body(exec_b, 0)["tools"] + assert [t["function"]["name"] for t in tools] == ["bash", "advisor"] + assert tools[-1]["type"] == "function" + assert tools[-1]["function"]["parameters"]["properties"] == {} # parameterless + + +async def test_steering_prepends_system_message_and_appends_length_line() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(), exec_b, _advisor()).call(ProxyContext(), _request()) + messages = _sent_body(exec_b, 0)["messages"] + assert messages[0] == {"role": "system", "content": "STEER\n\nsys"} + assert messages[1]["content"] == "build X\n\n(LENGTH)" + + +async def test_steering_inserts_system_message_when_absent() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(), exec_b, _advisor()).call( + ProxyContext(), + _request(messages=[{"role": "user", "content": "build X"}]), + ) + messages = _sent_body(exec_b, 0)["messages"] + assert messages[0] == {"role": "system", "content": "STEER"} + assert messages[1]["content"] == "build X\n\n(LENGTH)" + + +async def test_steering_disabled_leaves_request_untouched() -> None: + exec_b = _exec_backend(_text_turn()) + await _backend(_config(inject_steering=False), exec_b, _advisor()).call( + ProxyContext(), _request(), + ) + body = _sent_body(exec_b, 0) + assert body["messages"][0]["content"] == "sys" + assert body["messages"][1]["content"] == "build X" + assert [t["function"]["name"] for t in body["tools"]] == ["bash", "advisor"] + + +def test_prepend_system_message_shapes() -> None: + assert _prepend_system_message([], "S") == [{"role": "system", "content": "S"}] + assert _prepend_system_message( + [{"role": "developer", "content": "base"}], "S", + ) == [{"role": "developer", "content": "S\n\nbase"}] + assert _prepend_system_message( + [{"role": "system", "content": [{"type": "text", "text": "base"}]}], "S", + ) == [{"role": "system", "content": [{"type": "text", "text": "S"}, + {"type": "text", "text": "base"}]}] + + +# --------------------------------------------------------------------------- +# Loop behavior +# --------------------------------------------------------------------------- + + +async def test_advisor_free_turn_returns_without_consult() -> None: + exec_b = _exec_backend(_text_turn("done, all good")) + adv = _advisor() + ctx = ProxyContext() + resp = await _backend(_config(), exec_b, adv).call(ctx, _request()) + assert _content_text(resp.to_body()) == "done, all good" + assert exec_b.call.await_count == 1 + assert adv.advise.await_count == 0 + assert ctx.selected_model == "exec-model" + + +async def test_client_tool_call_turn_returns_without_consult() -> None: + """A turn calling only the client's own tools is terminal — hand it back.""" + exec_b = _exec_backend(_completion_resp( + {"role": "assistant", "content": None, + "tool_calls": [{"id": "b1", "type": "function", + "function": {"name": "bash", "arguments": "{\"command\": \"ls\"}"}}]}, + finish_reason="tool_calls", + )) + adv = _advisor() + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert resp.to_body()["choices"][0]["finish_reason"] == "tool_calls" + assert adv.advise.await_count == 0 + + +async def test_advisor_call_is_intercepted_and_fed_back() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn("informed answer")) + adv = _advisor("try the channel-based pattern") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + assert exec_b.call.await_count == 2 + messages = _sent_body(exec_b, 1)["messages"] + assistant_turn = messages[-2] + assert assistant_turn["role"] == "assistant" + assert [tc["function"]["name"] for tc in assistant_turn["tool_calls"]] == ["advisor"] + result_turn = messages[-1] + assert result_turn == { + "role": "tool", "tool_call_id": "adv1", + "content": "try the channel-based pattern", + } + assert _content_text(resp.to_body()) == "informed answer" + + +async def test_advisor_detected_even_with_stop_finish_reason() -> None: + """Some OSS servers mislabel tool-call turns; detection is by presence.""" + exec_b = _exec_backend(_advisor_turn(finish_reason="stop"), _text_turn()) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert adv.advise.await_count == 1 + + +async def test_parallel_advisor_calls_one_consult_fanned_results() -> None: + """N advisor calls in one turn → one consult, one tool message per id.""" + message = { + "role": "assistant", "content": None, + "tool_calls": [_advisor_call("adv1"), _advisor_call("adv2")], + } + exec_b = _exec_backend( + _completion_resp(message, finish_reason="tool_calls"), _text_turn(), + ) + adv = _advisor("shared advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + messages = _sent_body(exec_b, 1)["messages"] + tool_msgs = [m for m in messages if m.get("role") == "tool"] + assert [m["tool_call_id"] for m in tool_msgs] == ["adv1", "adv2"] + assert all(m["content"] == "shared advice" for m in tool_msgs) + + +async def test_nonempty_arguments_round_trip_verbatim() -> None: + """Arguments are never parsed (the tool is parameterless) but round-trip.""" + exec_b = _exec_backend( + _advisor_turn(arguments='{"question": "which approach?"}'), _text_turn(), + ) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assistant_turn = _sent_body(exec_b, 1)["messages"][-2] + assert assistant_turn["tool_calls"][0]["function"]["arguments"] == ( + '{"question": "which approach?"}' + ) + + +async def test_transcript_carries_tools_conversation_and_current_turn() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn()) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + transcript = adv.advise.await_args.kwargs["transcript"] + assert "bash" in transcript # function-tool summary + assert "Run a shell command" in transcript + assert "build X" in transcript # conversation + assert "it is consulting you now" in transcript # current-turn section + assert "let me ask" in transcript # current-turn text + assert "hidden chain of thought" not in transcript # reasoning_content excluded + assert adv.advise.await_args.kwargs["system"] == _config().advisor_system_prompt + + +async def test_max_uses_returns_error_result_without_consult() -> None: + exec_b = _exec_backend(_advisor_turn(), _advisor_turn(), _text_turn()) + adv = _advisor("first advice") + resp = await _backend(_config(max_uses=1), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + assert exec_b.call.await_count == 3 + second_result = _sent_body(exec_b, 2)["messages"][-1] + assert second_result["content"] == "[advisor unavailable: max_uses exceeded]" + assert _content_text(resp.to_body()) == "done" + + +async def test_fail_open_marks_advisor_unavailable_and_continues() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn("proceeded unadvised")) + adv = _failing_advisor(RuntimeError("advisor down")) + resp = await _backend(_config(fail_open=True), exec_b, adv).call( + ProxyContext(), _request(), + ) + result = _sent_body(exec_b, 1)["messages"][-1] + assert result["content"] == "[advisor unavailable: RuntimeError]" + assert _content_text(resp.to_body()) == "proceeded unadvised" + + +async def test_fail_closed_raises() -> None: + exec_b = _exec_backend(_advisor_turn()) + adv = _failing_advisor(RuntimeError("advisor down")) + with pytest.raises(RuntimeError, match="advisor down"): + await _backend(_config(fail_open=False), exec_b, adv).call( + ProxyContext(), _request(), + ) + + +async def test_sibling_tool_calls_dropped_and_vendor_fields_whitelisted() -> None: + """Mixed advisor + client calls: only the advisor call is kept, and vendor + fields like reasoning_content are dropped from the rebuilt turn.""" + sibling = {"id": "b9", "type": "function", + "function": {"name": "bash", "arguments": "{\"command\": \"ls\"}"}} + exec_b = _exec_backend(_advisor_turn(extra_calls=[sibling]), _text_turn()) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assistant_turn = _sent_body(exec_b, 1)["messages"][-2] + assert [tc["id"] for tc in assistant_turn["tool_calls"]] == ["adv1"] + assert assistant_turn["content"] == "let me ask" # text survives + assert "reasoning_content" not in assistant_turn # vendor field dropped + # Every kept tool_call id has exactly one tool message (OpenAI 1:1 rule). + tool_msgs = [m for m in _sent_body(exec_b, 1)["messages"] if m.get("role") == "tool"] + assert [m["tool_call_id"] for m in tool_msgs] == ["adv1"] + + +async def test_hard_cap_returns_last_round() -> None: + exec_b = _exec_backend(*[_advisor_turn() for _ in range(8)]) + adv = _advisor(*["advice"] * 8) + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + assert exec_b.call.await_count == 8 + assert adv.advise.await_count == 2 # default max_uses=2; the rest get error results + assert resp.to_body()["choices"][0]["finish_reason"] == "tool_calls" + + +# --------------------------------------------------------------------------- +# Streaming +# --------------------------------------------------------------------------- + + +async def test_streaming_advisor_call_reassembled_then_terminal_replayed() -> None: + terminal_events = _terminal_stream_events() + exec_b = _exec_backend( + _stream_resp(_advisor_stream_events()), _stream_resp(terminal_events), + ) + adv = _advisor("advice") + resp = await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + + assert adv.advise.await_count == 1 + # Reassembled from deltas: the fed-back tool message targets the streamed + # id, the fragmented arguments merged, and the streamed text reached the + # transcript. + messages = _sent_body(exec_b, 1)["messages"] + assert messages[-1] == {"role": "tool", "tool_call_id": "adv-s", "content": "advice"} + assert messages[-2]["tool_calls"][0]["function"]["arguments"] == "{}" + assert "consulting" in adv.advise.await_args.kwargs["transcript"] + # The terminal turn's buffered events are replayed verbatim. + assert resp.response_type == ChatResponseType.OPENAI_STREAM + replayed = [event async for event in resp.stream] + assert replayed == terminal_events + + +async def test_streaming_missing_tool_call_id_is_synthesized() -> None: + events = [ + _chunk({"role": "assistant", + "tool_calls": [{"index": 0, + "function": {"name": "advisor", "arguments": ""}}]}), + _chunk({}, finish_reason="tool_calls"), + ] + exec_b = _exec_backend(_stream_resp(events), _text_turn()) + adv = _advisor("advice") + await _backend(_config(), exec_b, adv).call(ProxyContext(), _request()) + result = _sent_body(exec_b, 1)["messages"][-1] + assert result["tool_call_id"] == "call_switchyard_0" + + +# --------------------------------------------------------------------------- +# Stats accounting +# --------------------------------------------------------------------------- + + +async def test_planner_bucket_prices_internal_turns_and_consults() -> None: + exec_b = _exec_backend(_advisor_turn(), _text_turn()) + adv = _advisor("advice") + stats = MagicMock() + stats.record_planner_usage = AsyncMock() + stats.record_success = AsyncMock() + backend = AdvisorToolCallBackend( + _config(), stats_accumulator=stats, executor_backend=exec_b, advisor_caller=adv, + ) + await backend.call(ProxyContext(), _request()) + + assert stats.record_planner_usage.await_count == 2 + by_model = {c.kwargs["model"]: c.kwargs for c in stats.record_planner_usage.await_args_list} + # The intercepted executor turn is priced under the executor model, with + # OpenAI cached tokens read from prompt_tokens_details. + assert by_model["exec-model"]["prompt_tokens"] == 10 + assert by_model["exec-model"]["cached_tokens"] == 4 + assert by_model["adv-model"]["prompt_tokens"] == 100 + assert by_model["adv-model"]["completion_tokens"] == 9 + stats.record_success.assert_awaited_once() + + +async def test_missing_usage_records_zeros() -> None: + exec_b = _exec_backend( + _completion_resp( + {"role": "assistant", "content": None, "tool_calls": [_advisor_call()]}, + finish_reason="tool_calls", usage={}, + ), + _text_turn(), + ) + adv = _advisor("advice") + stats = MagicMock() + stats.record_planner_usage = AsyncMock() + stats.record_success = AsyncMock() + backend = AdvisorToolCallBackend( + _config(), stats_accumulator=stats, executor_backend=exec_b, advisor_caller=adv, + ) + await backend.call(ProxyContext(), _request()) + internal = stats.record_planner_usage.await_args_list[0].kwargs + assert internal["prompt_tokens"] == 0 + assert internal["cached_tokens"] == 0 + + +# --------------------------------------------------------------------------- +# Format dispatch +# --------------------------------------------------------------------------- + + +def test_openai_executor_advertises_openai_chat() -> None: + backend = _backend(_config(), _exec_backend(), _advisor()) + assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT] + + +def test_mixed_tiers_follow_executor_wire() -> None: + anthropic_advisor = {"model": "claude-opus-4-8", "base_url": "http://adv.test", + "api_key": "k", "format": "anthropic"} + backend = _backend( + _config(advisor=anthropic_advisor), _exec_backend(), _advisor(), + ) + assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT] + + openai_advisor = {"model": "deepseek/deepseek-r2", "base_url": "http://adv.test", + "api_key": "k", "format": "openai"} + anthropic_exec = {"model": "claude-opus-4-7", "base_url": "http://exec.test", + "api_key": "k", "format": "anthropic"} + backend = _backend( + _config(executor=anthropic_exec, advisor=openai_advisor), + _exec_backend(), _advisor(), + ) + assert backend.supported_request_types == [ChatRequestType.ANTHROPIC] + + +def test_auto_format_with_injected_backend_raises() -> None: + auto_exec = {"model": "m", "base_url": "http://exec.test", "api_key": "k", + "format": "auto"} + with pytest.raises(ValueError, match="pin format"): + _backend(_config(executor=auto_exec), _exec_backend(), _advisor()) diff --git a/tests/test_route_bundle.py b/tests/test_route_bundle.py index 425a45d2..9a198356 100644 --- a/tests/test_route_bundle.py +++ b/tests/test_route_bundle.py @@ -1462,6 +1462,45 @@ def test_enable_stats_false_disables_stats_processors(self): assert not any(isinstance(c, StatsRequestProcessor) for c in components) assert not any(isinstance(c, StatsResponseProcessor) for c in components) + def test_openai_tiers_build_on_openai_wire(self): + """OSS executor + advisor pairs (format: openai) are first-class.""" + from switchyard.lib.backends.advisor_tool_call_backend import ( + AdvisorToolCallBackend, + ) + from switchyard_rust.core import ChatRequestType + + bundle = { + "routes": { + "myrouter/advisor-oss": { + "type": "advisor", + "executor": { + "model": "qwen/qwen3-max", + "api_key": "sk-exec", + "base_url": "https://exec.invalid/v1", + "format": "openai", + }, + "advisor": { + "model": "deepseek/deepseek-r2", + "api_key": "sk-adv", + "base_url": "https://adv.invalid/v1", + "format": "openai", + }, + }, + }, + } + table = build_route_bundle_table(bundle) + backend = next( + c for c in table.iter_components() + if isinstance(c, AdvisorToolCallBackend) + ) + assert backend.supported_request_types == [ChatRequestType.OPENAI_CHAT] + + def test_responses_format_rejected(self): + bundle = self._bundle() + bundle["routes"]["myrouter/advisor"]["executor"]["format"] = "responses" + with pytest.raises(Exception, match="responses"): + build_route_bundle_table(bundle) + def test_stage_router_route_hydrates_tier_catalogs( monkeypatch: pytest.MonkeyPatch,