From 0685825775f2d4b275e6b2ddb86c595a593c597f Mon Sep 17 00:00:00 2001 From: Luke Alonso Date: Sat, 4 Jul 2026 12:54:54 -0700 Subject: [PATCH 1/2] Revert "fix(anthropic): demote interleaved system messages to user role" This reverts commit 1a3868c913f9db7d794d65023780d31f96e38976. --- src/adapters/anthropic_to_responses.rs | 343 +------------------------ tests/gateway.rs | 8 +- 2 files changed, 6 insertions(+), 345 deletions(-) diff --git a/src/adapters/anthropic_to_responses.rs b/src/adapters/anthropic_to_responses.rs index 0e3c01d..fa4ad80 100644 --- a/src/adapters/anthropic_to_responses.rs +++ b/src/adapters/anthropic_to_responses.rs @@ -3,7 +3,6 @@ use crate::error::AppResult; use crate::models::anthropic::AnthropicContent; use crate::models::anthropic::AnthropicContentBlock; use crate::models::anthropic::AnthropicImageSource; -use crate::models::anthropic::AnthropicMessage; use crate::models::anthropic::AnthropicRequest; use crate::models::anthropic::AnthropicSystemContent; use crate::models::anthropic::AnthropicThinking; @@ -30,17 +29,7 @@ pub fn convert_request(request: AnthropicRequest) -> AppResult )); } let instructions = extract_system_text(&request.system); - // Claude Code injects hook output + system-reminders as standalone - // `role:"system"` items scattered through `messages[]`. Interleaved system - // turns break OpenAI-style chat templates (DeepSeek-V4-Flash: tool calling - // stops, a raw `` leaks) because they expect ONE system message at - // position 0. Forward-merge them into the next user turn so the upstream - // sees a single leading system message (top-level `system` -> instructions). - // Forward-only (never backward) merge avoids rewriting already-sent prefix - // tokens, keeping KV-prefix growth append-only for the common case where a - // reminder precedes the next user prompt. - let messages = normalize_system_messages(&request.messages); - let input = convert_messages(&messages)?; + let input = convert_messages(&request.messages)?; let tools = convert_tools(&request.tools); let thinking_on = matches!( request.thinking.as_ref(), @@ -251,152 +240,6 @@ fn convert_messages( Ok(items) } -/// Forward-merge mid-conversation `system` messages into the next `user` -/// message so the upstream chat template sees a single leading system message. -/// -/// Claude Code injects hook output and system-reminders as standalone -/// `role:"system"` items scattered through `messages[]`. Most chat templates -/// (DeepSeek-V4-Flash included) only tolerate one system message at position 0; -/// interleaved system turns corrupt tool-call and `` token structure. -/// -/// Rules: -/// - Consecutive system messages are buffered and merged into the *next* real -/// user turn (forward merge -> stays inside the newly appended suffix, so the -/// already-cached KV prefix is not rewritten for the common case where a -/// reminder is injected right before the next user prompt). -/// - Tool-result user turns are NOT merge targets: an OpenAI tool message must -/// immediately follow its assistant `tool_calls`, so injected context is -/// carried *past* them (never wedged between a tool call and its result) and -/// attached to the next real user prompt or assistant turn instead. -/// - A system run that reaches an assistant turn (or end of transcript) with no -/// real user to merge into is emitted as a synthetic `user` note in place. -/// - Backward merge (into a prior user message) is deliberately avoided: it -/// would mutate already-sent prefix tokens and bust the vLLM prefix cache. -fn normalize_system_messages(messages: &[AnthropicMessage]) -> Vec { - let mut out: Vec = Vec::with_capacity(messages.len()); - let mut pending: Vec = Vec::new(); - for message in messages { - match message.role.as_str() { - "system" => { - let text = system_message_text(&message.content); - if !text.trim().is_empty() { - pending.push(text); - } - } - // Tool-result turns must stay adjacent to their assistant tool_call. - // Pass them through untouched and keep the reminder buffered so it - // lands *after* the tool exchange, not inside it. - "user" if message_has_tool_result(&message.content) => { - out.push(message.clone()); - } - "user" if !pending.is_empty() => { - out.push(prepend_injected_context(message, &pending)); - pending.clear(); - } - _ => { - if !pending.is_empty() { - // Assistant boundary (or a plain user turn with nothing - // pending falls through here harmlessly): keep the reminder - // at its position as a synthetic user turn. - out.push(synthetic_user_note(&pending)); - pending.clear(); - } - out.push(message.clone()); - } - } - } - if !pending.is_empty() { - // Trailing system run (or one carried past trailing tool results). - out.push(synthetic_user_note(&pending)); - } - out -} - -/// True when a user turn carries tool-result output (a replayed `tool_result` or -/// web-search result). Such turns must remain adjacent to the assistant -/// `tool_calls` they answer, so injected context is never merged into them. -fn message_has_tool_result(content: &AnthropicContent) -> bool { - match content { - AnthropicContent::Text(_) => false, - AnthropicContent::Blocks(blocks) => blocks.iter().any(|block| { - matches!( - block, - AnthropicContentBlock::ToolResult { .. } - | AnthropicContentBlock::WebSearchToolResult { .. } - ) - }), - } -} - -/// Concatenate the text blocks of a `system` message (reminders are text-only). -fn system_message_text(content: &AnthropicContent) -> String { - match content { - AnthropicContent::Text(text) => text.clone(), - AnthropicContent::Blocks(blocks) => blocks - .iter() - .filter_map(|block| match block { - AnthropicContentBlock::Text { text } => Some(text.as_str()), - _ => None, - }) - .collect::>() - .join("\n"), - } -} - -/// Prepend buffered injected-context to a user message as a leading text block. -fn prepend_injected_context(message: &AnthropicMessage, pending: &[String]) -> AnthropicMessage { - let mut blocks = match &message.content { - AnthropicContent::Text(text) => vec![AnthropicContentBlock::Text { text: text.clone() }], - AnthropicContent::Blocks(blocks) => blocks.clone(), - }; - blocks.insert( - 0, - AnthropicContentBlock::Text { - text: format_injected_context(pending), - }, - ); - AnthropicMessage { - role: "user".to_string(), - content: AnthropicContent::Blocks(blocks), - } -} - -/// Emit buffered injected-context as a standalone synthetic `user` turn. -fn synthetic_user_note(pending: &[String]) -> AnthropicMessage { - AnthropicMessage { - role: "user".to_string(), - content: AnthropicContent::Blocks(vec![AnthropicContentBlock::Text { - text: format_injected_context(pending), - }]), - } -} - -/// Render demoted system content as user-role text. -/// -/// Each reminder is wrapped PER-REMINDER in `` tags -- Claude -/// Code's own convention for harness-injected context, which the model already -/// sees inline in user turns -- so it reads as injected operational context, not -/// the user's own words (e.g. "CAVEMAN MODE ACTIVE", "tools haven't been used"). -/// Per-reminder (not one blob) preserves provenance boundaries between unrelated -/// injections. Already-wrapped payloads pass through unchanged (no double-wrap). -fn format_injected_context(pending: &[String]) -> String { - pending - .iter() - .filter_map(|text| { - let text = text.trim(); - if text.is_empty() { - None - } else if text.starts_with("") && text.ends_with("") - { - Some(text.to_string()) - } else { - Some(format!("\n{text}\n")) - } - }) - .collect::>() - .join("\n") -} - fn convert_message( role: &str, content: &AnthropicContent, @@ -860,12 +703,7 @@ mod tests { } #[test] - fn demotes_trailing_standalone_system_message_to_user() { - // A standalone `role:"system"` message in `messages[]` (CC hook/reminder - // injection) must NOT reach the upstream as a system-role turn -- that - // breaks DeepSeek's chat template. With no following user turn, it is - // emitted as a synthetic `user` note in place. Top-level `system` still - // maps to `instructions` (the single leading system message). + fn preserves_claude_code_system_history_role() { let skill_listing = concat!( "The following skills are available for use with the Skill tool:\n\n", "- deep-research: Deep research harness. Use when the user wants research.\n", @@ -906,181 +744,8 @@ mod tests { assert!(matches!( &result.input[1], ResponseItem::Message { role, content, .. } - if role == "user" - && matches!(&content[0], ContentItem::InputText { text } - if text.contains("deep-research") && text.contains("")) - )); - } - - #[test] - fn forward_merges_system_into_following_user() { - // The dominant CC pattern: a reminder is injected right before the next - // user prompt. Merge forward into that user turn so we get clean - // user/assistant alternation and stay inside the appended suffix - // (append-only -> KV prefix cache preserved). - let request = AnthropicRequest { - model: "claude-3-5-sonnet-20241022".to_string(), - max_tokens: Some(1024), - system: None, - messages: vec![ - AnthropicMessage { - role: "system".to_string(), - content: AnthropicContent::Text("REMINDER-TEXT".to_string()), - }, - AnthropicMessage { - role: "user".to_string(), - content: AnthropicContent::Text("real prompt".to_string()), - }, - ], - tools: None, - tool_choice: None, - stream: false, - temperature: None, - top_p: None, - top_k: None, - stop_sequences: None, - metadata: None, - thinking: None, - output_config: None, - }; - - let result = convert_request(request).expect("convert"); - // Single user turn: injected context prepended, real prompt after. - assert_eq!(result.input.len(), 1); - assert!(matches!( - &result.input[0], - ResponseItem::Message { role, content, .. } - if role == "user" - && matches!(&content[0], ContentItem::InputText { text } if text.contains("REMINDER-TEXT")) - && matches!(&content[1], ContentItem::InputText { text } if text == "real prompt") - )); - } - - #[test] - fn system_before_assistant_stays_in_place_as_user() { - // A system run followed by a non-user turn is NOT floated across the - // boundary (semantic drift) -- it becomes a synthetic user note in place, - // preserving user/assistant order. - let request = AnthropicRequest { - model: "claude-3-5-sonnet-20241022".to_string(), - max_tokens: Some(1024), - system: None, - messages: vec![ - AnthropicMessage { - role: "system".to_string(), - content: AnthropicContent::Text("SESSION-HOOK".to_string()), - }, - AnthropicMessage { - role: "assistant".to_string(), - content: AnthropicContent::Text("ok".to_string()), - }, - ], - tools: None, - tool_choice: None, - stream: false, - temperature: None, - top_p: None, - top_k: None, - stop_sequences: None, - metadata: None, - thinking: None, - output_config: None, - }; - - let result = convert_request(request).expect("convert"); - assert_eq!(result.input.len(), 2); - assert!(matches!( - &result.input[0], - ResponseItem::Message { role, content, .. } - if role == "user" - && matches!(&content[0], ContentItem::InputText { text } if text.contains("SESSION-HOOK")) - )); - assert!(matches!( - &result.input[1], - ResponseItem::Message { role, .. } if role == "assistant" - )); - } - - #[test] - fn format_injected_context_wraps_per_reminder_and_guards_double_wrap() { - // Raw payload -> wrapped. - assert_eq!( - format_injected_context(&["raw text".to_string()]), - "\nraw text\n" - ); - // Already-tagged payload -> passed through, not double-wrapped. - assert_eq!( - format_injected_context(&["hi".to_string()]), - "hi" - ); - // Multiple reminders -> per-reminder wrap, boundaries preserved. - let out = format_injected_context(&["one".to_string(), "two".to_string()]); - assert_eq!( - out, - "\none\n\n\ntwo\n" - ); - } - - #[test] - fn reminder_never_splits_assistant_tool_call_from_tool_result() { - // A reminder injected between an assistant tool_call and its tool_result - // must NOT be wedged between them (OpenAI requires the tool message to - // immediately follow the tool_calls). It is carried past the tool result - // and emitted afterward. - let request = AnthropicRequest { - model: "claude-3-5-sonnet-20241022".to_string(), - max_tokens: Some(1024), - system: None, - messages: vec![ - AnthropicMessage { - role: "assistant".to_string(), - content: AnthropicContent::Blocks(vec![AnthropicContentBlock::ToolUse { - id: "toolu_1".to_string(), - name: "get_weather".to_string(), - input: json!({"location": "Seattle"}), - }]), - }, - AnthropicMessage { - role: "system".to_string(), - content: AnthropicContent::Text("REMINDER-TEXT".to_string()), - }, - AnthropicMessage { - role: "user".to_string(), - content: AnthropicContent::Blocks(vec![AnthropicContentBlock::ToolResult { - tool_use_id: "toolu_1".to_string(), - content: Some(AnthropicContent::Text("72F sunny".to_string())), - is_error: None, - }]), - }, - ], - tools: None, - tool_choice: None, - stream: true, - temperature: None, - top_p: None, - top_k: None, - stop_sequences: None, - metadata: None, - thinking: None, - output_config: None, - }; - - let result = convert_request(request).expect("convert"); - // function_call, function_call_output (adjacent!), then the reminder. - assert_eq!(result.input.len(), 3); - assert!(matches!( - &result.input[0], - ResponseItem::FunctionCall { call_id, .. } if call_id == "toolu_1" - )); - assert!(matches!( - &result.input[1], - ResponseItem::FunctionCallOutput { call_id, .. } if call_id == "toolu_1" - )); - assert!(matches!( - &result.input[2], - ResponseItem::Message { role, content, .. } - if role == "user" - && matches!(&content[0], ContentItem::InputText { text } if text.contains("REMINDER-TEXT")) + if role == "system" + && matches!(&content[0], ContentItem::OutputText { text } if text.contains("deep-research")) )); } diff --git a/tests/gateway.rs b/tests/gateway.rs index 08db506..50574d2 100644 --- a/tests/gateway.rs +++ b/tests/gateway.rs @@ -10035,7 +10035,7 @@ async fn anthropic_messages_preserves_claude_code_skill_listing_as_user_input() } #[tokio::test] -async fn anthropic_messages_demotes_late_system_skill_listing_to_user() { +async fn anthropic_messages_preserves_late_system_skill_listing_position() { let upstream = MockUpstream::default(); upstream .push_response(vec![Ok(content_chunk("chat-1", "Hello."))]) @@ -10084,11 +10084,7 @@ async fn anthropic_messages_demotes_late_system_skill_listing_to_user() { .iter() .map(|message| message.role.as_str()) .collect(); - // Leading system = top-level `system` -> instructions. The standalone - // `role:"system"` skill listing is demoted to a synthetic `user` turn in - // place (no following user to forward-merge into) so the upstream template - // sees a single leading system message. - assert_eq!(roles, vec!["system", "user", "user"]); + assert_eq!(roles, vec!["system", "user", "system"]); let system = messages[0] .content .as_ref() From da09073f66a7dde72fe992eb4d3ffc34cb0dc765 Mon Sep 17 00:00:00 2001 From: Florian Bernd Date: Fri, 26 Jun 2026 15:38:08 +0200 Subject: [PATCH 2/2] docs: add architecture diagram, GLM-5.2 profile, and README refresh Add an architecture diagram (architecture.svg) to the README overview, plus profiles/glm52.yaml as a drop-in reassembly of the GLM-5.2 profile from the README examples, with comments explaining the gateway-level behavior of each section. Refresh the README to match: GLM-5.2 as the running example for capabilities and reasoning_effort, a bullet list for profile precedence, an expanded Brave Search section, and drop the batch capability (neither the gateway nor vLLM exposes a batch route). --- README.md | 173 ++++++++++++++---------------- architecture.svg | 253 ++++++++++++++++++++++++++++++++++++++++++++ profiles/glm52.yaml | 81 ++++++++++++++ 3 files changed, 412 insertions(+), 95 deletions(-) create mode 100644 architecture.svg create mode 100644 profiles/glm52.yaml diff --git a/README.md b/README.md index 86cd024..3f36842 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ requests, normalizes them, and forwards them to an upstream `/v1/chat/completions` server. It can also run server-side tools such as Brave Search. +![Architecture: clients (Claude Code via Anthropic Messages, Codex/OpenAI via Responses, OpenAI chat clients) route through the HTTP router and adapters into the gateway engine, which applies per-profile shaping (roles, reasoning_effort, capabilities, parallel_tool_calls) and runs server-side tools, then forwards to OpenAI-compatible upstreams (vLLM, OpenRouter) via the upstream client with routing, failover, and cooldown; config.yaml supplies profiles and upstreams.](architecture.svg) + ## Build ```bash @@ -103,13 +105,7 @@ model_profile_templates: enable_thinking: true model_profiles: - GLM-5.1: - extends: - - thinking - chat_template_kwargs: - clear_thinking: false - - Kimi-K2.6: + Kimi-K2.7: extends: - thinking system_prompt_prefix: | @@ -120,6 +116,8 @@ model_profiles: GLM-5.2: extends: - thinking + chat_template_kwargs: + clear_thinking: false upstream_chat_kwargs: parallel_tool_calls: true ``` @@ -149,9 +147,9 @@ profiles, not `*`. Per-model profile matching precedence, highest to lowest: -1. The request model - matched by name (case-insensitive) against `model_profiles`. -2. The resolved/upstream model (after `upstream_model` rewriting) - matched by name. -3. The reserved `*` profile - used only when neither 1 nor 2 matches. +- The request model - matched by name (case-insensitive) against `model_profiles`. +- The resolved/upstream model (after `upstream_model` rewriting) - matched by name. +- The reserved `*` profile - used only when neither of the above matches. Top-level config is the base below all profiles: `upstream_chat_kwargs` is the deep-merge base, and `system_prompt_prefix` is always prepended. Client request @@ -163,17 +161,17 @@ model_profiles: "*": upstream_chat_kwargs: chat_template_kwargs: - enable_thinking: true + enable_thinking: false GLM-5.2: upstream_chat_kwargs: chat_template_kwargs: - enable_thinking: false + enable_thinking: true ``` With this config, a request for `GLM-5.2` uses only the `GLM-5.2` profile -(`enable_thinking: false`); the `*` profile contributes nothing. A request for -any other model (e.g. `Qwen-3`) falls back to `*` (`enable_thinking: true`). +(`enable_thinking: true`); the `*` profile contributes nothing. A request for +any other model (e.g. `Qwen-3`) falls back to `*` (`enable_thinking: false`). ### Model capabilities @@ -182,13 +180,15 @@ advertised on `/v1/models` for Anthropic clients. ```yaml model_profiles: - "*": + GLM-5.2: capabilities: thinking: types: [adaptive, enabled] effort: levels: [max, xhigh, high, medium, low, minimal, none] + structured_outputs: true image_input: false + pdf_input: false ``` - `supported` is the only knob and defaults to `true`. The simple caps (`batch`, @@ -200,102 +200,73 @@ model_profiles: are rejected at load. - A configured cap replaces the base (upstream-supplied, else the default capabilities) for that cap key, wholesale; unconfigured caps keep the base. - A matched profile without a `capabilities` block gets no fill-in from the `*` - profile. Caps resolve per upstream id: an id-keyed profile, else the first alias - whose `upstream_model` targets the id, else the reserved `*` profile. ### Reasoning effort A profile's `reasoning_effort` block shapes the upstream `reasoning_effort` field -and controls the thinking template kwarg injected on Anthropic routes. Effort -shaping applies on `/v1/messages`, `/v1/responses`, `/v1/chat/completions`, and -`/v1/messages/count_tokens`; thinking-kwarg injection applies on the Anthropic -routes. - -```yaml -model_profiles: - "*": - reasoning_effort: - default: high - map: - low: high - xhigh: max - "*": high - thinking_param_name: enable_thinking - thinking_param_value_on: true - thinking_param_value_off: false -``` - -- `map` translates effort levels case-insensitively. An explicit key wins; `*` - rewrites any otherwise-unlisted effort. -- `default` is emitted when the client does not send an effort. Omitting it sends - no `reasoning_effort` field. -- Anthropic requests always state thinking on/off through the configured template - kwarg (default `enable_thinking: true`/`false`), overriding static defaults. - A resolved `none` effort forces the off value. -- A matching profile without `reasoning_effort` is not back-filled from `*`. - -The fork's advanced fragment form is also retained. It resolves at the final -provider leaf, so a routed or failover model receives its own vocabulary, and -it can place controls anywhere in the request rather than only remapping the -top-level field: - -```yaml -model_profiles: - GLM-5.2: - reasoning_effort_default: max - reasoning_effort_map: - high: {chat_template_kwargs: {reasoning_effort: high}} - max: {chat_template_kwargs: {reasoning_effort: max}} - none: {chat_template_kwargs: {enable_thinking: false}} -``` - -`reasoning_effort` and the fragment-based -`reasoning_effort_map`/`reasoning_effort_default` form are mutually exclusive -within a resolved profile. - -### Example: GLM-5.2 on vLLM +(the value Claude Code sends as `output_config.effort`, and the value OpenAI +clients send as `reasoning_effort`) and controls the thinking template kwarg the +gateway injects on the Anthropic route. On that route an absent `output_config.effort` +means thinking is disabled, and the upstream chat template would otherwise infer +on/off from the effort field or default it on when the kwarg is absent, so the +gateway injects an explicit `enable_thinking` template kwarg to state the intent +rather than leave it implicit. Effort shaping applies on every converting route +(`/v1/messages`, `/v1/responses`, `/v1/chat/completions`, and +`/v1/messages/count_tokens`); the thinking-template-kwarg injection applies only +on the Anthropic routes (`/v1/messages` and `/v1/messages/count_tokens`). ```yaml model_profiles: GLM-5.2: reasoning_effort: + default: high map: none: none minimal: none low: high medium: high + high: high + "*": high xhigh: max + max: max + thinking_param_name: enable_thinking + thinking_param_value_on: true + thinking_param_value_off: false ``` -`parallel_tool_calls` is a typed default: when a client omits it, the resolved -`upstream_chat_kwargs.parallel_tool_calls` default applies, and an explicit -client value always wins. The default is the global `upstream_chat_kwargs` -deep-merged with the matching profile, so a profile value overrides the global -one. The Anthropic (`/v1/messages`) route has no client field for it, so that -resolved default is the only way to control it there. Setting it to `true` (as -on `GLM-5.2` above) lets Claude Code fan out independent tool calls in one turn; -setting it to `false` forces sequential calls for a model that mishandles -parallel tool use. llmconduit always forces `false` while a gateway-owned Brave -Search or image-analysis tool is active, because those internal tool/result -loops must remain sequential. - -### Token counting - -`POST /v1/messages/count_tokens` applies the same model resolution, system -prefix, role rules, tools, chat-template defaults, and backend finalization as -generation, then calls the upstream server-root `/tokenize` endpoint. An -upstream without `/tokenize` returns an Anthropic `not_found_error`; that -unsupported result is cached for the lifetime of the process. +- `map` translates a client effort level to an upstream effort string. Keys match + case-insensitively. A level that is not listed passes through verbatim, unless + the reserved `*` entry is set, which rewrites every otherwise-unlisted level. An + explicit level always wins over `*`. +- `default` is the effort emitted when the client sends no effort string. `default: + null` (or omitting it) sends no `reasoning_effort` field. `*` does not apply to + this case. +- Anthropic clients expect thinking to be **off** unless the request explicitly + enables it, but some upstreams treat an absent `enable_thinking` kwarg as thinking + *on*. So on the Anthropic route the gateway always injects a + thinking template kwarg into `chat_template_kwargs`, stating on/off explicitly + rather than leaving it to the upstream default or inferring it from the effort + value. `thinking_param_name` is the kwarg name (default `enable_thinking`); + `thinking_param_value_on` / `_off` are the values for thinking-on and thinking-off + (defaults `true` / `false`, but any JSON value is allowed). The injected value + overrides any static `chat_template_kwargs` default for that key, and a profile + with no `reasoning_effort` block still injects the built-in `enable_thinking: + true`/`false`. +- A resolved effort of `none` also forces the off-value on the Anthropic route, even + when the request enabled thinking. This is what makes a `map` that clamps low levels + to `none` (e.g. z.ai's `minimal`/`none` -> `none`) actually skip thinking. +- Chat Completions and native Responses clients control the thinking kwarg + themselves via `chat_template_kwargs` in the request; the gateway never injects + one for them. +- A profile with no `reasoning_effort` block applies no effort shaping: the client + effort is forwarded if present, otherwise omitted (no clamp). ### Roles A per-profile `roles` block maps whole-message roles before the conversation is sent upstream. It is fail-closed: a role with no matching rule is rejected with -HTTP 400. With no `roles` block configured, the compatibility behavior remains: -`developer` is rewritten to `system`, and system messages are hoisted. Adding a -`roles` block opts that model into the exact policy below, including arbitrary -role pass-through. +HTTP 400. With no `roles` block configured, messages pass through **verbatim** - all +role shaping is opt-in. `roles` holds an optional `merge_adjacent` list plus a map of role name to a rule, or an ordered list of rules. `*` is the wildcard role: it matches any role @@ -327,8 +298,7 @@ coalesces each maximal run of consecutive messages that share a final role in the list into one content-only message joined with `\n\n`. There is no inline/leading distinction at this level - it only looks at the role messages end up as and whether they are adjacent. Folding system and tool into `user` is -`rewrite` to `user` plus `merge_adjacent: [user]`, which coalesces the -resulting adjacent user messages into one while keeping their relative order. +`rewrite` to `user` plus `merge_adjacent: [user]`, which preserves order. Resolution order for a message: the explicit role key, then the `*` wildcard, then fail-closed `reject`. @@ -347,9 +317,8 @@ model_profiles: developer: { action: rewrite, target_role: system } # System-FIRST only (Qwen3.5 raises on a non-first system message). An INLINE - # system or developer message is rewritten to `user` in place; the index-0 - # system message stays system and a leading developer message is rewritten to - # system, so Qwen never sees a non-first system. + # system/developer message is rewritten to `user` in place; the index-0 + # message stays system, so Qwen never sees a non-first system. Qwen3.5: roles: "*": { action: reject } @@ -375,10 +344,24 @@ model_profiles: tool: { action: rewrite, target_role: user, tag: tool_result } ``` -Optional Brave Search: +### Brave Search + +Setting `brave_api_key` enables a server-side `web_search` tool: when a request +asks for the built-in `web_search` tool and the model calls it, the gateway runs +the Brave Search API itself and feeds the results back into the conversation so +the model can answer (or search again) without its own internet access. With no +key set, the gateway strips `web_search` from the tool list so the upstream +never sees it. Related knobs: `brave_max_results` caps results per query +(default `5`); `max_web_search_rounds` caps how many search rounds a single +request may run (default `5`; `0` means unlimited, with a hard ceiling of `25`); +`brave_base_url` is the Brave API endpoint (default +`https://api.search.brave.com/res/v1`). ```yaml brave_api_key: "..." +brave_max_results: 5 +max_web_search_rounds: 5 +brave_base_url: "https://api.search.brave.com/res/v1" ``` Optional vision offload — forward images to a separate vision-capable model instead diff --git a/architecture.svg b/architecture.svg new file mode 100644 index 0000000..5cd1ba7 --- /dev/null +++ b/architecture.svg @@ -0,0 +1,253 @@ + + llmconduit architecture + Diagram of the llmconduit LLM gateway: clients, HTTP router, adapters, gateway engine, upstream client, config/profiles, server-side tools, and upstream backends, with their request and response flow. + + + + + + + + + + + + + + + + + + + + + + + + + llmconduit architecture + Normalize Anthropic Messages · OpenAI Responses · Chat Completions → upstream /v1/chat/completions + + + + llmconduit gateway + gateway process · axum + tokio + + + + + Config / Profiles · config.yaml + + + + + + + + + + + + model_profiles + templates + * + capabilities + upstreams + fallback + cooldown + + + + + profiles · caps + + upstreams · fallback + + + + + + + + + Claude Code + Anthropic Messages + /v1/messages + Codex / OpenAI + Responses API + /v1/responses + OpenAI clients + Chat Completions + /v1/chat/completions + + + requests + + + + + + + + + + + + HTTP Router + axum + + + + /v1/messages + /v1/responses + /v1/chat/completions + /v1/models + /health + + + + + + + + Adapters + wire ↔ Responses + + + + + + + + + + anthropic_to_responses + chat_to_responses + responses_to_anthropic + responses_to_chat + + + + + + + + Gateway Engine + stream_responses( ) + + + + • model resolution + • per-profile shaping + + · roles + · reasoning_effort + · capabilities + · parallel_tool_calls + + • system_prompt_prefix + • server-side tool loop + + + + + + + + Upstream Client + routing · failover + + + + RoutingUpstreamClient + FailoverUpstreamClient + ReqwestUpstreamClient + + + prefetch · cooldown + /v1/chat/completions + + + + + + + + + + + + + + Server-side tools + Brave Search · web_search + model calls tool → gateway runs it → feed back + + + tool round-trip + + + + + + + + + vLLM (local) + /v1/chat/completions + OpenRouter + openrouter.ai/api/v1 + OpenAI-compatible + local / remote + + + + + + + + routing · failover · cooldown + + + + + + + Brave Search API + api.search.brave.com + + + + web_search + + + + response stream (SSE) → adapters convert to client wire format → client + + + + observability & tooling + + + + + + + + + monitor / debug UI + replay store + request log · JSONL + raw mode + CLI: configure · start · analyze-log + + diff --git a/profiles/glm52.yaml b/profiles/glm52.yaml new file mode 100644 index 0000000..3d4e594 --- /dev/null +++ b/profiles/glm52.yaml @@ -0,0 +1,81 @@ +# GLM-5.2 reference profile for llmconduit. +# +# Drop-in: merge this into your config.yaml (or load it as a profile fragment) +# to get the gateway's GLM-5.2 shaping: thinking-kwarg injection on the +# Anthropic route, z.ai effort clamping, capability advertisement, and the +# full-role message mapping. Every field below is optional; trim what you +# don't need. + +# Shared template inherited by GLM-5.2 via `extends`. chat_template_kwargs is +# profile-level shorthand for upstream_chat_kwargs, forwarded to the upstream +# verbatim. The static enable_thinking applies on Chat Completions and native +# Responses; on the Anthropic route the reasoning_effort block below overrides +# it with an injected value. +model_profile_templates: + thinking: + chat_template_kwargs: + enable_thinking: true + +model_profiles: + GLM-5.2: + extends: + - thinking + + # Profile-level shorthand for upstream_chat_kwargs.chat_template_kwargs, + # forwarded to GLM's chat template verbatim. + chat_template_kwargs: + clear_thinking: false + + upstream_chat_kwargs: + # Lets Claude Code fan out independent tool calls in one turn. The Anthropic + # (/v1/messages) route has no client field for this, so the profile is the + # only way to enable it there. + parallel_tool_calls: true + + # Overrides the Anthropic model capabilities advertised on /v1/models. A + # configured cap replaces the base wholesale for that key; unconfigured caps + # keep the base. + capabilities: + thinking: + types: [adaptive, enabled] + effort: + levels: [max, xhigh, high, medium, low, minimal, none] + structured_outputs: true + image_input: false + pdf_input: false + + # Full-role mapping: system is accepted at any position (GLM handles inline + # system), tool is supported, and developer is rewritten to system. The + # wildcard rejects any role not listed. Do NOT set merge_adjacent on tool; + # GLM groups tool runs in-template. + roles: + "*": { action: reject } + user: {} + assistant: {} + tool: {} + system: {} + developer: { action: rewrite, target_role: system } + + # Shapes the upstream reasoning_effort field and the thinking kwarg injected + # on the Anthropic route. On that route an absent output_config.effort means + # thinking is off, but the upstream chat template defaults an absent + # enable_thinking to on, so the gateway injects enable_thinking explicitly + # to state on/off rather than leave it to the upstream default. + reasoning_effort: + default: high + map: + # Mimics z.ai clamp: none/minimal skip thinking, low/medium raise to + # high, xhigh raises to max. high and max are native and pass through. + none: none + minimal: none + low: high + medium: high + high: high + xhigh: max + max: max + # Catch-all for any level outside GLM's set; explicit entries above win. + "*": high + # These match GLM's defaults and are optional; shown for clarity. + thinking_param_name: enable_thinking + thinking_param_value_on: true + thinking_param_value_off: false