diff --git a/third_party/minja b/third_party/minja index 1d9d12a..1e65bd7 160000 --- a/third_party/minja +++ b/third_party/minja @@ -1 +1 @@ -Subproject commit 1d9d12a48e9f7a26b39aca82f4758ccb4893cd02 +Subproject commit 1e65bd7254aa974d4bc01f9819acec0a0f7bb2d3 diff --git a/xllm_service/chat_template/jinja_chat_template.cpp b/xllm_service/chat_template/jinja_chat_template.cpp index 7ae3374..8809c8d 100644 --- a/xllm_service/chat_template/jinja_chat_template.cpp +++ b/xllm_service/chat_template/jinja_chat_template.cpp @@ -184,12 +184,37 @@ std::optional JinjaChatTemplate::apply( if (message.tool_calls.has_value()) { nlohmann::ordered_json tool_calls_json = nlohmann::json::array(); for (const auto& tool_call : *message.tool_calls) { + // Tool-call arguments arrive as a JSON string (OpenAI wire format), but + // chat templates such as GLM iterate `arguments.items()`, which only + // works on a JSON object. A non-object value (truncated/malformed JSON, + // or a JSON array/scalar) makes minja throw "Unknown method: items", + // which would abort the whole process. Always hand the template an + // object: parse when possible and fall back to an empty object + // otherwise, so rendering can never throw on this field. + nlohmann::ordered_json arguments_json = + nlohmann::ordered_json::object(); + if (!tool_call.function.arguments.empty()) { + try { + auto parsed = nlohmann::json::parse(tool_call.function.arguments); + if (parsed.is_object()) { + arguments_json = std::move(parsed); + } else { + LOG(WARNING) << "Tool-call arguments are not a JSON object, " + "rendering empty args: " + << tool_call.function.arguments; + } + } catch (const nlohmann::json::exception& e) { + LOG(WARNING) << "Failed to parse tool-call arguments, rendering " + "empty args: " + << e.what(); + } + } tool_calls_json.emplace_back(nlohmann::ordered_json{ {"id", tool_call.id}, {"type", tool_call.type}, {"function", {{"name", tool_call.function.name}, - {"arguments", tool_call.function.arguments}}}}); + {"arguments", std::move(arguments_json)}}}}); } message_json["tool_calls"] = std::move(tool_calls_json); } @@ -249,7 +274,18 @@ std::optional JinjaChatTemplate::apply( input.extra_context = chat_template_kwargs; minja::chat_template_options options; - return template_->apply(input, options); + // minja throws std::runtime_error on any rendering failure (e.g. calling + // `.items()` on a non-object value). The OpenAI HTTP path does not wrap the + // scheduler call in a try/catch, so an uncaught exception here would escape + // the brpc handler and terminate the whole process. Contain it and degrade + // to an empty result, letting callers report a normal "failed to construct + // prompt" error instead of crashing. + try { + return template_->apply(input, options); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to apply chat template: " << e.what(); + return std::nullopt; + } } nlohmann::ordered_json JinjaChatTemplate::get_mm_content( diff --git a/xllm_service/common/call_data.h b/xllm_service/common/call_data.h index f485d75..0a9702f 100644 --- a/xllm_service/common/call_data.h +++ b/xllm_service/common/call_data.h @@ -97,7 +97,7 @@ class StreamCallData : public CallData { } else { controller_->http_response().SetHeader("Content-Type", - "text/javascript; charset=utf-8"); + "application/json; charset=utf-8"); } json_options_.bytes_to_base64 = false; diff --git a/xllm_service/http_service/service.cpp b/xllm_service/http_service/service.cpp index 40f64b5..5c22954 100644 --- a/xllm_service/http_service/service.cpp +++ b/xllm_service/http_service/service.cpp @@ -299,18 +299,28 @@ constexpr char kInferContentLength[] = "Infer-Content-Length"; constexpr char kContentLength[] = "Content-Length"; size_t GetJsonContentLength(const brpc::Controller* ctrl) { + const auto parse_length = [](const std::string& value) -> size_t { + try { + return std::stoul(value); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid Content-Length value: " << value + << ", error: " << e.what(); + return (size_t)-1L; + } + }; + const auto infer_content_len = ctrl->http_request().GetHeader(kInferContentLength); if (infer_content_len != nullptr) { - return std::stoul(*infer_content_len); + return parse_length(*infer_content_len); } const auto content_len = ctrl->http_request().GetHeader(kContentLength); if (content_len != nullptr) { - return std::stoul(*content_len); + return parse_length(*content_len); } - LOG(FATAL) << "Content-Length header is missing."; + LOG(ERROR) << "Content-Length header is missing."; return (size_t)-1L; } @@ -569,6 +579,10 @@ void XllmHttpServiceImpl::ChatCompletions( arena); auto content_len = GetJsonContentLength(cntl); + if (content_len == (size_t)-1L) { + cntl->SetFailed("Content-Length header is missing or invalid."); + return; + } std::string attachment; cntl->request_attachment().copy_to(&attachment, content_len, 0); @@ -594,7 +608,28 @@ void XllmHttpServiceImpl::ChatCompletions( if (req_pb->messages_size() > 0) { service_request->messages.reserve(req_pb->messages_size()); for (const auto& message : req_pb->messages()) { - service_request->messages.emplace_back(message.role(), message.content()); + Message msg(message.role(), message.content()); + if (message.has_reasoning_content()) { + msg.reasoning_content = message.reasoning_content(); + } + if (!message.tool_call_id().empty()) { + msg.tool_call_id = message.tool_call_id(); + } + if (message.tool_calls_size() > 0) { + Message::ToolCallVec tool_calls; + tool_calls.reserve(message.tool_calls_size()); + for (const auto& tool_call : message.tool_calls()) { + Message::ToolCall parsed_tool_call; + parsed_tool_call.id = tool_call.id(); + parsed_tool_call.type = tool_call.type(); + parsed_tool_call.function.name = tool_call.function().name(); + parsed_tool_call.function.arguments = + tool_call.function().arguments(); + tool_calls.emplace_back(std::move(parsed_tool_call)); + } + msg.tool_calls = std::move(tool_calls); + } + service_request->messages.emplace_back(std::move(msg)); } if (req_pb->has_chat_template_kwargs()) { service_request->chat_template_kwargs = @@ -670,6 +705,10 @@ void XllmHttpServiceImpl::AnthropicMessages( ::xllm::proto::AnthropicMessagesResponse>(arena); auto content_len = GetJsonContentLength(cntl); + if (content_len == (size_t)-1L) { + cntl->SetFailed("Content-Length header is missing or invalid."); + return; + } std::string attachment; cntl->request_attachment().copy_to(&attachment, content_len, 0); diff --git a/xllm_service/scheduler/response_handler.cpp b/xllm_service/scheduler/response_handler.cpp index 25b0c89..ba293da 100644 --- a/xllm_service/scheduler/response_handler.cpp +++ b/xllm_service/scheduler/response_handler.cpp @@ -49,6 +49,31 @@ size_t find_tool_start(const std::string& text) { return pos; } +// GLM-family control markers that may leak into the streamed output when +// skip_special_tokens is disabled for tool calling (e.g. the model emits +// <|observation|> right after ). They carry no visible content. +// The non-stream path already drops trailing text after tool calls, so this +// keeps the streaming behavior consistent and prevents these markers from +// reaching the client. +void strip_special_control_tokens(std::string* text) { + if (!text || text->empty()) { + return; + } + static const std::vector kControlTokens = { + "<|observation|>", + "<|user|>", + "<|assistant|>", + "<|system|>", + "<|endoftext|>", + }; + for (const auto& token : kControlTokens) { + size_t pos = 0; + while ((pos = text->find(token, pos)) != std::string::npos) { + text->erase(pos, token.length()); + } + } +} + AnthropicTracer make_anthropic_tracer( const std::shared_ptr& call_data) { return AnthropicTracer( @@ -89,7 +114,9 @@ bool send_normal_text_chunk(std::shared_ptr call_data, const std::string& request_id, int64_t created_time, const std::string& model) { - if (content.empty()) { + std::string filtered_content = content; + strip_special_control_tokens(&filtered_content); + if (filtered_content.empty()) { return true; } @@ -102,7 +129,7 @@ bool send_normal_text_chunk(std::shared_ptr call_data, auto* choice = response.add_choices(); choice->set_index(index); auto* delta = choice->mutable_delta(); - delta->set_content(content); + delta->set_content(filtered_content); return call_data->write(response); } @@ -303,18 +330,21 @@ bool ResponseHandler::send_delta_to_client( return false; } } else { - response.Clear(); - response.set_object("chat.completion.chunk"); - response.set_id(request_id); - response.set_created(created_time); - response.set_model(model); - auto* choice = response.add_choices(); - choice->set_index(index); - set_logprobs(choice, seq_output.logprobs); - auto* message = choice->mutable_delta(); - message->set_content(cur_text); - if (!call_data->write(response)) { - return false; + strip_special_control_tokens(&cur_text); + if (!cur_text.empty()) { + response.Clear(); + response.set_object("chat.completion.chunk"); + response.set_id(request_id); + response.set_created(created_time); + response.set_model(model); + auto* choice = response.add_choices(); + choice->set_index(index); + set_logprobs(choice, seq_output.logprobs); + auto* message = choice->mutable_delta(); + message->set_content(cur_text); + if (!call_data->write(response)) { + return false; + } } } } @@ -515,7 +545,12 @@ bool ResponseHandler::send_delta_to_client( }; auto emit_text = [&](const std::string& text) -> bool { - auto result = encoder.on_text(output, text, &sse_events); + std::string filtered = text; + strip_special_control_tokens(&filtered); + if (filtered.empty()) { + return true; + } + auto result = encoder.on_text(output, filtered, &sse_events); if (!result.ok) { return call_data->finish_with_error(result.error); } @@ -701,6 +736,7 @@ bool ResponseHandler::send_result_to_client( reasoning_parser, force_reasoning, response.GetArena()); + strip_special_control_tokens(&result.text); message->set_content(result.text); if (result.reasoning_content.has_value()) { message->set_reasoning_content(result.reasoning_content.value());