diff --git a/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py b/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py
index 298389f47e19..4a76235923f1 100644
--- a/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py
+++ b/tensorrt_llm/serve/tool_parser/qwen3_tool_parser.py
@@ -7,11 +7,12 @@
from ..openai_protocol import ChatCompletionToolsParam as Tool
from .base_tool_parser import BaseToolParser
-from .core_types import StreamingParseResult, StructureInfo, _GetInfoFunc
+from .core_types import (StreamingParseResult, StructureInfo, ToolCallItem,
+ _GetInfoFunc)
class Qwen3ToolParser(BaseToolParser):
- """
+ r"""
Detector for Qwen 2.5 and Qwen 3 model function call format.
Format Structure:
@@ -23,9 +24,21 @@ class Qwen3ToolParser(BaseToolParser):
- Tool Call Tags: `` and `` wrap each individual call
- Function Call Object: JSON object with "name" and "arguments" fields
+ Some Qwen3 chat templates (notably Qwen3.6 FP8 served with
+ `--reasoning_parser qwen3_5 --tool_parser qwen3`) emit tool calls as bare
+ JSON objects without the `...` wrapper once the
+ reasoning parser strips the `` block. Both `detect_and_parse` and
+ `parse_streaming_increment` fall back to a bare-JSON parse in that case
+ (see NVBug 6240584).
+
Reference: https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct?chat_template=default
"""
+ # Streaming decision state for the bare-JSON fallback path.
+ _STREAM_MODE_UNDECIDED = "undecided"
+ _STREAM_MODE_WRAPPED = "wrapped"
+ _STREAM_MODE_BARE_JSON = "bare_json"
+
def __init__(self):
"""
Initializes the detector with necessary state variables.
@@ -35,9 +48,18 @@ def __init__(self):
self.eot_token = "\n" # nosec B105
self.tool_call_separator = "\n"
self._normal_text_buffer = "" # Buffer for handling partial end tokens
+ # Bare-JSON streaming fallback state (see NVBug 6240584).
+ self._bare_json_buffer = ""
+ self._stream_mode = self._STREAM_MODE_UNDECIDED
def has_tool_call(self, text: str) -> bool:
- """Check if the text contains a Qwen 3 format tool call."""
+ """Check if the text contains a Qwen 3 format tool call.
+
+ Note: intentionally strict — this only checks for the ``
+ wrapper. Existing callers rely on it as a wrapped-form gate. The
+ bare-JSON fallback lives inside `detect_and_parse` /
+ `parse_streaming_increment` and does not go through this method.
+ """
return self.bot_token in text
def detect_and_parse(self, text: str,
@@ -52,6 +74,19 @@ def detect_and_parse(self, text: str,
idx = text.find(self.bot_token)
normal_text = text[:idx].strip() if idx != -1 else text
if self.bot_token not in text:
+ # Some Qwen3 chat templates (e.g. Qwen3.6 FP8 with thinking enabled)
+ # emit tool calls as bare JSON without a wrapper. Try
+ # to recover those before dropping the text into normal_text. Use
+ # an explicit type guard to avoid relying on AttributeError to
+ # catch scalar JSON like "42" or null.
+ try:
+ parsed = json.loads(text.strip())
+ except json.JSONDecodeError:
+ return StreamingParseResult(normal_text=normal_text, calls=[])
+ if isinstance(parsed, (dict, list)):
+ calls = self.parse_base_json(parsed, tools)
+ if calls:
+ return StreamingParseResult(normal_text="", calls=calls)
return StreamingParseResult(normal_text=normal_text, calls=[])
# Find all \n...\n blocks
@@ -71,13 +106,107 @@ def detect_and_parse(self, text: str,
def parse_streaming_increment(self, new_text: str,
tools: List[Tool]) -> StreamingParseResult:
- """
+ r"""
Streaming incremental parsing for Qwen 3 tool calls.
- Uses base class implementation with buffering to handle partial end tokens.
+
+ Handles two shapes:
+ - Wrapped: `\n{...}\n` — delegates to the base
+ class implementation, which streams the name first and then the
+ arguments incrementally. Applies the same partial-end-token
+ scrubbing as before.
+ - Bare JSON (NVBug 6240584): when the stream never contains
+ `` but the accumulated buffer is a complete JSON object
+ matching `{"name": ..., "arguments"/"parameters": ...}`, parse it
+ via `parse_base_json` and emit the calls in the same
+ name-first-then-arguments pattern used by the base class.
+
+ The parser picks a mode once and stays in it for the remainder of the
+ stream. Modes:
+ - undecided: buffer input; peek to decide.
+ - wrapped: route everything through the base implementation.
+ - bare_json: bare-JSON tool call already emitted; drop trailing text.
"""
+ if self._stream_mode == self._STREAM_MODE_WRAPPED:
+ return self._wrapped_streaming(new_text, tools)
+ if self._stream_mode == self._STREAM_MODE_BARE_JSON:
+ # Tool call already emitted; ignore any trailing text.
+ return StreamingParseResult()
+
+ # Undecided: buffer and decide.
+ self._bare_json_buffer += new_text
+ accumulated = self._bare_json_buffer
+
+ if self.bot_token in accumulated:
+ # Wrapped form: replay the accumulated buffer through the base
+ # streaming path.
+ self._stream_mode = self._STREAM_MODE_WRAPPED
+ replay = accumulated
+ self._bare_json_buffer = ""
+ return self._wrapped_streaming(replay, tools)
+
+ if self._ends_with_partial_token(accumulated, self.bot_token):
+ # Could still resolve into a wrapped form — keep buffering.
+ return StreamingParseResult()
+
+ stripped = accumulated.lstrip()
+ if not stripped:
+ # Only whitespace so far — keep buffering.
+ return StreamingParseResult()
+
+ if stripped[0] not in "{[":
+ # Not JSON-like at all — flush as normal text via wrapped path.
+ self._stream_mode = self._STREAM_MODE_WRAPPED
+ replay = accumulated
+ self._bare_json_buffer = ""
+ return self._wrapped_streaming(replay, tools)
+
+ # Leading `{` or `[` — could be a bare JSON tool call, or partial.
+ try:
+ parsed = json.loads(stripped)
+ except json.JSONDecodeError:
+ # Not yet complete — buffer more.
+ return StreamingParseResult()
+
+ if isinstance(parsed, (dict, list)):
+ calls = self.parse_base_json(parsed, tools)
+ if calls:
+ streaming_calls: List[ToolCallItem] = []
+ for i, call_item in enumerate(calls):
+ # Emit name first with empty parameters, then the full
+ # arguments — mirrors the base class streaming pattern.
+ streaming_calls.append(
+ ToolCallItem(
+ tool_index=i,
+ name=call_item.name,
+ parameters="",
+ ))
+ if call_item.parameters:
+ streaming_calls.append(
+ ToolCallItem(
+ tool_index=i,
+ parameters=call_item.parameters,
+ ))
+ self._stream_mode = self._STREAM_MODE_BARE_JSON
+ self._bare_json_buffer = ""
+ # Best-effort bookkeeping for callers that inspect these.
+ self.current_tool_id = max(0, len(calls) - 1)
+ self.current_tool_name_sent = True
+ return StreamingParseResult(normal_text="",
+ calls=streaming_calls)
+
+ # Parsed as JSON but not a tool call — flush as normal text via
+ # wrapped path.
+ self._stream_mode = self._STREAM_MODE_WRAPPED
+ replay = accumulated
+ self._bare_json_buffer = ""
+ return self._wrapped_streaming(replay, tools)
+
+ def _wrapped_streaming(self, new_text: str,
+ tools: List[Tool]) -> StreamingParseResult:
+ """Wrapped-form streaming: base implementation + partial-end-token scrubbing."""
result = super().parse_streaming_increment(new_text, tools)
- # Handle partial end tokens that are streamed character by character
+ # Handle partial end tokens that are streamed character by character.
if result.normal_text:
self._normal_text_buffer += result.normal_text
diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py
index 3b7e19f3ce55..b2f519b95060 100644
--- a/tests/unittest/llmapi/apps/test_tool_parsers.py
+++ b/tests/unittest/llmapi/apps/test_tool_parsers.py
@@ -797,6 +797,158 @@ def test_qwen3_format_compliance(self, sample_tools, parser):
assert result.calls[0].name == "get_weather"
assert json.loads(result.calls[0].parameters) == {"location": "Tokyo"}
+ # ------------------------------------------------------------------
+ # NVBug 6240584: bare-JSON fallback in detect_and_parse
+ #
+ # Some Qwen3 chat templates (notably Qwen3.6 FP8 with
+ # `--reasoning_parser qwen3_5 --tool_parser qwen3`) emit tool calls
+ # as bare JSON, without a `...` wrapper, once
+ # the reasoning parser strips the `` block. The parser must
+ # recover those before dropping the text into `normal_text`.
+ # ------------------------------------------------------------------
+
+ def test_detect_and_parse_bare_json_dict(self, sample_tools, parser):
+ """Bare JSON dict without wrapper is parsed as a tool call."""
+ text = '{"name":"get_weather","arguments":{"location":"Paris"}}'
+
+ result = parser.detect_and_parse(text, sample_tools)
+
+ assert result.normal_text == ""
+ assert len(result.calls) == 1
+ assert result.calls[0].name == "get_weather"
+ assert json.loads(result.calls[0].parameters) == {"location": "Paris"}
+
+ def test_detect_and_parse_bare_json_list(self, sample_tools, parser):
+ """Bare JSON list of tool calls without wrapper is parsed."""
+ text = '[{"name":"get_weather","arguments":{"location":"Paris"}}]'
+
+ result = parser.detect_and_parse(text, sample_tools)
+
+ assert result.normal_text == ""
+ assert len(result.calls) == 1
+ assert result.calls[0].name == "get_weather"
+ assert json.loads(result.calls[0].parameters) == {"location": "Paris"}
+
+ def test_detect_and_parse_bare_json_parameters_key(self, sample_tools,
+ parser):
+ """Bare JSON with `parameters` (instead of `arguments`) is still parsed."""
+ text = '{"name":"get_weather","parameters":{"location":"Paris"}}'
+
+ result = parser.detect_and_parse(text, sample_tools)
+
+ assert len(result.calls) == 1
+ assert result.calls[0].name == "get_weather"
+ assert json.loads(result.calls[0].parameters) == {"location": "Paris"}
+
+ def test_detect_and_parse_non_json_text_falls_through(
+ self, sample_tools, parser):
+ """Plain non-JSON text passes through as normal_text with no calls."""
+ text = "Hello world"
+
+ result = parser.detect_and_parse(text, sample_tools)
+
+ assert result.normal_text == "Hello world"
+ assert result.calls == []
+
+ def test_detect_and_parse_bare_json_scalar_falls_through(
+ self, sample_tools, parser):
+ """A JSON scalar (e.g. `"42"`) must fall through cleanly, not crash.
+
+ This exercises the explicit `isinstance(parsed, (dict, list))` guard —
+ `parse_base_json` would raise `AttributeError` on a bare int, so the
+ guard prevents relying on exception catching for scalar JSON.
+ """
+ text = "42"
+
+ result = parser.detect_and_parse(text, sample_tools)
+
+ assert result.calls == []
+ # No crash is the important part.
+
+ def test_detect_and_parse_malformed_bare_json_falls_through(
+ self, sample_tools, parser):
+ """Malformed JSON without wrapper falls through cleanly."""
+ text = '{"name": "get_weather", "arguments": MALFORMED}'
+
+ result = parser.detect_and_parse(text, sample_tools)
+
+ assert result.calls == []
+ assert result.normal_text == text
+
+ # ------------------------------------------------------------------
+ # NVBug 6240584: bare-JSON fallback in parse_streaming_increment
+ #
+ # The streaming path must also recover bare-JSON tool calls when the
+ # `` wrapper never appears. Without this, streaming clients
+ # receive the JSON as `delta.content` with `finish_reason="stop"`.
+ # ------------------------------------------------------------------
+
+ def test_streaming_bare_json_one_chunk(self, sample_tools, parser):
+ """A complete bare-JSON tool call arriving in a single chunk emits calls."""
+ result = parser.parse_streaming_increment(
+ '{"name":"get_weather","arguments":{"city":"Paris"}}', sample_tools)
+
+ names = [c.name for c in result.calls if c.name]
+ assert "get_weather" in names
+ params = "".join(c.parameters for c in result.calls if c.parameters)
+ assert "Paris" in params
+ assert result.normal_text == ""
+
+ def test_streaming_bare_json_split_across_chunks(self, sample_tools,
+ parser):
+ """Bare-JSON tool call split across multiple chunks parses on completion."""
+ r1 = parser.parse_streaming_increment('{"name":"get_', sample_tools)
+ r2 = parser.parse_streaming_increment('weather","arguments":',
+ sample_tools)
+ r3 = parser.parse_streaming_increment('{"city":"Paris"}}', sample_tools)
+
+ all_calls = list(r1.calls) + list(r2.calls) + list(r3.calls)
+ names = [c.name for c in all_calls if c.name]
+ assert "get_weather" in names
+ params = "".join(c.parameters for c in all_calls if c.parameters)
+ assert "Paris" in params
+
+ def test_streaming_bare_json_does_not_leak_content(self, sample_tools,
+ parser):
+ """After a bare-JSON tool call is emitted, trailing text is not leaked.
+
+ This must be the case even for subsequent empty/whitespace chunks:
+ leaking any normal_text would flip `finish_reason` back to `stop`.
+ """
+ r1 = parser.parse_streaming_increment(
+ '{"name":"get_weather","arguments":{"city":"Paris"}}', sample_tools)
+ # Any subsequent chunks must not emit normal_text either.
+ r2 = parser.parse_streaming_increment("", sample_tools)
+
+ assert r1.normal_text == ""
+ assert r2.normal_text == ""
+
+ def test_streaming_non_json_text_flushed_as_normal(self, sample_tools,
+ parser):
+ """Non-JSON text without a wrapper is flushed to normal_text."""
+ result = parser.parse_streaming_increment("Hello world", sample_tools)
+
+ assert result.normal_text == "Hello world"
+ assert result.calls == []
+
+ def test_streaming_wrapped_form_unregressed(self, sample_tools, parser):
+ """The pre-existing wrapped-form streaming path continues to work."""
+ # Send bot token.
+ parser.parse_streaming_increment("\n", sample_tools)
+
+ # Partial JSON with name -> emits name with empty params.
+ r_name = parser.parse_streaming_increment('{"name":"get_weather"',
+ sample_tools)
+ assert len(r_name.calls) == 1
+ assert r_name.calls[0].name == "get_weather"
+ assert r_name.calls[0].parameters == ""
+
+ # Complete the JSON and the wrapper.
+ r_args = parser.parse_streaming_increment(
+ ',"arguments":{"location":"SF"}}\n', sample_tools)
+ assert len(r_args.calls) == 1
+ assert json.loads(r_args.calls[0].parameters) == {"location": "SF"}
+
class TestQwen3CoderToolParser(BaseToolParserTestClass):
"""Test suite for Qwen3CoderToolParser class."""
@@ -2691,6 +2843,68 @@ def test_auto_detect_laguna(self, tmp_path):
class TestToolParserIntegration:
"""Integration tests for tool parsers."""
+ def test_qwen3_5_reasoning_plus_qwen3_tool_parser_bare_json_pipeline(
+ self, sample_tools):
+ r"""NVBug 6240584: end-to-end reasoning + tool parser pipeline.
+
+ Reproduces the exact scenario from the bug report: the Qwen3.6 FP8
+ chat template pre-injects `\n` into the assistant prompt
+ prefix, so the model output starts *inside* the reasoning block
+ with no opening `` tag. Content up to `` is the
+ reasoning, and what follows is a bare JSON tool call (no
+ `` wrapper). The `qwen3_5` reasoning parser (registered
+ with `reasoning_at_start=True`) strips the thinking block, then
+ the `qwen3` tool parser must recover the tool call so
+ `args.has_tool_call[0]` is True and downstream logic sets
+ `finish_reason="tool_calls"` (see chat_response_post_processor).
+ """
+ from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest
+ from tensorrt_llm.serve.postprocess_handlers import (
+ ChatPostprocArgs, apply_reasoning_parser, apply_tool_parser)
+
+ # Bug-report input: reasoning content (no leading `` — the
+ # chat template already injected it into the prompt prefix) followed
+ # by a bare JSON tool call.
+ text = ('Reasoning here.\n'
+ '{"name":"get_weather","arguments":{"city":"Paris"}}')
+
+ # Build a minimal request so we can construct ChatPostprocArgs.
+ req = ChatCompletionRequest(
+ model="Qwen/Qwen3.6-27B-FP8",
+ messages=[{
+ "role": "user",
+ "content": "What is the weather in Paris?"
+ }],
+ tools=sample_tools,
+ )
+ args = ChatPostprocArgs.from_request(req)
+ args.reasoning_parser = "qwen3_5"
+ args.tool_parser = "qwen3"
+
+ # Non-streaming path.
+ content, reasoning_content = apply_reasoning_parser(args,
+ output_index=0,
+ text=text,
+ streaming=False)
+ assert reasoning_content == "Reasoning here."
+ # The reasoning parser strips `...` — the remaining
+ # content is the bare JSON, possibly with a leading newline.
+ assert '"name":"get_weather"' in content
+
+ normal_text, calls = apply_tool_parser(args,
+ output_index=0,
+ text=content,
+ streaming=False)
+
+ assert len(calls) == 1
+ assert calls[0].name == "get_weather"
+ assert json.loads(calls[0].parameters) == {"city": "Paris"}
+ # Downstream (chat_response_post_processor) checks this flag to flip
+ # finish_reason from "stop" to "tool_calls".
+ assert args.has_tool_call.get(0) is True
+ # And no bare JSON leaks into the visible content.
+ assert normal_text == ""
+
def test_end_to_end_single_tool(self, sample_tools):
"""Test end-to-end parsing of a single tool call."""
parser = Qwen3ToolParser()