diff --git a/tests/v1/sample/test_reasoning_temperature.py b/tests/v1/sample/test_reasoning_temperature.py new file mode 100644 index 000000000000..405b86ddd34e --- /dev/null +++ b/tests/v1/sample/test_reasoning_temperature.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the reasoning/answer temperature split on SamplingParams. + +These lock in the contract the InputBatch split gate relies on: an unset +``reasoning_temperature`` must mean "mirror temperature" (no split, no +overhead), while a set, differing value opts the request into phase-dependent +sampling. +""" + +from unittest.mock import Mock + +import pytest + +from vllm.sampling_params import _MAX_TEMP, SamplingParams, SamplingType + + +def test_default_is_unset(): + """An unspecified reasoning_temperature stays None (the no-split sentinel), + so an ordinary greedy request is not pushed onto the split path.""" + params = SamplingParams(temperature=0.0) + assert params.reasoning_temperature is None + # Plain greedy request remains greedy and gets the greedy top_p/k reset. + assert params.sampling_type == SamplingType.GREEDY + assert params.top_p == 1.0 + assert params.top_k == 0 + assert params.min_p == 0.0 + + +def test_from_optional_passes_none_through(): + """from_optional must not coerce an unset value to a concrete float; + coercing to 1.0 was what made the split fire for nearly every request.""" + params = SamplingParams.from_optional(temperature=0.7) + assert params.reasoning_temperature is None + + +@pytest.mark.parametrize("reasoning_temperature", [0.0, 0.7, 1.5]) +def test_split_is_set_when_specified(reasoning_temperature): + params = SamplingParams.from_optional( + temperature=0.0, reasoning_temperature=reasoning_temperature + ) + assert params.reasoning_temperature == reasoning_temperature + + +def test_greedy_reset_skipped_when_reasoning_is_random(): + """temperature=0 with a stochastic reasoning_temperature must NOT collapse + top_p/top_k/min_p, since the reasoning phase still samples randomly.""" + params = SamplingParams( + temperature=0.0, + reasoning_temperature=0.7, + top_p=0.5, + top_k=20, + min_p=0.1, + ) + assert params.top_p == 0.5 + assert params.top_k == 20 + assert params.min_p == 0.1 + + +def test_sampling_type_uses_reasoning_temperature(): + params = SamplingParams( + temperature=0.0, + reasoning_temperature=0.7, + seed=123, + ) + assert params.sampling_type == SamplingType.RANDOM_SEED + + +def test_spec_decode_disables_reasoning_temperature_split(): + params = SamplingParams( + temperature=0.0, + reasoning_temperature=0.7, + top_p=0.5, + top_k=20, + min_p=0.1, + ) + assert params.sampling_type == SamplingType.RANDOM + + params._validate_spec_decode(Mock()) + + assert params.reasoning_temperature is None + assert params.sampling_type == SamplingType.GREEDY + assert params.top_p == 1.0 + assert params.top_k == 0 + assert params.min_p == 0.0 + + +def test_greedy_reset_applies_when_both_greedy(): + """temperature=0 and reasoning_temperature=0 is fully greedy and should + reset the nucleus parameters just like a plain greedy request.""" + params = SamplingParams( + temperature=0.0, + reasoning_temperature=0.0, + top_p=0.5, + top_k=20, + min_p=0.1, + ) + assert params.top_p == 1.0 + assert params.top_k == 0 + assert params.min_p == 0.0 + + +def test_negative_reasoning_temperature_rejected(): + with pytest.raises(ValueError): + SamplingParams(reasoning_temperature=-0.1) + + +def test_tiny_positive_reasoning_temperature_clamped(): + params = SamplingParams(reasoning_temperature=_MAX_TEMP / 2) + assert params.reasoning_temperature == _MAX_TEMP + + +def test_unset_reasoning_temperature_not_clamped(): + """None must survive __post_init__ untouched (no clamp, no validation + error).""" + params = SamplingParams(temperature=0.0) + assert params.reasoning_temperature is None diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 73ecb3f35a1c..6d6b7b72ff46 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -608,16 +608,26 @@ def to_sampling_params( else replace(self.structured_outputs, **structured_outputs_kwargs) ) - extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {} + # Copy so popping known extension params below does not mutate the + # request's ``vllm_xargs`` in place. + extra_args: dict[str, Any] = dict(self.vllm_xargs) if self.vllm_xargs else {} if self.kv_transfer_params: # Pass in kv_transfer_params via extra_args extra_args["kv_transfer_params"] = self.kv_transfer_params + + # Extract known extension parameters from extra_args before passing + # rest downstream. Pop them so they don't end up in extra_args. + reasoning_temperature: float | None = extra_args.pop( + "reasoning_temperature", None + ) + return SamplingParams.from_optional( n=self.n, presence_penalty=self.presence_penalty, frequency_penalty=self.frequency_penalty, repetition_penalty=repetition_penalty, temperature=temperature, + reasoning_temperature=reasoning_temperature, top_p=top_p, top_k=top_k, min_p=min_p, diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index cb793a415633..ce3505efa682 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -308,16 +308,24 @@ def to_sampling_params( else replace(self.structured_outputs, **structured_outputs_kwargs) ) - extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {} + # Copy so popping known extension params below does not mutate the + # request's ``vllm_xargs`` in place. + extra_args: dict[str, Any] = dict(self.vllm_xargs) if self.vllm_xargs else {} if self.kv_transfer_params: # Pass in kv_transfer_params via extra_args extra_args["kv_transfer_params"] = self.kv_transfer_params + + reasoning_temperature: float | None = extra_args.pop( + "reasoning_temperature", None + ) + return SamplingParams.from_optional( n=self.n, presence_penalty=self.presence_penalty, frequency_penalty=self.frequency_penalty, repetition_penalty=repetition_penalty, temperature=temperature, + reasoning_temperature=reasoning_temperature, top_p=top_p, top_k=top_k, min_p=min_p, diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 30a920663651..791a4fb68f40 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -397,12 +397,19 @@ def to_sampling_params( if isinstance(stop, str): stop = [stop] - extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {} + # Copy so popping known extension params below does not mutate the + # request's ``vllm_xargs`` in place. + extra_args: dict[str, Any] = dict(self.vllm_xargs) if self.vllm_xargs else {} if self.kv_transfer_params: extra_args["kv_transfer_params"] = self.kv_transfer_params + reasoning_temperature: float | None = extra_args.pop( + "reasoning_temperature", None + ) + return SamplingParams.from_optional( temperature=temperature, + reasoning_temperature=reasoning_temperature, top_p=top_p, top_k=top_k, max_tokens=max_tokens, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 6e0be9dbff57..5e89cabe5c67 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -236,6 +236,14 @@ class SamplingParams( """Controls the randomness of the sampling. Lower values make the model more deterministic, while higher values make the model more random. Zero means greedy sampling.""" + reasoning_temperature: float | None = None + """Temperature used during the reasoning/thinking phase. + When set and different from ``temperature``, reasoning/thinking tokens are + sampled at ``reasoning_temperature`` while the answer is sampled at + ``temperature``. For example ``temperature=0.0, reasoning_temperature=0.7`` + gives a greedy answer with stochastic reasoning. Leave unset (``None``, + the default) to use ``temperature`` for both phases — no split, no + overhead.""" top_p: float = 1.0 """Controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to 1 to consider all tokens.""" @@ -358,6 +366,7 @@ def from_optional( frequency_penalty: float | None = 0.0, repetition_penalty: float | None = 1.0, temperature: float | None = 1.0, + reasoning_temperature: float | None = None, top_p: float | None = 1.0, top_k: int = 0, min_p: float = 0.0, @@ -399,6 +408,7 @@ def from_optional( if repetition_penalty is None else repetition_penalty, temperature=1.0 if temperature is None else temperature, + reasoning_temperature=reasoning_temperature, top_p=1.0 if top_p is None else top_p, top_k=top_k, min_p=min_p, @@ -436,6 +446,19 @@ def __post_init__(self) -> None: ) self.temperature = max(self.temperature, _MAX_TEMP) + if self.reasoning_temperature is not None and ( + 0 < self.reasoning_temperature < _MAX_TEMP + ): + logger.warning( + "reasoning_temperature %s is less than %s, which may cause " + "numerical errors nan or inf in tensors. We have maxed it " + "out to %s.", + self.reasoning_temperature, + _MAX_TEMP, + _MAX_TEMP, + ) + self.reasoning_temperature = max(self.reasoning_temperature, _MAX_TEMP) + if self.seed == -1: self.seed = None @@ -467,8 +490,16 @@ def __post_init__(self) -> None: self._verify_args() - if self.temperature < _SAMPLING_EPS: - # Zero temperature means greedy sampling. + # ``reasoning_temperature is None`` means the reasoning phase mirrors + # ``temperature``, so the request is fully greedy iff ``temperature`` + # is. A set, non-greedy reasoning_temperature keeps top_p/top_k/min_p + # alive for the (random) reasoning phase even when the answer is greedy. + reasoning_temp_greedy = ( + self.reasoning_temperature is None + or self.reasoning_temperature < _SAMPLING_EPS + ) + if self.temperature < _SAMPLING_EPS and reasoning_temp_greedy: + # Both phases are effectively zero -> fully greedy sampling. self.top_p = 1.0 self.top_k = 0 self.min_p = 0.0 @@ -514,6 +545,13 @@ def _verify_args(self) -> None: parameter="temperature", value=self.temperature, ) + if self.reasoning_temperature is not None and self.reasoning_temperature < 0.0: + raise VLLMValidationError( + f"reasoning_temperature must be non-negative, got " + f"{self.reasoning_temperature}.", + parameter="reasoning_temperature", + value=self.reasoning_temperature, + ) if not 0.0 < self.top_p <= 1.0: raise VLLMValidationError( f"top_p must be in (0, 1], got {self.top_p}.", @@ -660,7 +698,15 @@ def update_from_tokenizer(self, tokenizer: TokenizerLike) -> None: @cached_property def sampling_type(self) -> SamplingType: - if self.temperature < _SAMPLING_EPS: + reasoning_temperature = ( + self.temperature + if self.reasoning_temperature is None + else self.reasoning_temperature + ) + if ( + self.temperature < _SAMPLING_EPS + and reasoning_temperature < _SAMPLING_EPS + ): return SamplingType.GREEDY if self.seed is not None: return SamplingType.RANDOM_SEED @@ -818,6 +864,23 @@ def _validate_spec_decode( if speculative_config is None: return + if ( + self.reasoning_temperature is not None + and self.reasoning_temperature != self.temperature + ): + logger.warning_once( + "reasoning_temperature is not supported with speculative " + "decoding yet. Ignoring reasoning_temperature and using " + "temperature for all generated tokens." + ) + self.reasoning_temperature = None + self.__dict__.pop("sampling_type", None) + if self.temperature < _SAMPLING_EPS: + self.top_p = 1.0 + self.top_k = 0 + self.min_p = 0.0 + self._verify_greedy_sampling() + # Some sampling parameters are not yet compatible with spec decoding. if self.min_p > _SAMPLING_EPS or self.logit_bias: raise ValueError( @@ -968,6 +1031,7 @@ def __repr__(self) -> str: f"frequency_penalty={self.frequency_penalty}, " f"repetition_penalty={self.repetition_penalty}, " f"temperature={self.temperature}, " + f"reasoning_temperature={self.reasoning_temperature}, " f"top_p={self.top_p}, " f"top_k={self.top_k}, " f"min_p={self.min_p}, " diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 89d69c0bde64..f1b414b1ecbf 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -188,6 +188,47 @@ def __init__( (max_num_reqs,), dtype=torch.float32, device="cpu", pin_memory=pin_memory ) self.temperature_cpu = self.temperature_cpu_tensor.numpy() + + # Reasoning-phase temperature split. + self.reasoning_temperature = torch.empty( + (max_num_reqs,), dtype=torch.float32, device=device + ) + self.reasoning_temperature_cpu_tensor = torch.empty( + (max_num_reqs,), + dtype=torch.float32, + device="cpu", + pin_memory=PIN_MEMORY, + ) + self.reasoning_temperature_cpu = self.reasoning_temperature_cpu_tensor.numpy() + # Requests whose reasoning_temperature differs from temperature. The + # batch-level split path is active iff this set is non-empty (see the + # ``has_reasoning_temp_split`` property), so it clears automatically + # once all such requests leave the batch. + self.reasoning_temp_split_reqs: set[str] = set() + # Think state seeded from each row's prompt (fallback when the output + # contains no marker yet). + self.prompt_in_think_cpu = np.zeros(max_num_reqs, dtype=bool) + # Live per-step think mask, refreshed every decode step in + # ``update_reasoning_temperature`` and copied to the GPU for the blend. + self.think_in_think_cpu_tensor = torch.zeros( + (max_num_reqs,), dtype=torch.bool, device="cpu", pin_memory=PIN_MEMORY + ) + self.think_in_think_cpu = self.think_in_think_cpu_tensor.numpy() + self.think_in_think_gpu = torch.zeros( + (max_num_reqs,), dtype=torch.bool, device=device + ) + # Think-boundary token IDs, populated from reasoning_config. + self.think_start_token_ids: list[int] = [] + self.think_end_token_ids: list[int] = [] + if reasoning_config is not None: + if reasoning_config.reasoning_start_token_ids: + self.think_start_token_ids = ( + reasoning_config.reasoning_start_token_ids + ) + if reasoning_config.reasoning_end_token_ids: + self.think_end_token_ids = ( + reasoning_config.reasoning_end_token_ids + ) self.greedy_reqs: set[str] = set() self.random_reqs: set[str] = set() @@ -378,7 +419,32 @@ def add_request( self.block_table.add_row(request.block_ids, req_index) if sampling_params := request.sampling_params: - if sampling_params.sampling_type == SamplingType.GREEDY: + # ``reasoning_temperature is None`` means "mirror temperature", so + # the split is only active when it is set AND differs. Comparing + # the raw default (None) here is what keeps ordinary requests off + # the mixed sampling path. + reasoning_temperature = sampling_params.reasoning_temperature + has_reasoning_split = ( + reasoning_temperature is not None + and reasoning_temperature != sampling_params.temperature + ) + if has_reasoning_split: + # When the temperature split is active, the request must go + # into both greedy_reqs AND random_reqs so that the sampler + # takes the mixed path. This allows phase-dependent sampling: + # - thinking phase -> reasoning_temperature (may be >0) + # - answer phase -> temperature (may be 0.0, greedy) + # The mixed path ensures 0.0 temps don't cause NaN (clamped + # to 1.0 in apply_temperature), and torch.where() selects + # greedy vs random per-request based on the blended temp. + # Store the ORIGINAL temperature in temperature_cpu and the + # thinking-phase temp in reasoning_temperature_cpu; + # update_reasoning_temperature() blends them each step. + self.temperature_cpu[req_index] = sampling_params.temperature + self.greedy_reqs.add(req_id) + self.random_reqs.add(req_id) + self.reasoning_temp_split_reqs.add(req_id) + elif sampling_params.sampling_type == SamplingType.GREEDY: # Should avoid division by zero later when apply_temperature. self.temperature_cpu[req_index] = 0.0 self.greedy_reqs.add(req_id) @@ -386,6 +452,22 @@ def add_request( self.temperature_cpu[req_index] = sampling_params.temperature self.random_reqs.add(req_id) + if has_reasoning_split: + self.reasoning_temperature_cpu[req_index] = reasoning_temperature + # Seed the live think state from the prompt; the per-step update + # refines it from generated output. + in_think = self._prompt_in_think(request.prompt_token_ids) + self.prompt_in_think_cpu[req_index] = in_think + self.think_in_think_cpu[req_index] = in_think + else: + # Mirror temperature so the per-step blend is a no-op for rows + # that did not request a split. + self.reasoning_temperature_cpu[req_index] = self.temperature_cpu[ + req_index + ] + self.prompt_in_think_cpu[req_index] = False + self.think_in_think_cpu[req_index] = False + self.top_p_cpu[req_index] = sampling_params.top_p if sampling_params.top_p < 1: self.top_p_reqs.add(req_id) @@ -544,6 +626,7 @@ def remove_request(self, req_id: str) -> int | None: self.greedy_reqs.discard(req_id) self.random_reqs.discard(req_id) + self.reasoning_temp_split_reqs.discard(req_id) self.top_p_reqs.discard(req_id) self.top_k_reqs.discard(req_id) self.frequency_penalties_reqs.discard(req_id) @@ -644,6 +727,18 @@ def swap_states(self, i1: int, i2: int) -> None: self.temperature_cpu[i2], self.temperature_cpu[i1], ) + self.reasoning_temperature_cpu[i1], self.reasoning_temperature_cpu[i2] = ( + self.reasoning_temperature_cpu[i2], + self.reasoning_temperature_cpu[i1], + ) + self.prompt_in_think_cpu[i1], self.prompt_in_think_cpu[i2] = ( + self.prompt_in_think_cpu[i2], + self.prompt_in_think_cpu[i1], + ) + self.think_in_think_cpu[i1], self.think_in_think_cpu[i2] = ( + self.think_in_think_cpu[i2], + self.think_in_think_cpu[i1], + ) self.top_p_cpu[i1], self.top_p_cpu[i2] = self.top_p_cpu[i2], self.top_p_cpu[i1] self.top_k_cpu[i1], self.top_k_cpu[i2] = self.top_k_cpu[i2], self.top_k_cpu[i1] self.frequency_penalties_cpu[i1], self.frequency_penalties_cpu[i2] = ( @@ -772,6 +867,15 @@ def condense(self) -> None: ) self.temperature_cpu[empty_index] = self.temperature_cpu[last_req_index] + self.reasoning_temperature_cpu[empty_index] = ( + self.reasoning_temperature_cpu[last_req_index] + ) + self.prompt_in_think_cpu[empty_index] = self.prompt_in_think_cpu[ + last_req_index + ] + self.think_in_think_cpu[empty_index] = self.think_in_think_cpu[ + last_req_index + ] self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index] self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index] self.frequency_penalties_cpu[empty_index] = self.frequency_penalties_cpu[ @@ -828,6 +932,97 @@ def refresh_metadata(self): if batch_update: self.sampling_metadata = self._make_sampling_metadata() + @staticmethod + def _last_token_sequence_index(token_ids: list[int], seq: list[int]) -> int: + """Find the start index of the last occurrence of *seq* in + *token_ids*, or -1 if absent.""" + if not seq or len(token_ids) < len(seq): + return -1 + for i in range(len(token_ids) - len(seq), -1, -1): + if token_ids[i : i + len(seq)] == seq: + return i + return -1 + + def _prompt_in_think(self, prompt_token_ids: list[int] | None) -> bool: + """Whether *prompt_token_ids* ends inside a thinking section.""" + if not prompt_token_ids: + return False + if not self.think_start_token_ids or not self.think_end_token_ids: + return False + last_start = self._last_token_sequence_index( + prompt_token_ids, self.think_start_token_ids + ) + last_end = self._last_token_sequence_index( + prompt_token_ids, self.think_end_token_ids + ) + return last_start > last_end + + def _update_think_state(self, num_reqs: int) -> None: + """Refresh the per-row think mask in ``think_in_think_cpu``. + + For each split request, find the last think-start and think-end markers + in the generated output: the row is "in think" when a start marker is + the most recent of the two. If neither marker is present yet, fall back + to the prompt's think state. + + NOTE: this rescans the full output each step (O(output_len) per split + row). A scanned-prefix incremental version is unsafe under async + scheduling, where the in-flight token slot holds a placeholder (-1) + that is overwritten with the real token id a step later — advancing a + cursor over that slot skips the real marker. A correct incremental + version would need to track the last *committed* output length; left as + a follow-up optimization. + """ + start = self.think_start_token_ids + end = self.think_end_token_ids + for req_id in self.reasoning_temp_split_reqs: + i = self.req_id_to_index.get(req_id) + if i is None or i >= num_reqs: + continue + out = self.req_output_token_ids[i] + if not out: + self.think_in_think_cpu[i] = bool(self.prompt_in_think_cpu[i]) + continue + last_start = self._last_token_sequence_index(out, start) + last_end = self._last_token_sequence_index(out, end) + if last_start >= 0 or last_end >= 0: + self.think_in_think_cpu[i] = last_start > last_end + else: + self.think_in_think_cpu[i] = bool(self.prompt_in_think_cpu[i]) + + def update_reasoning_temperature(self) -> None: + """Apply the reasoning/answer temperature split for the current step. + + Called every decode step (unlike ``_make_sampling_metadata``, which + only runs on batch changes). Recomputes the live ``temperature`` tensor + the sampler reads as ``where(in_think, reasoning_temperature, + temperature)``, so the effective temperature follows the model across + the think/answer boundary mid-generation. No-op (zero overhead) when no + request in the batch requested a split. + """ + if not self.has_reasoning_temp_split: + return + if not self.think_start_token_ids or not self.think_end_token_ids: + return + num_reqs = self.num_reqs + if num_reqs == 0 or self.all_greedy: + return + + self._update_think_state(num_reqs) + # Reset to the answer-phase base, then blend in reasoning temps where + # the row is currently thinking. Starting from the CPU base each step + # keeps the blend idempotent (no drift from repeated in-place writes). + answer = copy_slice(self.temperature_cpu_tensor, self.temperature, num_reqs) + reasoning = copy_slice( + self.reasoning_temperature_cpu_tensor, + self.reasoning_temperature, + num_reqs, + ) + mask = copy_slice( + self.think_in_think_cpu_tensor, self.think_in_think_gpu, num_reqs + ) + torch.where(mask, reasoning, answer, out=self.temperature[:num_reqs]) + def _make_sampling_metadata(self) -> SamplingMetadata: num_reqs = self.num_reqs if not self.all_greedy: @@ -885,6 +1080,7 @@ def _make_sampling_metadata(self) -> SamplingMetadata: or bool(self.bad_words_token_ids) or self.logitsprocs_need_output_token_ids or thinking_budget_tracks_reqs + or self.has_reasoning_temp_split ) output_token_ids = ( cast(list[list[int]], self.req_output_token_ids) @@ -911,6 +1107,13 @@ def _make_sampling_metadata(self) -> SamplingMetadata: req_index = self.req_id_to_index[req_id] logprob_token_ids_by_index[req_index] = token_ids + # The reasoning/answer temperature split is applied per decode step in + # ``update_reasoning_temperature`` (called after ``refresh_metadata``), + # not here: this metadata is only rebuilt on batch changes, so blending + # here would freeze the phase at admission time. ``temperature`` is the + # answer-phase base; the per-step update overwrites the same tensor + # (``self.temperature[:num_reqs]``, which this view aliases) in place + # before the sampler reads it. return SamplingMetadata( temperature=temperature, all_greedy=self.all_greedy, @@ -1086,6 +1289,10 @@ def update_async_spec_token_ids(self, draft_token_ids: list[list[int]]) -> None: def num_reqs(self) -> int: return len(self.req_id_to_index) + @property + def has_reasoning_temp_split(self) -> bool: + return len(self.reasoning_temp_split_reqs) > 0 + @property def all_greedy(self) -> bool: return len(self.random_reqs) == 0 diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index f82d2224a413..d84428c3ae85 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -3496,6 +3496,7 @@ def _sample( # Update output token ids with tokens sampled in last step # if async scheduling and required by current sampling params. self.input_batch.update_async_output_token_ids() + self.input_batch.update_reasoning_temperature() if spec_decode_metadata is None: return self.sampler( logits=logits,