feat(sampler): SamplingMetadata + LogprobsTensors + make_sampling_metadata (M1.7 Task 1)

Port the host-side sampling types the V1 Sampler consumes:

- include/vllm/v1/sample/metadata.{h} + src/.../metadata.cpp: SamplingMetadata
  (T0 field subset, names 1:1 with vllm/v1/sample/metadata.py). The logitsprocs
  plugin graph is flattened to min_tokens / logit_bias / min_p; logprob_token_ids
  / spec_token_ids / thinking_budget stubbed with upstream cites.
- include/vllm/v1/outputs.{h} + src/.../outputs.cpp: LogprobsTensors
  (logprob_token_ids [n,k+1] i32, logprobs [n,k+1] f32, selected_token_ranks [n]
  i32) + empty_cpu, ported from vllm/v1/outputs.py.
- engine/types.h SamplerOutput: replace opaque optional<bool> logprobs_tensors
  with optional<LogprobsTensors> (field name kept).
- InputBatch::make_sampling_metadata(): port gpu_input_batch.py
  _make_sampling_metadata field-fill order (temperature/top_p/top_k None-gates,
  penalty slices, prompt/output-token-ids penalty gates). generators / max_num_
  logprobs / allowed-ids / bad-words / min-tokens / logit-bias / min-p left at
  faithful defaults with dependency cites (Task 2/3 / M1.8).

Tests: 2-req mixed greedy+random predicates+fields, all-greedy None temperature,
penalties enable prompt+output ids, remove+condense dense-prefix read,
LogprobsTensors shapes, SamplerOutput payload. ctest 48/48 green, warnings-as-errors.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: mudler

CMakeLists.txt

@@ -61,6 +61,8 @@ add_library(vllm STATIC
   src/vllm/tokenizer/bpe.cpp
   src/vllm/tokenizer/tokenizer.cpp
   src/vllm/v1/request.cpp
+  src/vllm/v1/outputs.cpp
+  src/vllm/v1/sample/metadata.cpp
   src/vllm/v1/kv_cache_interface.cpp
   src/vllm/v1/core/kv_cache_utils.cpp
   src/vllm/v1/core/block_pool.cpp

include/vllm/v1/engine/types.h

@@ -19,8 +19,9 @@
 //     external_req_id, reasoning_ended / reasoning_parser_kwargs,
 //     abort_immediately, and the params property. (eos_token_id is NOT a field
 //     upstream either — it rides on sampling_params.eos_token_id.)
-//   SamplerOutput: logprobs_tensors detail (kept as an opaque optional flag
-//     here — the tensor payload lands with the sampler/logprobs unit).
+//   SamplerOutput: logprobs_tensors now carries the real LogprobsTensors payload
+//     (vllm/v1/outputs.py, ported at M1.7); the sampler's gather_logprobs fills
+//     it. It stays std::optional (None => no logprobs requested this step).
 //   ModelRunnerOutput: logprobs (LogprobsLists), prompt_logprobs_dict,
 //     pooler_output, kv_connector_output / ec_connector_output (P/D KV
 //     transfer), num_nans_in_logits, cudagraph_stats, routed_experts, and the
@@ -53,6 +54,7 @@
 #include <vector>
 
 #include "vllm/sampling_params.h"
+#include "vllm/v1/outputs.h"  // vllm::v1::LogprobsTensors (SamplerOutput payload)
 #include "vllm/v1/request.h"  // vllm::v1::FinishReason (reused, not redefined)
 
 namespace vllm::v1 {
@@ -78,9 +80,9 @@ struct EngineCoreRequest {
 struct SamplerOutput {
   // [num_reqs, max_num_generated_tokens]; T0 non-spec decode is [num_reqs, 1].
   std::vector<std::vector<int32_t>> sampled_token_ids;
-  // logprobs_tensors upstream (LogprobsTensors | None). Payload deferred; the
-  // engaged/disengaged state is preserved so downstream branching still ports.
-  std::optional<bool> logprobs_tensors;
+  // logprobs_tensors upstream (LogprobsTensors | None): the sampler's
+  // gather_logprobs payload, None when no logprobs were requested this step.
+  std::optional<LogprobsTensors> logprobs_tensors;
 };
 
 // ModelRunnerOutput (vllm/v1/outputs.py): the runner -> scheduler result,

include/vllm/v1/outputs.h

@@ -0,0 +1,56 @@
+// Ported from: vllm/v1/outputs.py @ e24d1b24
+//
+// The logprobs payload the V1 sampler produces. Upstream `LogprobsTensors` is a
+// NamedTuple of three torch tensors (+ an optional cumulative-count list); the
+// paired numpy form `LogprobsLists` is what crosses the process boundary. At T0
+// there is no torch: each "tensor" is a flat host vector with explicit dims, the
+// closest C++ analogue of the fixed [num_positions, k+1] / [num_positions] shapes.
+//
+// Field NAMES + shapes mirror upstream 1:1:
+//   logprob_token_ids   [num_positions, num_tokens_per_position] int32
+//   logprobs            [num_positions, num_tokens_per_position] float32
+//   selected_token_ranks[num_positions]                          int32
+// where num_tokens_per_position == max_num_logprobs + 1 (the sampled token plus
+// the top-k). `num_positions == num_reqs x num_generated_tokens` (T0 non-spec
+// decode: one position per request).
+//
+// DEFERRED upstream members, intentionally omitted (marked): the
+// `cu_num_generated_tokens` slicing cursor (spec/jump decode — one position per
+// req at T0), and the torch/device helpers tolists / to_cpu_nonblocking /
+// filter / slice_request. The LogprobsLists numpy twin is represented by the
+// same flat-vector layout (the C++ side never re-serializes to numpy), so it is
+// not given a separate type here; add it when the OutputProcessor logprobs path
+// (M1.8) needs the distinct numpy-vs-tensor split.
+#ifndef VLLM_V1_OUTPUTS_H_
+#define VLLM_V1_OUTPUTS_H_
+
+#include <cstdint>
+#include <vector>
+
+namespace vllm::v1 {
+
+// LogprobsTensors (vllm/v1/outputs.py): the (token_ids, logprobs, ranks) triple
+// the sampler's gather_logprobs (M1.7 Task 4) fills. Flat row-major storage;
+// element (pos, j) lives at index pos * num_tokens_per_position + j.
+struct LogprobsTensors {
+  // Leading dim: num_reqs x num_generated_tokens (T0 non-spec: num_reqs).
+  int num_positions = 0;
+  // Trailing dim: max_num_logprobs + 1 (sampled token + top-k).
+  int num_tokens_per_position = 0;
+
+  // [num_positions, num_tokens_per_position]
+  std::vector<int32_t> logprob_token_ids;
+  // [num_positions, num_tokens_per_position]
+  std::vector<float> logprobs;
+  // [num_positions]
+  std::vector<int32_t> selected_token_ranks;
+
+  // Upstream LogprobsTensors.empty_cpu: allocate the (uninitialized) buffers at
+  // the given shape.
+  static LogprobsTensors empty_cpu(int num_positions,
+                                   int num_tokens_per_position);
+};
+
+}  // namespace vllm::v1
+
+#endif  // VLLM_V1_OUTPUTS_H_

include/vllm/v1/sample/metadata.h

@@ -0,0 +1,107 @@
+// Ported from: vllm/v1/sample/metadata.py @ e24d1b24
+//
+// SamplingMetadata — the per-slot sampling state the V1 Sampler (M1.7) consumes,
+// built once per step by InputBatch::make_sampling_metadata() (the port of
+// gpu_input_batch.py::_make_sampling_metadata). Field NAMES mirror upstream 1:1.
+//
+// ─── T0 field subset ────────────────────────────────────────────────────────
+// The scalar/vector fields are the num_reqs-dense slices of the InputBatch
+// per-slot arrays. Upstream keeps them as device tensors sliced `[:num_reqs]`;
+// here each is a host std::vector already truncated to num_reqs. The
+// None-when-not-needed optionals (temperature/top_p/top_k/prompt_token_ids/
+// allowed_token_ids_mask) preserve upstream's "skip the copy" semantics so the
+// sampler's branching ports unchanged.
+//
+// ─── logitsprocs → flat fields (recorded deviation) ─────────────────────────
+// Upstream carries a `logitsprocs: LogitsProcessors` plugin object graph plus
+// `logprob_token_ids`, `spec_token_ids`, `thinking_budget_state_holder`. We do
+// NOT port the plugin interface; instead the three T0 builtins
+// (vllm/v1/sample/logits_processor/builtin.py) are represented as flat inputs —
+// `min_tokens`, `logit_bias`, `min_p` — since Task 3 ports the three builtins
+// directly as functions rather than as a plugin dispatch. The remaining plugin
+// members are marked stubs below (defaulted empty/None) with their upstream cite.
+#ifndef VLLM_V1_SAMPLE_METADATA_H_
+#define VLLM_V1_SAMPLE_METADATA_H_
+
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <set>
+#include <vector>
+
+namespace vllm::v1 {
+
+// Per-request min-tokens state (the flattened MinTokensLogitsProcessor input,
+// vllm/v1/sample/logits_processor/builtin.py::MinTokensLogitsProcessor). While
+// `output_len < min_tokens`, the sampler masks every id in `stop_token_ids`
+// (eos + stop_token_ids, i.e. upstream `params.all_stop_token_ids`) to -inf.
+struct MinTokensState {
+  int min_tokens = 0;
+  std::set<int32_t> stop_token_ids;
+};
+
+// SamplingMetadata (vllm/v1/sample/metadata.py::SamplingMetadata) — T0 subset.
+struct SamplingMetadata {
+  // None when all_greedy (upstream skips the temperature copy). Else [num_reqs].
+  std::optional<std::vector<float>> temperature;
+  bool all_greedy = true;
+  bool all_random = false;
+
+  // None when no_top_p / no_top_k. Else [num_reqs].
+  std::optional<std::vector<float>> top_p;
+  std::optional<std::vector<int32_t>> top_k;
+
+  // req_index -> per-request RNG seed. Upstream is `dict[int, torch.Generator]`;
+  // we don't have torch.Generator, so we carry the seed (the actual seeded RNG
+  // lands in Task 2's random_sample). Requests without their own seed are absent
+  // from the map (upstream NOTE at gpu_input_batch.py:251-252).
+  std::map<int, uint64_t> generators;
+
+  // None => no logprobs; 0 => sampled-token logprob only; k => top-k; -1 => all.
+  std::optional<int> max_num_logprobs;
+
+  bool no_penalties = true;
+  // None unless penalties (or a token-id-consuming proc) need it. Ragged per-req
+  // prompt token ids (upstream: a padded [num_reqs, max_prompt_len] i32 tensor).
+  std::optional<std::vector<std::vector<int32_t>>> prompt_token_ids;
+  // [num_reqs] each (dense slices of the InputBatch penalty arrays).
+  std::vector<float> frequency_penalties;
+  std::vector<float> presence_penalties;
+  std::vector<float> repetition_penalties;
+
+  // Per-request generated tokens so far (empty when no proc needs them, matching
+  // upstream's needs_output_token_ids gate).
+  std::vector<std::vector<int32_t>> output_token_ids;
+
+  // None unless a request restricts allowed ids. Upstream is a 2D bool tensor
+  // [num_reqs, vocab]; represented here as row-major bool rows [num_reqs][vocab].
+  std::optional<std::vector<std::vector<uint8_t>>> allowed_token_ids_mask;
+
+  // req_index -> list of bad-words token-id n-grams
+  // (vllm/v1/sample/ops/bad_words.py::apply_bad_words input).
+  std::map<int, std::vector<std::vector<int32_t>>> bad_words_token_ids;
+
+  // ─── T0 builtin logits-processor inputs (flat; see header deviation) ───────
+  // req_index -> min-tokens state (MinTokensLogitsProcessor).
+  std::map<int, MinTokensState> min_tokens;
+  // req_index -> (token_id -> additive bias) (LogitBiasLogitsProcessor).
+  std::map<int, std::map<int32_t, float>> logit_bias;
+  // [num_reqs] min-p thresholds (MinPLogitsProcessor); 0 disables per row.
+  std::vector<float> min_p;
+
+  // ─── STUBS (marked; defaulted empty/None at T0) ────────────────────────────
+  // Upstream `logitsprocs: LogitsProcessors` plugin graph — NOT ported (the
+  // three T0 builtins are the flat fields above). No field.
+  //
+  // Upstream `logprob_token_ids: dict[int, list[int]] | None` (generative-
+  // scoring: gather logprobs for specific ids). Deferred behind this stub.
+  std::optional<std::map<int, std::vector<int32_t>>> logprob_token_ids;
+  // Upstream `spec_token_ids: list[list[int]] | None` (speculative decode).
+  // Always empty lists at T0 (kept so the sampler's spec branch ports).
+  std::optional<std::vector<std::vector<int32_t>>> spec_token_ids;
+  // Upstream `thinking_budget_state_holder` — deferred; no field/flag ported.
+};
+
+}  // namespace vllm::v1
+
+#endif  // VLLM_V1_SAMPLE_METADATA_H_

include/vllm/v1/worker/gpu/input_batch.h

@@ -77,6 +77,7 @@
 
 #include "vllm/sampling_params.h"
 #include "vllm/v1/core/sched/output.h"  // NewRequestData (from_new_request)
+#include "vllm/v1/sample/metadata.h"    // SamplingMetadata (make_sampling_metadata)
 #include "vllm/v1/worker/gpu/block_table.h"
 
 namespace vllm::v1 {
@@ -150,6 +151,14 @@ class InputBatch {
   // num_reqs (property): len(req_id_to_index).
   int num_reqs() const { return static_cast<int>(req_id_to_index.size()); }
 
+  // make_sampling_metadata: build a SamplingMetadata for the current dense
+  // [0, num_reqs) prefix from the per-slot arrays + predicates. Port of
+  // gpu_input_batch.py::_make_sampling_metadata (M1.5-deferred, landed at M1.7).
+  // Upstream caches this on self.sampling_metadata and rebuilds it whenever the
+  // batch changes (add_request / condense); here it is recomputed on demand (the
+  // caching + refresh_metadata scheduling is the model runner's concern, M1.8).
+  SamplingMetadata make_sampling_metadata() const;
+
   // Sampling predicates the M1.7 sampler keys on (upstream properties).
   bool all_greedy() const { return random_reqs.empty(); }
   bool all_random() const { return greedy_reqs.empty(); }

src/vllm/v1/outputs.cpp

@@ -0,0 +1,23 @@
+// Ported from: vllm/v1/outputs.py @ e24d1b24
+// See include/vllm/v1/outputs.h for scope + the flat-vector layout.
+
+#include "vllm/v1/outputs.h"
+
+#include <cstddef>
+
+namespace vllm::v1 {
+
+LogprobsTensors LogprobsTensors::empty_cpu(int num_positions,
+                                           int num_tokens_per_position) {
+  LogprobsTensors out;
+  out.num_positions = num_positions;
+  out.num_tokens_per_position = num_tokens_per_position;
+  const size_t area = static_cast<size_t>(num_positions) *
+                      static_cast<size_t>(num_tokens_per_position);
+  out.logprob_token_ids.resize(area);
+  out.logprobs.resize(area);
+  out.selected_token_ranks.resize(static_cast<size_t>(num_positions));
+  return out;
+}
+
+}  // namespace vllm::v1

src/vllm/v1/sample/metadata.cpp

@@ -0,0 +1,9 @@
+// Ported from: vllm/v1/sample/metadata.py @ e24d1b24
+//
+// SamplingMetadata is a header-only value carrier (see
+// include/vllm/v1/sample/metadata.h). This translation unit exists so the header
+// is compiled standalone (self-containment check) and has a home in the build.
+// It is built by InputBatch::make_sampling_metadata()
+// (src/vllm/v1/worker/gpu/input_batch.cpp — the port of
+// gpu_input_batch.py::_make_sampling_metadata).
+#include "vllm/v1/sample/metadata.h"

src/vllm/v1/worker/gpu/input_batch.cpp

@@ -219,6 +219,108 @@ int InputBatch::add_request(const CachedRequestState& request) {
   return req_index;
 }
 
+SamplingMetadata InputBatch::make_sampling_metadata() const {
+  // Port of gpu_input_batch.py::_make_sampling_metadata (@ e24d1b24). Fills the
+  // dense [0, num_reqs) prefix, matching upstream's field-fill order + the
+  // "skip the copy when not needed" None/[]-defaults.
+  const int n = num_reqs();
+  const size_t nn = static_cast<size_t>(n);
+  SamplingMetadata md;
+
+  md.all_greedy = all_greedy();
+  md.all_random = all_random();
+  md.no_penalties = no_penalties();
+
+  // temperature: None when all_greedy, else the [:num_reqs] slice
+  // (gpu_input_batch.py:834-839).
+  if (!md.all_greedy) {
+    md.temperature = std::vector<float>(temperature_cpu.begin(),
+                                        temperature_cpu.begin() + nn);
+  }
+  // top_p / top_k: None when the corresponding predicate is empty
+  // (gpu_input_batch.py:919-920).
+  if (!no_top_p()) {
+    md.top_p =
+        std::vector<float>(top_p_cpu.begin(), top_p_cpu.begin() + nn);
+  }
+  if (!no_top_k()) {
+    md.top_k =
+        std::vector<int32_t>(top_k_cpu.begin(), top_k_cpu.begin() + nn);
+  }
+
+  // Penalties are always sliced [:num_reqs] in the returned metadata
+  // (gpu_input_batch.py:925-927); the device-copy is what upstream gates on
+  // no_penalties, not the slice itself.
+  md.frequency_penalties = std::vector<float>(
+      frequency_penalties_cpu.begin(), frequency_penalties_cpu.begin() + nn);
+  md.presence_penalties = std::vector<float>(
+      presence_penalties_cpu.begin(), presence_penalties_cpu.begin() + nn);
+  md.repetition_penalties = std::vector<float>(
+      repetition_penalties_cpu.begin(), repetition_penalties_cpu.begin() + nn);
+
+  // prompt_token_ids: only when penalties (or a token-id-consuming proc, always
+  // false at T0) need them (gpu_input_batch.py:861-876). Ragged per-req prompt
+  // slice of token_ids_cpu[:, :num_prompt_tokens].
+  const bool needs_prompt_token_ids = !md.no_penalties;
+  if (needs_prompt_token_ids) {
+    std::vector<std::vector<int32_t>> prompts(nn);
+    for (int i = 0; i < n; ++i) {
+      const int np = num_prompt_tokens[static_cast<size_t>(i)];
+      const size_t row =
+          static_cast<size_t>(i) * static_cast<size_t>(max_model_len);
+      prompts[static_cast<size_t>(i)].assign(
+          token_ids_cpu.begin() + static_cast<std::ptrdiff_t>(row),
+          token_ids_cpu.begin() + static_cast<std::ptrdiff_t>(row) + np);
+    }
+    md.prompt_token_ids = std::move(prompts);
+  }
+
+  // output_token_ids: only when a proc needs them (gpu_input_batch.py:884-894).
+  // At T0 that is !no_penalties (bad_words / logitsprocs_need_output_token_ids
+  // are always empty/false). Empty [] otherwise, matching upstream.
+  const bool needs_output_token_ids = !md.no_penalties;
+  if (needs_output_token_ids) {
+    md.output_token_ids.resize(nn);
+    for (int i = 0; i < n; ++i) {
+      const auto& row = req_output_token_ids[static_cast<size_t>(i)];
+      if (row.has_value()) {
+        md.output_token_ids[static_cast<size_t>(i)] = *row;
+      }
+    }
+  }
+
+  // spec_token_ids: pass the dense prefix (always empty lists at T0). Upstream
+  // passes self.spec_token_ids directly (gpu_input_batch.py:929).
+  md.spec_token_ids = std::vector<std::vector<int32_t>>(
+      spec_token_ids.begin(), spec_token_ids.begin() + nn);
+
+  // ─── Fields whose InputBatch-side tracking is NOT yet landed (marked) ──────
+  // Each is a faithful upstream default (empty/None) with its dependency cite;
+  // wiring them requires per-slot state this InputBatch does not yet keep.
+  //
+  //  * generators (gpu_input_batch.py:921, sourced :413-414 from
+  //    request.generator == sampling_params.seed): InputBatch keeps no per-req
+  //    generator/seed slot yet. Left empty; the seeded RNG + this map's
+  //    population is a Task-2 (random_sample) dependency. Type is present so
+  //    Task 2 can fill req_index -> seed.
+  //  * max_num_logprobs (gpu_input_batch.py:922 / :1122, from the num_logprobs
+  //    dict populated by sampling_params.logprobs): no num_logprobs tracking
+  //    here — left None (no logprobs). Logprobs wiring is an M1.8 dependency.
+  //  * allowed_token_ids_mask (gpu_input_batch.py:896-904) + bad_words_token_ids
+  //    (:932): no has_allowed_token_ids / bad_words slot tracking — left
+  //    None / empty. A Task-3 / M1.8 dependency (SamplingParams also defers
+  //    allowed_token_ids / bad_words_token_ids).
+  //  * min_tokens / logit_bias / min_p (the T0 builtins,
+  //    logits_processor/builtin.py): InputBatch keeps no per-slot min_p array /
+  //    min_tokens+stop set / logit_bias map yet — left empty. Populating them
+  //    (min_p is on SamplingParams; logit_bias / all_stop_token_ids are
+  //    deferred on SamplingParams) is a Task-3 dependency.
+  // md.generators / max_num_logprobs / allowed_token_ids_mask /
+  // bad_words_token_ids / min_tokens / logit_bias / min_p keep their defaults.
+
+  return md;
+}
+
 std::optional<int> InputBatch::remove_request(const std::string& req_id) {
   const auto it = req_id_to_index.find(req_id);
   if (it == req_id_to_index.end()) {

tests/CMakeLists.txt

@@ -32,6 +32,7 @@ vllm_cpp_add_test(test_sched_output vllm/v1/test_sched_output.cpp)
 vllm_cpp_add_test(test_scheduler vllm/v1/test_scheduler.cpp)
 vllm_cpp_add_test(test_block_table vllm/v1/worker/test_block_table.cpp)
 vllm_cpp_add_test(test_input_batch vllm/v1/worker/test_input_batch.cpp)
+vllm_cpp_add_test(test_sampling_metadata vllm/v1/sample/test_metadata.cpp)
 vllm_cpp_add_test(test_prepare_inputs vllm/v1/worker/test_prepare_inputs.cpp)
 vllm_cpp_add_test(test_common_attn_metadata vllm/v1/attention/test_common_attn_metadata.cpp)
 vllm_cpp_add_test(test_gdn_metadata_builder vllm/v1/attention/test_gdn_metadata_builder.cpp)