fix(server): fail-fast unschedulable KV waits

docs/ENVIRONMENT.md

@@ -22,6 +22,7 @@ These change how the engine runs and have no CLI flag (or complement one).
 | `VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES` | `1` (on) | Whether published KV-cache events carry block hashes as an int (the low 64 bits of the sha256 digest) rather than the raw 32 bytes, mirroring vLLM's env of the same name and its default. Set `0` to publish the raw bytes. Only affects the KV-cache event payload (`--kv-events-config`); it does not change the internal block hashing or the cache itself |
 | `VLLM_PLUGINS` | unset (load all registered) | Comma-separated allowlist of general plugins to load in `LoadGeneralPlugins()`, mirroring vLLM's `VLLM_PLUGINS`. Unset loads every registered plugin; an empty string loads none; a list loads only the named plugins. A plugin that throws is logged and skipped (the load never aborts the engine). See [.agents/specs/plugin-system.md](../.agents/specs/plugin-system.md) |
 | `VT_LMCACHE_HOST` | `127.0.0.1` | Default LMCache server host for the `lm://` connector. The `kv_connector_extra_config.host` key overrides it. See [KV-OFFLOAD.md](KV-OFFLOAD.md) |
+| `VT_ENGINE_STEP_LOG` | off | When `1`, EngineCoreProc prints a short per-step heartbeat on stderr (debug). |
 | `VT_LMCACHE_PORT` | `65432` | Default LMCache server port. The `kv_connector_extra_config.port` key overrides it |
 | `VT_LMCACHE_HASH_ALGO` | `blake3` | Default LMCache key-derivation algorithm. Set `vllm` (alias `sha256_cbor`) for byte-for-byte interop with a real vLLM + LMCache peer. The `kv_connector_extra_config.hash_algo` key overrides it |
 | `VT_SERVER_MAX_PROMPT_CHARS` | `200000` characters | Rejects chat-completion prompts larger than this many characters. Set `0` to disable the prompt-size guard |

docs/USAGE.md

@@ -1004,3 +1004,7 @@ were removed when the example became a thin ABI client; see the header comment i
 Served over HTTP too: pass `--video-dit` (plus the VAEs and configs) to `examples/server` and
 `POST /v1/videos`, `POST /v1/videos/sync` and `GET /v1/videos/{id}` register. Without it the
 routes stay unregistered.
+
+## Scheduler fail-fast (KV too small)
+
+If a waiting prompt cannot fit in the configured KV pool (`--num-blocks` / `--kv-cache-memory` / max model len) while nothing is running, the scheduler **aborts immediately** with a finished-aborted status instead of spinning forever at `model_executed=0`. Adequately sized requests are unchanged.

include/vllm/v1/core/sched/scheduler.h

@@ -153,6 +153,14 @@ class Scheduler {
   void finish_requests(const std::string& request_id,
                        RequestStatus finished_status);
 
+
+  // When RUNNING is empty but WAITING requests can never admit (KV pool too
+  // small for full_sequence_must_fit / free blocks), abort the head waiter(s)
+  // so AsyncLLM does not spin forever at model_executed=0. Returns aborted ids
+  // for the engine to emit FinishReason::kAbort to clients. Also tries one
+  // prefix-cache reset when APC is holding the only free blocks.
+  std::vector<std::string> abort_unschedulable_waiting();
+
   // schedule(): the core token-budget algorithm. See the file header.
   SchedulerOutput schedule();
 

src/vllm/v1/core/sched/scheduler.cpp

@@ -1147,4 +1147,93 @@ SchedulerStats Scheduler::make_stats() const {
   return stats;
 }
 
+std::vector<std::string> Scheduler::abort_unschedulable_waiting() {
+  // Deadlock class (lab 2026-08-09): Hermes full SOUL (~35k tok) with default
+  // num_blocks=256 (8k tok capacity) → allocate_slots fails forever while
+  // running is empty → core-step unfinished>0 model_executed=0 spin, GPU idle.
+  std::vector<std::string> aborted;
+  if (!running.empty() || waiting == nullptr || waiting->empty()) {
+    return aborted;
+  }
+
+  // One APC reset attempt: cached prefix blocks can pin the whole pool even
+  // with no RUNNING request.
+  bool tried_reset = false;
+  while (!waiting->empty()) {
+    Request* request = waiting->peek_request();
+    if (request == nullptr) break;
+
+    int num_computed_tokens = 0;
+    // Pure prefix match count — do NOT call get_computed_blocks here (it
+    // records prefix_cache_stats and would double-count every stall step).
+    // Real admission still uses get_computed_blocks in schedule().
+    KVCacheBlocks new_computed_blocks = kv_cache_manager->empty_kv_cache_blocks;
+    if (request->num_computed_tokens == 0) {
+      num_computed_tokens =
+          static_cast<int>(kv_cache_manager->num_matched_prefix_tokens(*request));
+    } else {
+      num_computed_tokens = request->num_computed_tokens;
+    }
+    int num_new_tokens = request->NumTokens() - num_computed_tokens;
+    if (num_new_tokens <= 0) {
+      // Nothing to compute — should not stay waiting; abort defensively.
+      const std::string id = request->request_id;
+      std::cerr << "ERROR schedule: waiting request id=" << id
+                << " has num_new_tokens<=0 — aborting\n";
+      std::cerr.flush();
+      finish_requests(id, RequestStatus::kFinishedAborted);
+      aborted.push_back(id);
+      continue;
+    }
+    if (0 < long_prefill_token_threshold_ &&
+        long_prefill_token_threshold_ < num_new_tokens) {
+      num_new_tokens = long_prefill_token_threshold_;
+    }
+    num_new_tokens = std::min(num_new_tokens, max_num_scheduled_tokens);
+    if (num_new_tokens <= 0) break;
+
+    std::optional<KVCacheBlocks> new_blocks = kv_cache_manager->allocate_slots(
+        *request, num_new_tokens, /*num_new_computed_tokens=*/num_computed_tokens,
+        new_computed_blocks, /*num_lookahead_tokens=*/num_lookahead_tokens_,
+        /*num_external_computed_tokens=*/0, /*delay_cache_blocks=*/false,
+        /*num_encoder_tokens=*/0,
+        /*full_sequence_must_fit=*/scheduler_reserve_full_isl_,
+        /*reserved_blocks=*/0, /*has_scheduled_reqs=*/false);
+    if (new_blocks.has_value()) {
+      // Probe only — free the trial allocation and leave admission to schedule().
+      kv_cache_manager->free(*request);
+      break;
+    }
+
+    if (!tried_reset) {
+      tried_reset = true;
+      if (kv_cache_manager->reset_prefix_cache()) {
+        std::cerr << "INFO schedule: reset prefix cache to free blocks for waiting "
+                     "request id="
+                  << request->request_id << "\n";
+        std::cerr.flush();
+        continue;  // retry same head
+      }
+    }
+
+    const int free_b =
+        static_cast<int>(kv_cache_manager->block_pool.get_num_free_blocks());
+    const int total_b =
+        static_cast<int>(kv_cache_manager->block_pool.num_gpu_blocks);
+    const std::string id = request->request_id;
+    std::cerr << "ERROR schedule: cannot admit waiting id=" << id
+              << " prompt_tokens=" << request->NumTokens()
+              << " max_model_len=" << max_model_len << " free_blocks=" << free_b
+              << "/" << total_b << " block_size=" << block_size_
+              << " (~" << (total_b * std::max(block_size_, 1))
+              << " tok capacity) — aborting. Raise --num-blocks / "
+                 "--kv-cache-memory or shrink the prompt.\n";
+    std::cerr.flush();
+    finish_requests(id, RequestStatus::kFinishedAborted);
+    aborted.push_back(id);
+    // Continue: abort further waiters that also cannot fit (e.g. Hermes retry pile).
+  }
+  return aborted;
+}
+
 }  // namespace vllm::v1

src/vllm/v1/engine/core_proc.cpp

@@ -184,7 +184,30 @@ void EngineCoreProc::process_input_queue() {
 bool EngineCoreProc::process_engine_step() {
   // core.py:1300-1318. "Called only when there are unfinished local requests."
   // core.py:1303: step the engine core.
+  static const bool kStepHb = [] {
+    const char* e = std::getenv("VT_ENGINE_STEP_LOG");
+    if (e != nullptr && e[0] == '1') return true;
+    return false;
+  }();
+  const double t0 = MonotonicSeconds();
+  if (kStepHb) {
+    std::fprintf(stderr,
+                 "INFO core-step begin unfinished=%d finished_pending=%d\n",
+                 scheduler_.get_num_unfinished_requests(),
+                 scheduler_.has_finished_requests() ? 1 : 0);
+    std::fflush(stderr);
+  }
   auto [outputs, model_executed] = (this->*step_fn_)();
+  if (kStepHb) {
+    int n_out = 0;
+    for (const auto& kv : outputs) {
+      n_out += static_cast<int>(kv.second.outputs.size());
+    }
+    std::fprintf(stderr,
+                 "INFO core-step end model_executed=%d n_out=%d elapsed_s=%.3f\n",
+                 model_executed ? 1 : 0, n_out, MonotonicSeconds() - t0);
+    std::fflush(stderr);
+  }
   // core.py:1305-1306: put EngineCoreOutputs into the output queue.
   for (auto& [client_index, engine_core_outputs] : outputs) {
     EngineCoreOutputItem out;
@@ -205,6 +228,16 @@ bool EngineCoreProc::process_engine_step() {
   // double-install, and a step that proposed nothing pulls nullopt.
   post_step(model_executed);
 
+  // Lab reliability: unfinished WAITING that can never admit (KV too small
+  // for Hermes SOUL etc.) used to spin forever at model_executed=0.
+  if (!model_executed && scheduler_.get_num_unfinished_requests() > 0) {
+    std::vector<std::string> aborted = scheduler_.abort_unschedulable_waiting();
+    if (!aborted.empty()) {
+      // finish_requests already ran inside abort_unschedulable_waiting.
+      send_finish_outputs(aborted, FinishReason::kAbort);
+    }
+  }
+
   // core.py:1314: `if not model_executed and self.scheduler.has_requests():`
   // yield briefly (upstream: lets KV-connector background threads take the GIL;
   // here it keeps a 0-token step from hot-spinning). Mirror vLLM EXACTLY — the

tests/vllm/v1/test_scheduler.cpp

@@ -1393,3 +1393,55 @@ TEST_CASE(
   CHECK(out.num_scheduled_tokens.at(req_id) == 1);
   CHECK(out.scheduled_spec_decode_tokens.empty());
 }
+
+// Fail-fast unschedulable waiting (lab 2026-08-09 Hermes SOUL / tiny KV).
+// ---------------------------------------------------------------------------
+TEST_CASE("Scheduler.abort_unschedulable_waiting: aborts prompt that cannot fit KV") {
+  // ~64 token capacity (4 blocks × 16). Full-ISL reserve rejects a 200-token prompt.
+  auto scheduler = CreateScheduler(/*max_num_seqs=*/4, /*max_num_batched_tokens=*/8192,
+                                   /*enable_chunked_prefill=*/true, /*num_blocks=*/4,
+                                   /*block_size=*/16, /*max_model_len=*/8192);
+  auto requests = CreateRequests(/*num_requests=*/1, /*num_tokens=*/200);
+  const std::string id = requests[0]->request_id;
+  AddRequest(*scheduler, std::move(requests[0]));
+
+  CHECK(scheduler->get_num_unfinished_requests() == 1);
+  auto aborted = scheduler->abort_unschedulable_waiting();
+  REQUIRE(aborted.size() == 1);
+  CHECK(aborted[0] == id);
+  CHECK(scheduler->get_num_unfinished_requests() == 0);
+}
+
+TEST_CASE("Scheduler.abort_unschedulable_waiting: leaves admittable waiters alone") {
+  // Plenty of blocks — probe allocate succeeds; nothing aborted.
+  auto scheduler = CreateScheduler(/*max_num_seqs=*/4, /*max_num_batched_tokens=*/8192,
+                                   /*enable_chunked_prefill=*/true, /*num_blocks=*/1000,
+                                   /*block_size=*/16, /*max_model_len=*/8192);
+  auto requests = CreateRequests(/*num_requests=*/2, /*num_tokens=*/32);
+  AddRequest(*scheduler, std::move(requests[0]));
+  AddRequest(*scheduler, std::move(requests[1]));
+
+  auto aborted = scheduler->abort_unschedulable_waiting();
+  CHECK(aborted.empty());
+  CHECK(scheduler->get_num_unfinished_requests() == 2);
+  // schedule() can still admit them
+  auto out = scheduler->schedule();
+  CHECK(out.scheduled_new_reqs.size() == 2);
+}
+
+TEST_CASE("Scheduler.abort_unschedulable_waiting: no-op while running is non-empty") {
+  auto scheduler = CreateScheduler(/*max_num_seqs=*/2, /*max_num_batched_tokens=*/8192,
+                                   /*enable_chunked_prefill=*/true, /*num_blocks=*/1000,
+                                   /*block_size=*/16, /*max_model_len=*/8192);
+  auto ok = CreateRequests(1, /*num_tokens=*/8);
+  AddRequest(*scheduler, std::move(ok[0]));
+  auto out = scheduler->schedule();
+  CHECK(out.scheduled_new_reqs.size() == 1);
+
+  // Add a huge waiter while first is running — abort must not fire (running non-empty).
+  auto huge = CreateRequests(1, /*num_tokens=*/500, {"huge"});
+  AddRequest(*scheduler, std::move(huge[0]));
+  auto aborted = scheduler->abort_unschedulable_waiting();
+  CHECK(aborted.empty());
+  CHECK(scheduler->get_num_unfinished_requests() >= 2);
+}