From 0830495b60d2d0cbf374b297ea78f773ee97f8f9 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Thu, 6 Aug 2026 12:24:31 +0800 Subject: [PATCH] A CUDA fallback is a property of the call, not of the tensor Closes the permanence half of #767. @JustVugg's framing there: A one-off allocation failure taking a device out for good, with no way back short of a restart, is the actual defect. The 2 GB per-device headroom being too small for a realistic prompt is what triggers it; the permanence is what makes it bad. The failure this guards is almost always a scratch cudaMalloc under memory pressure, and that pressure scales with S. A boolean cannot say that: it records "this tensor is broken" when what happened is "this width did not fit". So record the width instead: if (coli_cuda_matmul(...)) { w->cuda_fail_s = 0; return; } w->cuda_fail_s = S; and admit a call when S is narrower than the narrowest failure seen. A prefill chunk that OOMs at S=512 no longer condemns decode at S=1, which is exactly the reported symptom -- "throughput collapses and does not recover until restart" -- and it recovers without a restart. A genuine device fault fails at S=1, records 1, and is never retried, because S < 1 is never true. The old permanent behaviour falls out of the general rule as a special case rather than needing a branch of its own: no retry counter, no threshold, no generation number, no error-code classification, and no change to the DLL ABI. The two operator-facing notices said "disabled" and "this run did NOT use the GPU for them". That is no longer true, so they now describe the fallback per width and say that narrower calls still try the GPU -- which is also the "say it out loud" item from the same comment. NOT VERIFIED ON A GPU. I have no CUDA device; triggering this needs a real OOM. CPU build and a -DCOLI_CUDA syntax build are clean and make test-c passes, but the runtime behaviour wants confirmation from someone who can reproduce #767. --- c/colibri.c | 54 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/c/colibri.c b/c/colibri.c index 3368782d..2723e6f9 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -199,7 +199,15 @@ typedef struct { #ifdef COLI_VULKAN ColiVkTensor *vk; int vk_eligible; /* resident on the Vulkan expert tier */ #endif - int cuda_eligible, cuda_failed, cuda_device; /* resident tensor, never a reused expert slot */ + int cuda_eligible, cuda_device; /* resident tensor, never a reused expert slot */ + /* #767: the row count of the smallest call that has failed on this tensor, or 0 if + * none has. A CUDA failure here is almost always a scratch cudaMalloc under memory + * pressure, and that pressure scales with S -- so it is a property of the CALL, not + * of the tensor. Recording S instead of a boolean lets a narrower call retry: a + * prefill chunk that OOMs at S=512 does not condemn decode at S=1. A genuine device + * fault fails at S=1, records 1, and is never retried -- the old behaviour, reached + * as a special case of the general rule rather than as a separate one. */ + int cuda_fail_s; } QT; static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */ int64_t n=(int64_t)t->O*t->I; @@ -519,22 +527,27 @@ static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr; static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES]; static void qt_cuda_reset(QT *t){ if(t->cuda){ coli_cuda_tensor_free(t->cuda); t->cuda=NULL; } - t->cuda_failed=0; + t->cuda_fail_s=0; } -/* #687: a resident tensor that fails to upload is disabled PERMANENTLY and silently - * falls back to CPU. The per-tensor line below is easy to lose in a busy log, and the - * end state -- GPU fully allocated, ~0% useful -- reads as healthy. Count them and say - * it once, loudly, naming the knob that actually fixes it. */ +/* #687: a resident tensor that fails to upload falls back to CPU silently. The + * per-tensor line below is easy to lose in a busy log, and the end state -- GPU fully + * allocated, ~0% useful -- reads as healthy. Count them and say it once, loudly, + * naming the knob that actually fixes it. + * + * Since #767 the fallback is per-width rather than permanent (see cuda_fail_s), so a + * tensor counted here may recover on its own at a narrower S. The count is still worth + * shouting about: it means the tier left too little VRAM for the prompt being run. */ static int g_cuda_disabled_n; static void cuda_disabled_note(void){ enum { CUDA_DISABLED_LOUD_AT = 8 }; if(++g_cuda_disabled_n != CUDA_DISABLED_LOUD_AT) return; fprintf(stderr, - "[CUDA] ***** %d resident tensors have been disabled and moved to CPU *****\n" - "[CUDA] The GPU still holds its expert tier, but the dense/attention path is now\n" - "[CUDA] on CPU: this run will be at or below CPU speed while the card stays\n" - "[CUDA] allocated. The usual cause is the expert tier claiming all VRAM and\n" - "[CUDA] leaving nothing for the lazily-uploaded resident tensors.\n" + "[CUDA] ***** %d resident tensors have fallen back to CPU *****\n" + "[CUDA] The GPU still holds its expert tier, but the dense/attention path is\n" + "[CUDA] running on CPU at this prompt width: expect at or below CPU speed while\n" + "[CUDA] the card stays allocated. They are retried at narrower S, so decode may\n" + "[CUDA] recover even when prefill does not. The usual cause is the expert tier\n" + "[CUDA] claiming all VRAM and leaving nothing for the lazily-uploaded tensors.\n" "[CUDA] Fix: set an explicit CUDA_EXPERT_GB below the auto value (leave room\n" "[CUDA] per device for dense + attention + workspace). See issue #687.\n", CUDA_DISABLED_LOUD_AT); @@ -600,8 +613,8 @@ static void cuda_stats_print(void){ /* #687: say it again at the end -- by now the per-tensor lines are thousands of * log lines back, and this is the number that explains a CPU-speed "GPU" run. */ if(g_cuda_disabled_n) fprintf(stderr, - "[CUDA] %d tensors ran on CPU after failed uploads: this run did NOT use the GPU for " - "them. Lower CUDA_EXPERT_GB (#687).\n", g_cuda_disabled_n); + "[CUDA] %d tensors fell back to CPU after failed uploads at least once. Lower " + "CUDA_EXPERT_GB to leave room for them (#687, #767).\n", g_cuda_disabled_n); if(g_cuda_ndev>1) for(int i=0;icuda_eligible && !w->cuda_failed && w->fmt!=5 && - (w->fmt!=8 || g_cuda_fp8_ready) && !omp_in_parallel()){ + if(g_cuda_enabled && w->cuda_eligible && (!w->cuda_fail_s || S < w->cuda_fail_s) && + w->fmt!=5 && (w->fmt!=8 || g_cuda_fp8_ready) && !omp_in_parallel()){ const void *weights = w->fmt==0 ? (const void*)w->qf : (w->fmt==1||w->fmt==8) ? (const void*)w->q8 : (const void*)w->q4; - if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)) return; - w->cuda_failed=1; + if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device,w->gs)){ + w->cuda_fail_s=0; /* it fits again: forget the width that did not */ + return; + } + w->cuda_fail_s = S; if(g_cuda_disabled_n < 8) /* keep the detail for the first few, then the summary */ - fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d disabled after an error; falling back to CPU\n", - w->O,w->I,w->cuda_device); + fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d fell back to CPU at S=%d; " + "narrower calls will still try the GPU\n", w->O,w->I,w->cuda_device,S); cuda_disabled_note(); } #endif