diff --git a/c/Makefile b/c/Makefile index 0c4caac29..2f4819434 100644 --- a/c/Makefile +++ b/c/Makefile @@ -439,8 +439,8 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ) $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) -o inkling$(EXE) $(LDFLAGS) -kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h - $(CC) $(CFLAGS) kimi_k3.c -o kimi_k3$(EXE) $(LDFLAGS) +kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h $(VK_OBJ) $(VK_SPV) + $(CC) $(CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(LDFLAGS) # Use a baseline that matches the compiler target. macOS already targets a # portable baseline when ARCH is empty; forcing the x86 value there breaks diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c index 9d22d2df4..d779607d8 100644 --- a/c/backend_vulkan.c +++ b/c/backend_vulkan.c @@ -211,7 +211,8 @@ static int rowwords(int fmt, int I) { * (one f32 per 64-input group). upload_tensor and tensor_free must agree on this. */ static size_t scale_floats(int fmt, int I, int O, int gs) { if (fmt == 5) return (size_t)O * (((size_t)I + 63) / 64); - if (fmt == 4) return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng] + if (fmt == 4 || fmt == 7) + return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng] return (size_t)O; } @@ -506,11 +507,11 @@ static int arena_suballoc(size_t bytes, VkBuffer *buf, void **ptr) { static int upload_tensor(ColiVkTensor **out, const void *weights, const float *scales, int fmt, int I, int O, int gs) { if (*out) return (*out)->fmt == fmt && (*out)->I == I && (*out)->O == O; - if (fmt != 1 && fmt != 2 && fmt != 5 && - !(fmt == 4 && gs >= 8 && gs % 8 == 0)) return 0; /* fmt=4: word-aligned groups only */ + if (fmt != 1 && fmt != 2 && fmt != 5 && /* fmt=4/7: word-aligned groups only */ + !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0)) return 0; ColiVkTensor *t = calloc(1, sizeof(*t)); if (!t) return 0; - t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = fmt == 4 ? gs : 0; + t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0; size_t stride = (size_t)t->rowWords * 4; // padded row bytes size_t cpu_rb = fmt == 1 ? (size_t)I : fmt == 5 ? ((size_t)I + 63) / 64 * 24 : (size_t)(I + 1) / 2; @@ -923,7 +924,7 @@ static int upload_tensor_d2(ColiVkTensor **out, const void *weights, const float !(fmt == 4 && gs >= 8 && gs % 8 == 0)) return 0; ColiVkTensor *t = calloc(1, sizeof(*t)); if (!t) return 0; - t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = fmt == 4 ? gs : 0; + t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0; t->dev = 1; size_t stride = (size_t)t->rowWords * 4; size_t cpu_rb = fmt == 1 ? (size_t)I diff --git a/c/kimi_k3.c b/c/kimi_k3.c index 4ea1e2882..24df36ef1 100644 --- a/c/kimi_k3.c +++ b/c/kimi_k3.c @@ -40,6 +40,13 @@ * K3_MLA_BITS=8|4|32 MLA projections (default 8) * K3_HEAD_BITS=8|4|32 lm_head (default 8) * K3_EXPERT_GB=N routed-expert LRU cache budget (default 8) + * K3_VK=0|1 Vulkan tier (build with `make VK=1 kimi_k3`; default + * 1 when built): shared experts VRAM-resident + a + * fill-once native-MXFP4 routed-expert tier; resident + * experts skip disk AND CPU at decode. CPU fallback + * everywhere; output identical. + * K3_VK_GB=N VRAM cap for the tier (default: driver budget) + * K3_VK_UP=N routed-expert uploads per step (default 8) * K3_DIRECT=0|1 O_DIRECT expert reads (default 1; buffered fallback) * K3_IDOT=0|1 int8-activation expert matmuls (default 1; 0 = float) * K3_PIPE=0|1 overlap expert loads with compute (default 1) @@ -94,7 +101,8 @@ typedef struct { } Cfg; /* ---------- RAM-resident weight, quantized at load ---------- */ -typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; } W; +typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; + void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */ } W; typedef struct { /* KDA layer */ W q, k, v, o, g; @@ -174,7 +182,62 @@ static void rmsnorm_(float *out, const float *x, const float *w, int D, float ep } /* ---------- W: load-time quantization + matvec ---------- */ +/* ---------- Vulkan tier (K3_VK, build with `make VK=1 kimi_k3`) ---------- + * Two residency classes on the card, both with transparent CPU fallback: + * - dense W tensors uploaded once at init (shared experts first): computed + * by the existing fmt=1/4 shaders whenever S==1 (w_matmul hook below); + * - a fill-once routed-expert tier in fmt=7 (native MXFP4, never re-encoded): + * experts enter from freshly-read RAM slots (K3_VK_UP per step) until the + * VRAM budget (K3_VK_GB) is reached; resident experts then skip BOTH the + * disk read and the CPU matmuls at decode (C==1). */ +static int g_k3_vk=0; /* backend live (K3_VK=0 disables) */ +#ifdef COLI_VULKAN +#include "backend_vulkan.h" +typedef struct { void *w1, *w2, *w3; } VkExp; /* ColiVkTensor* triple */ +static VkExp *g_vkexp; static int64_t g_vkexp_n; +static int g_vk_upcap=8, g_vk_up_left=0, g_vk_full=0; +static long g_vk_hit=0, g_vk_res=0; +static double g_vk_gb=0; /* K3_VK_GB cap (0 = driver budget) */ +static const char *k3_vk_spv(char *buf, size_t n){ + const char *env=getenv("COLI_VK_SHADERS"); + struct stat st; + if(env&&*env){ + if(!stat(env,&st)&&S_ISDIR(st.st_mode)){ snprintf(buf,n,"%s/qmatmul.spv",env); return buf; } + return env; + } +#ifdef __linux__ + ssize_t k=readlink("/proc/self/exe",buf,n-1); + if(k>0){ + buf[k]=0; + char *sl=strrchr(buf,'/'); + if(sl&&(size_t)(sl+1-buf)+sizeof("shaders/qmatmul.spv")<=n){ + strcpy(sl+1,"shaders/qmatmul.spv"); + if(!stat(buf,&st)) return buf; + } + } +#endif + return "shaders/qmatmul.spv"; +} +static int vk_budget_full(void){ + double used=0,budget=0; + if(!coli_vk_mem_budget(&used,&budget)) return 0; + double cap=budget-0.5; if(g_vk_gb>0&&g_vk_gb=cap; +} +static int w_vk_upload(W *w){ + if(w->vk) return 1; + int fmt = w->fmt==1?1 : w->fmt==4?4 : -1; + if(fmt<0) return 0; + return coli_vk_tensor_ensure((ColiVkTensor**)&w->vk, + fmt==1?(const void*)w->q8:(const void*)w->q4,w->s,fmt,w->I,w->O,w->gs); +} +#endif static void w_matmul(float *y, const float *x, const W *w, int S){ +#ifdef COLI_VULKAN + if(g_k3_vk&&S==1&&w->vk&& + coli_vk_matmul((ColiVkTensor**)&((W*)w)->vk,y,x,NULL,NULL,w->fmt,1,w->I,w->O,w->gs)) + return; +#endif if(w->fmt==0) matmul(y,x,w->f,S,w->I,w->O); else if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O); else if(w->fmt==4) matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs); @@ -519,6 +582,35 @@ static void model_init(Model *m, const char *snap, int n_layers_env){ free(rn); free(rp); } w_load(m,&m->lm_head,"lm_head.weight",c->vocab,c->hidden,hbits); } else fprintf(stderr,"[K3] final norm/lm_head not present — trace-only mode\n"); +#ifdef COLI_VULKAN + { const char *ev=getenv("K3_VK"); + if(!ev||atoi(ev)){ + char sbuf[512]; const char *spv=k3_vk_spv(sbuf,sizeof(sbuf)); + g_k3_vk=coli_vk_init(spv); + if(!g_k3_vk) fprintf(stderr,"[K3-VK] Vulkan unavailable (tried %s) — CPU only\n",spv); + } + if(g_k3_vk){ + g_vk_gb=getenv("K3_VK_GB")?atof(getenv("K3_VK_GB")):0; + g_vk_upcap=getenv("K3_VK_UP")?atoi(getenv("K3_VK_UP")):8; + g_vkexp_n=(int64_t)c->n_layers*c->n_experts; + g_vkexp=calloc((size_t)g_vkexp_n,sizeof(VkExp)); + if(!g_vkexp) g_k3_vk=0; + } + if(g_k3_vk){ + /* dense residency: shared experts first — they run every token and are + * the biggest always-on RAM-bandwidth slice that fits VRAM */ + int nsh=0; + for(int i=0;in_layers&&!vk_budget_full();i++){ + if(!m->L[i].sparse) continue; + Moe *sm2=&m->L[i].moe; + nsh+=w_vk_upload(&sm2->sh_gate)+w_vk_upload(&sm2->sh_up)+w_vk_upload(&sm2->sh_down); + } + double used=0,budget=0; coli_vk_mem_budget(&used,&budget); + fprintf(stderr,"[K3-VK] resident: %d shared-expert mats (%.1f/%.1f GB); routed MXFP4 tier fills at decode (K3_VK_UP=%d/step, cap %s)\n", + nsh,used,budget,g_vk_upcap,g_vk_gb>0?"K3_VK_GB":"driver budget"); + } + } +#endif expert_table_init(m); /* expert LRU cache, per-layer slots from the global budget */ double egb = getenv("K3_EXPERT_GB")?atof(getenv("K3_EXPERT_GB")):8.0; @@ -530,7 +622,8 @@ static void model_init(Model *m, const char *snap, int n_layers_env){ * regardless of K3_EXPERT_GB. */ if(cap<1) cap=1; if(cap>c->n_experts) cap=c->n_experts; - m->ecache=calloc(c->n_layers,sizeof(LCache)); + { int ncl=c->n_layers>0?c->n_layers:1; + m->ecache=calloc((size_t)ncl,sizeof(LCache)); } for(int i=0;in_layers;i++) if(m->L[i].sparse){ m->ecache[i].cap=cap; m->ecache[i].s=calloc(cap,sizeof(Slot)); for(int j2=0;j2ecache[i].s[j2].eid=-1; @@ -775,6 +868,59 @@ static void expert_apply(Model *m, Slot *s, const float *z, float wk, for(int i=0;ilatent;i++) u[i]+=wk*hz[i]; } +#ifdef COLI_VULKAN +/* GPU apply for a tier-resident expert (decode, S==1): w1/w3 in one paired + * submit, SiTU-GLU on CPU, w2 down. Returns 0 untouched on any failure so + * the caller can run the normal disk+CPU path. */ +static int vk_expert_apply(Model *m, int li, int eid, const float *z, float wk, + float *u, float *gate, float *up, float *hz){ + Cfg *c=&m->c; + VkExp *v=&g_vkexp[(int64_t)li*c->n_experts+eid]; + if(!v->w1||!v->w2||!v->w3) return 0; + if(!coli_vk_matmul_pair((ColiVkTensor**)&v->w1,gate,NULL,NULL,c->moe_inter, + (ColiVkTensor**)&v->w3,up,NULL,NULL,c->moe_inter, + 7,z,1,c->latent,32)) return 0; + for(int i=0;imoe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2); + if(!coli_vk_matmul((ColiVkTensor**)&v->w2,hz,gate,NULL,NULL,7,1,c->moe_inter,c->latent,32)) + return 0; + for(int i=0;ilatent;i++) u[i]+=wk*hz[i]; + g_vk_hit++; + return 1; +} +/* Fill the tier from a freshly-read RAM slot (main thread only). The ue8m0 + * exponents expand to f32 group scales at upload (the shader is float-only). */ +static void vk_expert_try_upload(Model *m, int li, int eid, Slot *s){ + if(!g_k3_vk||g_vk_full||g_vk_up_left<=0) return; + VkExp *v=&g_vkexp[(int64_t)li*m->c.n_experts+eid]; + if(v->w1) return; + if(vk_budget_full()){ g_vk_full=1; + fprintf(stderr,"[K3-VK] expert tier full: %ld experts resident\n",g_vk_res); + return; } + uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p, + *w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p; + int LT=m->c.latent, MI=m->c.moe_inter; + int64_t n1=m->e_w1s, n2=m->e_w2s; /* scale counts = scale bytes (u8) */ + float *sc=falloc(n1>n2?n1:n2); + int ok=1; + for(int64_t i=0;iw1,w1p,sc,7,LT,MI,32); + if(ok){ for(int64_t i=0;iw2,w2p,sc,7,MI,LT,32); } + if(ok){ for(int64_t i=0;iw3,w3p,sc,7,LT,MI,32); } + free(sc); + if(!ok){ + if(v->w1){ coli_vk_tensor_free(v->w1); v->w1=NULL; } + if(v->w2){ coli_vk_tensor_free(v->w2); v->w2=NULL; } + if(v->w3){ coli_vk_tensor_free(v->w3); v->w3=NULL; } + g_vk_full=1; + fprintf(stderr,"[K3-VK] expert tier full: %ld experts resident\n",g_vk_res); + return; + } + g_vk_res++; g_vk_up_left--; +} +#endif + /* ---------- async loader pool (K3_PIPE): expert preads overlap compute ---- * A batch of jobs is submitted per token+layer; the compute loop below waits * per-expert on its ready flag, so expert j's math runs while j+1.. load. @@ -869,6 +1015,9 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids, usleep(50); m->t_eload+=now_s()-t0; } +#ifdef COLI_VULKAN + if(g_k3_vk&&qof[j]>=0) vk_expert_try_upload(m,li,uids[base+j],use[j]); +#endif int f=pfirst[base+j]; for(int p2=0;p2contig){ if(er->fd[0]>=0) posix_fadvise(er->fd[0],er->off[0],m->e_slot,POSIX_FADV_WILLNEED); } else for(int k2=0;k2<6;k2++) if(er->fd[k2]>=0) posix_fadvise(er->fd[k2],er->off[k2],sizes[k2],POSIX_FADV_WILLNEED); } +#ifdef COLI_VULKAN + /* decode: tier-resident experts run on the GPU and drop out of the + * disk union — that skip is the I/O saving. C>1 prefill stays on the + * CPU-batched path (and still feeds the tier via the upload hook). */ + if(g_k3_vk&&C==1){ + int keep=0; + for(int j=0;jc; int D=c->hidden; +#ifdef COLI_VULKAN + g_vk_up_left=g_vk_upcap; /* routed-tier upload budget per step */ +#endif int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs; float *hidden=falloc((int64_t)C*D), *bres=falloc((int64_t)C*nbmax*D); float *prefix=falloc((int64_t)C*D), *nrm=falloc((int64_t)C*D); @@ -1372,6 +1542,10 @@ static void serve_one(Model *m, Tok *T, ServeReq *q){ printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n", dt,np,gen,disk,0.0,moe>disk?moe-disk:moe,m->t_attn-a0,m->t_head-h0,gen+1); fflush(stdout); +#ifdef COLI_VULKAN + if(g_k3_vk) fprintf(stderr,"[K3-VK] routed tier: %ld resident, %ld GPU hits so far\n", + g_vk_res,g_vk_hit); +#endif } static void serve_loop(Model *m, Tok *T){ diff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.comp index a42205856..25de4f201 100644 --- a/c/shaders/qmatmul.comp +++ b/c/shaders/qmatmul.comp @@ -46,6 +46,15 @@ float i8(uint word, int lane) { float i4(uint word, int lane) { return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8); } +// fmt=7 MXFP4 (Kimi K3 experts): e2m1 nibble, bit3 = sign, LOW nibble = even +// column (same packing order as i4). Scales arrive as f32 (host pre-expands +// the ue8m0 exponents), one per 32-input group. +float mx4(uint word, int lane) { + uint n = (word >> (uint(lane) * 4u)) & 0xfu; + const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0); + float v = lut[n & 7u]; + return (n & 8u) != 0u ? -v : v; +} void main() { int s = int(gl_WorkGroupID.y); @@ -97,16 +106,22 @@ void main() { } sum += a * scale[sb + uint(g)]; } - } else if (p.fmt == 4) { // grouped int4 (#298 semantics): nibble decode, one scale - // per gs inputs — gs % 8 == 0 (host-gated), so a packed word never straddles - // a group and its scale multiplies the 8-wide partial. Per-row scale must NOT run. + } else if (p.fmt == 4 || p.fmt == 7) { // grouped int4 (#298 semantics) or MXFP4: + // nibble decode, one scale per gs inputs — gs % 8 == 0 (host-gated), so a + // packed word never straddles a group and its scale multiplies the 8-wide + // partial. Per-row scale must NOT run. int words = (p.I + 7) / 8, ng = (p.I + p.gs - 1) / p.gs; uint sb = uint(o) * uint(ng); for (int wi = lane; wi < words; wi += sgsize) { uint pk = w[rowBase + uint(wi)]; int i0 = wi * 8; float a = 0.0; - for (int k = 0; k < 8; k++) { int i = i0 + k; - if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * i4(pk, k); } + if (p.fmt == 4) { + for (int k = 0; k < 8; k++) { int i = i0 + k; + if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * i4(pk, k); } + } else { + for (int k = 0; k < 8; k++) { int i = i0 + k; + if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * mx4(pk, k); } + } sum += a * scale[sb + uint(i0 / p.gs)]; } } else { // int4 @@ -120,6 +135,6 @@ void main() { } } float tot = subgroupAdd(sum); - if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4) ? tot : tot * scale[o]; + if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7) ? tot : tot * scale[o]; } } diff --git a/c/shaders/qmatmul.spv b/c/shaders/qmatmul.spv new file mode 100644 index 000000000..f64886140 Binary files /dev/null and b/c/shaders/qmatmul.spv differ diff --git a/c/tests/test_vk_mxfp4.c b/c/tests/test_vk_mxfp4.c new file mode 100644 index 000000000..e5faf176e --- /dev/null +++ b/c/tests/test_vk_mxfp4.c @@ -0,0 +1,43 @@ +/* fmt=7 (MXFP4) Vulkan matmul vs the CPU reference kernel (quant.h). + * Random e2m1 nibbles + ue8m0 group exponents, K3 expert dims. + * Skips (exit 0) when no Vulkan device is available. */ +#include +#include +#include +#include +#include "../quant.h" +#include "../backend_vulkan.h" + +int main(void){ + if(!coli_vk_init("shaders/qmatmul.spv")||!coli_vk_available()){ + fprintf(stderr,"vk-mxfp4: no Vulkan device — skipped\n"); return 0; + } + srand(7); + int S=2, I=3584, O=3072; /* K3 expert w1/w3 shape */ + int rb=(I+1)/2, ng=(I+31)/32; + uint8_t *q4=malloc((size_t)O*rb), *e8=malloc((size_t)O*ng); + float *x=malloc((size_t)S*I*sizeof(float)); + float *yc=calloc((size_t)S*O,sizeof(float)), *yg=calloc((size_t)S*O,sizeof(float)); + float *sc=malloc((size_t)O*ng*sizeof(float)); + for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)rand(); + for(size_t i=0;i<(size_t)O*ng;i++){ e8[i]=(uint8_t)(120+rand()%12); sc[i]=mx4_scale(e8[i]); } + for(int i=0;imx)mx=r; + } + double rel=sqrt(num/(den+1e-30)); + printf("vk-mxfp4: rel_l2 %.3e max_rel %.3e (S=%d I=%d O=%d)\n",rel,mx,S,I,O); + if(rel>1e-5){ fprintf(stderr,"vk-mxfp4: FAIL rel_l2 %.3e\n",rel); return 1; } + printf("vk-mxfp4: OK\n"); + coli_vk_shutdown(); + return 0; +} diff --git a/docs/kimi_k3.md b/docs/kimi_k3.md index b9b9e8319..7e1272075 100644 --- a/docs/kimi_k3.md +++ b/docs/kimi_k3.md @@ -154,6 +154,9 @@ Judge quantization choices on real-text logits, not synthetic-vector norms. | `K3_MLA_BITS` | 8 | load-time bits for MLA projections | | `K3_HEAD_BITS` | 8 | load-time bits for lm_head | | `K3_EXPERT_GB` | 8 | routed-expert LRU budget | +| `K3_VK` | 1 | Vulkan tier when built with `make VK=1 kimi_k3` (0 = pure CPU) | +| `K3_VK_GB` | driver budget | VRAM cap for the Vulkan tier | +| `K3_VK_UP` | 8 | routed-expert uploads per step (fill-once tier) | | `K3_DIRECT` | 1 | O_DIRECT expert reads (0 = buffered + WILLNEED) | | `K3_IDOT` | 1 | int8-activation expert matmuls (0 = exact-float kernel) | | `K3_PIPE` | 1 | overlap expert loads with compute (loader threads) | @@ -220,12 +223,37 @@ is returned as `reasoning_content`, response text as `content`, and `<|end_of_msg|>` remains the model-owned stop token. `STOP` and `CANCEL` are honoured between generated tokens. +## Vulkan tier (`make VK=1 kimi_k3`) + +The shared Vulkan backend (`backend_vulkan.c`) gained an **fmt=7 MXFP4** +decode path for K3's expert format — e2m1 nibbles with the ue8m0 exponents +expanded to f32 per-32-group scales at upload, so the QAT bytes are uploaded +exactly as stored and never re-encoded (kernel vs `matmul_mxfp4`: rel_l2 +2.2e-07 on an RX 9070/RADV, 2.6e-07 on llvmpipe; +`tests/test_vk_mxfp4.c`). The engine keeps two residency classes on the +card, both with transparent CPU fallback and identical output: + +- **shared experts**, uploaded once at init (int4/int8, the existing + fmt-1/4 shaders): they run every token and are the largest always-on + dense slice that fits VRAM (7.5 GB for all 92 MoE layers at int4); +- a **fill-once routed-expert tier** in fmt=7: experts enter from + freshly-read RAM slots (`K3_VK_UP` per step) until the VRAM budget + (`K3_VK_GB`) is reached. At decode, tier-resident experts skip **both** + the 17.5 MB disk read and the CPU matmuls (one paired w1/w3 submit, + SiTU-GLU on CPU, w2 down). Chunked prefill stays on the CPU-batched path + and still warms the tier. + +K3's Quantile-Balancing-flat routing caps what any cache tier can do — the +tier's value scales with how long the server lives (fill-once) and with the +measured short-term reuse (temporal locality), not with marginal expert +heat. `K3_VK=0` disables the tier at runtime. + ## Current limitations - Decode is single-token (no speculative decoding — K3 has no MTP head). - Tool declarations/calls and image content are not exposed through the shared gateway yet; unsupported requests fail explicitly. -- CPU only (no CUDA/Metal/Vulkan tier). +- CPU + optional Vulkan tier (no CUDA/Metal). - The protocol, tokenizer, gateway, TUI, and Web client paths are locally testable without the 1.5 TB checkpoint. A release claim still requires one full-model multi-turn TUI/Web run on a host that owns the complete snapshot.