From 81ef1a9d278ada270929e0c398971bf08a094a86 Mon Sep 17 00:00:00 2001 From: bherald <641463+bherald@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:07:37 -0400 Subject: [PATCH] perf(cache): preserve adaptive LRU during autopin Automatic history pinning took up to half of the expert RAM budget before the adaptive LRU was sized, allowing learned pins to reduce cache capacity and increase expert I/O on constrained systems. Reserve the affordable no-pin LRU capacity before calculating the automatic pin budget. Keep explicit PIN and PIN_GB settings authoritative, and document the shared RAM policy. Add focused budget regression cases. Verify the change with the complete Linux and Windows suites and a balanced fixed-token Windows replay that improves median throughput by 3.85% while reducing expert I/O by 4.5%. --- c/colibri.c | 42 ++++++++++++++++++++++++++++++++++- c/tests/test_cap_precedence.c | 31 ++++++++++++++++++++++++++ docs/ENVIRONMENT.md | 2 +- docs/tuning.md | 4 ++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/c/colibri.c b/c/colibri.c index aa8bc12cc..7362e39cf 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -8267,6 +8267,34 @@ static double expert_avail(Model *m, double ram_gb, int ebits, int max_ctx){ return ram_gb*1e9 - (double)m->resident_bytes - slack; } +/* Automatic history pinning and the adaptive LRU share the expert RAM budget. + * Preserve the LRU capacity affordable before pinning, up to the requested + * cap; explicit PIN/PIN_GB settings bypass this policy and remain authoritative. */ +static double autopin_preserve_lru(double planned_pin, double expert_available, + double lru_reserve){ + if(planned_pin<=0.0 || expert_available<=lru_reserve) return 0.0; + double max_pin=expert_available-lru_reserve; + return planned_pin0.0 && bytes_per_slot>0.0 && requested_cap>0){ + int affordable=(int)(expert_available/bytes_per_slot); + cap=requested_capc.n_layers;i++) if(m->L[i].sparse) nsp++; + if(m->has_mtp) nsp+=2; + return (double)nsp*(double)expert_bytes_probe(m,ebits); +} + /* clampa la cache expert a un budget RAM (GB): cap t.c. residente + cache + slack <= budget. * ram_gb<=0 -> budget AUTO = 88% della RAM disponibile adesso (lascia respiro a OS+wrapper: * sforare = OOM-kill del kernel a meta' generazione, molto peggio di una cache piu' piccola). */ @@ -9307,7 +9335,19 @@ int main(int argc, char **argv){ * sbaglia expert e ruba slot alla LRU adattiva; a regime (>=200k selezioni, * qualche ora di chat) arriva a meta' del budget expert. */ double conf = (double)hist/200000.0; if(conf>1) conf=1; - double pin_gb = expert_avail(&m,ram_env,ebits,est_ctx)*0.5*conf/1e9; + double expert_available=expert_avail(&m,ram_env,ebits,est_ctx); + double planned_pin=expert_available*0.5*conf; + int preserved_cap=0; + double lru_reserve=autopin_lru_reserve( + expert_available,expert_cache_bytes_per_slot(&m,ebits), + m.ecap,&preserved_cap); + double pin_bytes=autopin_preserve_lru( + planned_pin,expert_available,lru_reserve); + if(pin_bytes+1.0=0.5) pin_load(&m, g_usage_path, pin_gb, 0); /* auto-discovered: not trusted */ } /* SEMPRE: senza clamp la LRU cresce fino a cap*76 layer = decine di GB -> OOM-kill. diff --git a/c/tests/test_cap_precedence.c b/c/tests/test_cap_precedence.c index eb7c45d23..19e3b6e00 100644 --- a/c/tests/test_cap_precedence.c +++ b/c/tests/test_cap_precedence.c @@ -26,6 +26,26 @@ static void check(const char *label, int cli_given, int cli, int env, int platfo } } +static void check_autopin(const char *label, double planned, double available, + double lru, double expected){ + double got=autopin_preserve_lru(planned,available,lru); + if(fabs(got-expected)>1e-9){ + fprintf(stderr,"FAIL %-36s -> %.3f (want %.3f)\n",label,got,expected); + failures++; + } +} + +static void check_lru_reserve(const char *label, double available, double slot, + int requested, int expected_cap, double expected_bytes){ + int cap=-1; + double got=autopin_lru_reserve(available,slot,requested,&cap); + if(cap!=expected_cap || fabs(got-expected_bytes)>1e-9){ + fprintf(stderr,"FAIL %-36s -> cap %d, %.3f (want %d, %.3f)\n", + label,cap,got,expected_cap,expected_bytes); + failures++; + } +} + int main(void){ /* bare invocation (argc<=1, no positional): byte-identical to the old * argc>1?atoi(argv[1]):64 fallback on every non-qualifying path. */ @@ -52,6 +72,17 @@ int main(void){ * test pins the precedence contract, not value sanity. */ check("negative CLI still explicit", 1, -1, 0, 1, -1, 1); + check_autopin("nine slots preserve eight",4.5,9.0,8.0,1.0); + check_autopin("sixteen slots keep half",8.0,16.0,8.0,8.0); + check_autopin("twenty slots keep plan",10.0,20.0,8.0,10.0); + check_autopin("insufficient for requested LRU",4.0,6.0,8.0,0.0); + check_autopin("small plan capped",2.0,9.0,8.0,1.0); + check_autopin("zero plan",0.0,9.0,8.0,0.0); + check_lru_reserve("requested cap fits",16.0,1.0,8,8,8.0); + check_lru_reserve("reserve only affordable cap",9.0,2.0,8,4,8.0); + check_lru_reserve("zero slot size",9.0,0.0,8,0,0.0); + check_lru_reserve("zero requested cap",9.0,1.0,0,0,0.0); + if(failures){ fprintf(stderr, "cap precedence tests: %d FAILURE(S)\n", failures); return 1; diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 1d1484f20..a1ecafb69 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -53,7 +53,7 @@ Format: `VAR` — default — effect. | `COLI_MMAP` | `0` | `mmap` the weights instead of read()-ing into slabs. | | `PIN` | unset | Path to a `.coli_usage`/stats file; pins the hottest experts into a resident "hot store" at startup. **`PIN=auto`** seeds from the model dir's live `.coli_usage` (appended after every turn, so each restart's pin placement follows the accumulated real workload) with `stats.txt` as the fallback for a virgin model dir; neither present → no pin this run. | | `PIN_GB` | `10.0` | Size budget (GB) for the pinned hot store when `PIN` is set. | -| `AUTOPIN` | `1` (on) | Auto-pin the hot store from usage history once ≥5000 selections are recorded. | +| `AUTOPIN` | `1` (on) | Auto-pin the hot store from usage history once ≥5000 selections are recorded. Automatic pinning is capped so it cannot reduce the adaptive LRU capacity that fits before pinning; explicit `PIN`/`PIN_GB` settings remain authoritative. | | `REPIN` | `0` (off) | Live re-pin the hot store every N emitted tokens (RFC). | | `PILOT` | `0` (off) | Router-piloted cross-layer expert prefetch. | | `PILOT_REAL` | `0` (off) | Value-preserving real cross-layer prefetch loads (`PILOT_REAL=1` opts in). | diff --git a/docs/tuning.md b/docs/tuning.md index af2d09e1c..10312a0eb 100644 --- a/docs/tuning.md +++ b/docs/tuning.md @@ -28,6 +28,10 @@ is safe on any machine. See also [SETTINGS.md](SETTINGS.md) and | `KVSAVE=0` | disable KV-cache persistence | | `TF=1` | teacher-forcing validation | +Automatic history pinning and the adaptive LRU share the same expert RAM +budget. Colibri caps automatic pinning to preserve the no-pin LRU capacity; +explicit `PIN` and `PIN_GB` settings remain authoritative. + ## Resource policy `coli plan` reports the planned hot (VRAM), warm (RAM), and cold backing (disk)