From 53661f5dfde6e5c801d9e74e1136b74949090996 Mon Sep 17 00:00:00 2001 From: Ben Colsey Date: Mon, 3 Aug 2026 22:07:15 -0400 Subject: [PATCH 1/2] reconstruct PR1: engine RAMMAP/NUMA/telemetry from donor 5f6f31a Co-Authored-By: Claude --- c/Makefile | 13 +- c/backend_cuda.cu | 8 +- c/colibri.c | 1073 +++++++++++++++++++++++---- c/compat.h | 15 +- c/openai_server.py | 250 ++++++- c/resource_plan.py | 105 ++- c/st.h | 51 +- c/telemetry.h | 103 ++- c/tests/test_backend_cuda.cu | 10 + c/tests/test_fp8_e2e_repack_load.py | 8 +- c/tests/test_openai_server.py | 335 ++++++++- c/tests/test_openai_tools_e2e.py | 5 +- c/tests/test_rammap.c | 292 ++++++++ c/tests/test_rammap_e2e.py | 242 ++++++ c/tests/test_resource_masks.c | 172 +++++ c/tests/test_resource_plan.py | 102 ++- c/tests/test_serve_sentinel.c | 13 +- c/tests/test_uring.c | 64 +- docs/cuda.md | 43 ++ docs/serve_protocol.md | 39 +- 20 files changed, 2706 insertions(+), 237 deletions(-) create mode 100644 c/tests/test_rammap.c create mode 100644 c/tests/test_rammap_e2e.py create mode 100644 c/tests/test_resource_masks.c diff --git a/c/Makefile b/c/Makefile index 06f4e8df6..64c3669b1 100644 --- a/c/Makefile +++ b/c/Makefile @@ -665,6 +665,11 @@ tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c colibri.c st.h uring.h json.h tests/test_uring$(EXE): tests/test_uring.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_rammap$(EXE): tests/test_rammap.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + +tests/test_resource_masks$(EXE): tests/test_resource_masks.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) tests/test_pipe_block$(EXE): tests/test_pipe_block.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) @@ -719,9 +724,14 @@ install: colibri$(EXE) olmoe$(EXE) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR) $(INSTALL) -d $(DESTDIR)$(LIBEXECDIR)/tools $(INSTALL) -m 755 coli $(DESTDIR)$(BINDIR)/coli + printf '%s\n' '$(LIBEXECDIR)' > '$(DESTDIR)$(BINDIR)/coli.libexec' + chmod 644 '$(DESTDIR)$(BINDIR)/coli.libexec' $(INSTALL) -m 755 colibri$(EXE) $(DESTDIR)$(LIBEXECDIR)/colibri$(EXE) $(INSTALL) -m 755 olmoe$(EXE) $(DESTDIR)$(LIBEXECDIR)/olmoe$(EXE) - $(INSTALL) -m 644 resource_plan.py doctor.py autotune.py openai_server.py version.py $(DESTDIR)$(LIBEXECDIR)/ + $(INSTALL) -m 644 resource_plan.py doctor.py autotune.py openai_server.py version.py ramdisk.py ramdisk_ui.py ramdisk_textual.py requirements-tui.txt $(DESTDIR)$(LIBEXECDIR)/ + rm -rf "$(DESTDIR)$(LIBEXECDIR)/ramdisk_support" + $(INSTALL) -d -m 755 "$(DESTDIR)$(LIBEXECDIR)/ramdisk_support" + $(INSTALL) -m 644 ramdisk_support/*.py "$(DESTDIR)$(LIBEXECDIR)/ramdisk_support/" $(INSTALL) -m 644 tools/*.py $(DESTDIR)$(LIBEXECDIR)/tools/ @# The dashboard is an optional build artifact (cd web && npm run build), so install @# it only when it exists. It goes NEXT TO openai_server.py, which probes ./web/dist. @@ -735,6 +745,7 @@ install: colibri$(EXE) olmoe$(EXE) uninstall: rm -f $(DESTDIR)$(BINDIR)/coli + rm -f $(DESTDIR)$(BINDIR)/coli.libexec rm -rf $(DESTDIR)$(LIBEXECDIR) clean: diff --git a/c/backend_cuda.cu b/c/backend_cuda.cu index 0ccd1496f..89deadeb4 100644 --- a/c/backend_cuda.cu +++ b/c/backend_cuda.cu @@ -1717,7 +1717,13 @@ extern "C" size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor) { tensor->compressed ? tensor->archive_bytes : #endif tensor->weight_bytes; - return storage_bytes + (tensor->fmt ? (size_t)tensor->O * ng * sizeof(float) : 0); + /* Report exactly what upload charged to the device counter. In + * particular, fmt=6/E8 stores scales inside each weight block and has no + * separate scale allocation. */ + return storage_bytes + + ((tensor->fmt && tensor->fmt != 6) + ? (size_t)tensor->O * ng * sizeof(float) + : 0); } extern "C" int coli_cuda_tensor_device(const ColiCudaTensor *tensor) { diff --git a/c/colibri.c b/c/colibri.c index 4d5ed9ea7..a0facba8c 100644 --- a/c/colibri.c +++ b/c/colibri.c @@ -313,13 +313,25 @@ typedef struct { * VISTE dentro `slab` (una sola pread coalescente); nel fallback hanno buffer propri. * slab_cap/fslab_cap: capienza allocata — gli slot ws[] sono riusati TRA layer e gli * expert non hanno tutti la stessa taglia (layer MTP int8 = 2x i layer int4). */ +enum { + ESLOT_BACKING_NONE=0, + ESLOT_BACKING_OWNED=1, /* slab or individually allocated legacy buffers */ + ESLOT_BACKING_MMAP=2, /* legacy COLI_MMAP page-cache view */ + ESLOT_BACKING_RAMMAP=3 /* immutable tmpfs direct tier (PR #377) */ +}; typedef struct { int eid; QT g,u,d; uint8_t *slab; float *fslab; int64_t slab_cap, fslab_cap; uint64_t used; /* pin-arena backing (#419): when set, slab/fslab are interior * slices of a per-layer arena and must never be free()d — * expert_host_release detaches them, expert_host_ensure * re-attaches. NULL for every individually-allocated slot. */ - uint8_t *aslab; float *afslab; } ESlot; + uint8_t *aslab; float *afslab; + unsigned char backing; } ESlot; /* PR #377: ESLOT_BACKING_* — composes with #419 arena fields */ + +/* A mapped expert has valid host QT views without owning an anonymous slab. */ +static int expert_host_ready(const ESlot *s){ + return s->slab || s->backing==ESLOT_BACKING_RAMMAP || s->backing==ESLOT_BACKING_MMAP; +} typedef struct { float **Lc, **Rc, **Ic; @@ -355,6 +367,9 @@ typedef struct { #endif ESlot ws[64]; /* working set del layer corrente (load paralleli) */ ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */ + ESlot *rammap; /* PR #377: immutable [layer][expert] tmpfs views */ + int rammap_experts; int64_t rammap_bytes; uint64_t rammap_calls; + double rammap_prefault_s; uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */ uint32_t **eheat; /* calore recente per promotion/demotion live */ uint32_t **elast, eaccess_clock; /* recency per LFRU session-local */ @@ -657,7 +672,40 @@ static double rss_gb(void){ * the existing PROFILE line. Additive only: with PROF unset the output of * every mode stays byte-identical. */ static int g_prof=0; -static _Atomic int64_t g_prof_io; /* bytes pread()/faulted from expert files */ +/* Bytes requested from descriptors whose backing filesystem is not tmpfs (PR #377). + * Deliberately separate from RAM-map coverage: hybrid namespaces may send some + * tensors through the ordinary slab/io_uring path while still doing zero physical + * reads for tmpfs-backed descriptors. */ +static _Atomic int64_t g_prof_io; +static int g_rammap=0, g_ram_prefault=0; /* PR #377: COLI_RAMMAP / COLI_RAM_PREFAULT */ +static int rammap_modes_conflict(int legacy_mmap, int rammap){ + return legacy_mmap!=0 && rammap!=0; +} +/* SSD-backed expert bytes for one tensor (PR #377). st_fd_is_tmpfs is ported by the + * st.h region (COLLISION #3 / st_fd_is_tmpfs addition); st_tensor/shards come from st.h. */ +static int64_t prof_ssd_tensor_bytes(shards *S, const st_tensor *t){ + return t && !st_fd_is_tmpfs(S,t->fd) ? t->nbytes : 0; +} +/* Linux exposes bytes actually fetched from the block layer in /proc/self/io (PR #377). + * Process-wide, kept separate from g_prof_io: the latter records requested expert + * bytes (page cache may serve them); read_bytes is the closest dependency-free + * measure of physical storage traffic. */ +static int prof_physical_read_bytes(uint64_t *bytes){ +#ifdef __linux__ + FILE *f=fopen("/proc/self/io","r"); + if(!f) return 0; + char line[256]; unsigned long long value=0; int found=0; + while(fgets(line,sizeof(line),f)){ + if(sscanf(line,"read_bytes: %llu",&value)==1){ found=1; break; } + } + fclose(f); + if(found) *bytes=(uint64_t)value; + return found; +#else + (void)bytes; + return 0; +#endif +} /* Disk service: wall time inside expert_load on whichever thread runs the read * (PIPE I/O workers, OMP loaders, the speculative pilot). It overlaps compute, * so it is NOT a wall-time phase — the stall the compute thread actually felt @@ -736,6 +784,8 @@ typedef struct { uint64_t hit_pin,hit_ecache; uint64_t dc_n[2], dc_direct_n[2]; int64_t dc_bytes[2], dc_ns[2]; /* DISK-CLASS */ int64_t dc_wall_ns[2], dc_wall_all_ns; /* busy-wall (per class + combined) */ + uint64_t physical_read_bytes, rammap_calls; /* RAM-map physical-read telemetry */ + int physical_read_valid; } ProfBase; static void prof_base(Model *m, ProfBase *b){ b->edisk=edisk_s(); b->ewait=m->t_ewait; b->emm=m->t_emm; @@ -746,6 +796,8 @@ static void prof_base(Model *m, ProfBase *b){ b->hit_pin=m->hit_pin; b->hit_ecache=m->hit_ecache; b->n_fw=m->n_fw; b->n_emit=m->n_emit; b->nlat=g_prof_nlat; b->n_p2p=m->n_p2p; b->cpu_bytes=m->cpu_expert_bytes;b->cpu_rows=m->cpu_expert_rows; + b->rammap_calls=m->rammap_calls; + b->physical_read_valid=prof_physical_read_bytes(&b->physical_read_bytes); for(int i=0;i<2;i++){ b->dc_n[i]=atomic_load_explicit(&g_dc_n[i],memory_order_relaxed); b->dc_bytes[i]=atomic_load_explicit(&g_dc_bytes[i],memory_order_relaxed); @@ -754,6 +806,24 @@ static void prof_base(Model *m, ProfBase *b){ } dc_wall_read(b->dc_wall_ns,&b->dc_wall_all_ns); } +static int64_t prof_physical_read_delta(const ProfBase *b){ + uint64_t now; + if(!b->physical_read_valid || !prof_physical_read_bytes(&now) || + nowphysical_read_bytes) return -1; + uint64_t delta=now-b->physical_read_bytes; + return delta>(uint64_t)INT64_MAX ? INT64_MAX : (int64_t)delta; +} +/* The legacy numeric PROF field must remain parseable, but zero has two very + * different meanings: a successful measurement with no block reads, or no + * /proc/self/io measurement at all. Carry an additive validity bit on the + * wire instead of erasing that distinction. Older consumers still see the + * conservative numeric zero; validity-aware consumers map valid=0 to null. */ +typedef struct { int64_t bytes; int valid; } ProfPhysicalWire; +static ProfPhysicalWire prof_physical_wire(int64_t delta){ + ProfPhysicalWire w={0,0}; + if(delta>=0){ w.bytes=delta; w.valid=1; } + return w; +} static float *falloc(int64_t n){ /* guardia anti-wrap (report PR #25): n assurdo da file modello ostili non deve @@ -939,7 +1009,8 @@ static int g_route_m=12; /* ROUTE_M: max-rank window for cache-preferring fi static float g_route_p=0; /* ROUTE_P: if >0, choose M from cumulative router mass instead */ static float g_route_alpha=1.f; /* ROUTE_ALPHA: scale gate mass of CACHE_ROUTE substitutes before renorm (1=off) */ static int g_route_agree=0; /* ROUTE_AGREE=1: footer overlap% + mean KL vs true top-K */ -static int expert_is_resident(Model *m, int layer, int eid); /* pin∪LRU; defined near pilot */ +static ESlot *expert_resident_slot(Model *m, int layer, int eid, int touch); /* RAM map -> pin -> LRU */ +static int expert_is_resident(Model *m, int layer, int eid); /* defined near moe() */ static int g_spec=1; /* metodo C: SPEC=0 disabilita il prefetch speculativo cross-layer */ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward via n-gram lookup * (0=off). LOSSLESS: verifica = output identico al greedy. Default OFF: @@ -1035,48 +1106,364 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD ( * every bind here lands before the pread that first-touches the pages, so * there is nothing to migrate. */ #ifdef __linux__ -static int g_numa_nodes=0; /* only touched under __linux__; off-Linux NUMA is a no-op */ +/* Linux CPU/node lists use the same grammar: N or N-M terms separated by + * commas. Keep the mask dynamically sized so node 65 is represented by bit + * 65, not mistaken for "the 66th contiguous node". The runtime callers cap + * untrusted environment input to the kernel's possible/online domain; the + * generous absolute cap only bounds a corrupt sysfs file. */ +#define COLI_IDMASK_MAX_ID 1048575UL +typedef struct { + unsigned long *words; + size_t nwords, count; + unsigned long maxnode; /* kernel ABI: highest selected ID + 1 */ +} ColiIdMask; + +static void coli_idmask_free(ColiIdMask *m){ + if(!m) return; + free(m->words); + *m=(ColiIdMask){0}; +} + +static int coli_idmask_parse_uint(const char **cursor,unsigned long max_id, + unsigned long *value){ + const char *p=*cursor; + if(!p || *p<'0' || *p>'9'){ errno=EINVAL; return -1; } + unsigned long v=0; + do { + unsigned long digit=(unsigned long)(*p-'0'); + if(v>max_id/10 || (v==max_id/10 && digit>max_id%10)){ + errno=ERANGE; return -1; + } + v=v*10+digit; p++; + } while(*p>='0' && *p<='9'); + *cursor=p; *value=v; return 0; +} + +static int coli_idmask_parse_pass(const char *spec,unsigned long max_id, + unsigned long *words,size_t nwords, + size_t *count,unsigned long *maxnode){ + const unsigned long word_bits=(unsigned long)(sizeof(unsigned long)*CHAR_BIT); + const char *p=spec; + size_t selected=0; + unsigned long high=0; + if(!p || !*p){ errno=EINVAL; return -1; } + for(;;){ + unsigned long first,last; + if(coli_idmask_parse_uint(&p,max_id,&first)) return -1; + last=first; + if(*p=='-'){ + p++; + if(coli_idmask_parse_uint(&p,max_id,&last)) return -1; + if(last=nwords){ errno=ERANGE; return -1; } + if(words[wi]&bit){ errno=EINVAL; return -1; } + words[wi]|=bit; + } + if(selected==(size_t)-1){ errno=EOVERFLOW; return -1; } + selected++; + if(id+1>high) high=id+1; + if(id==last) break; + } + if(!*p) break; + if(*p!=','){ errno=EINVAL; return -1; } + p++; + if(!*p){ errno=EINVAL; return -1; } + } + if(count) *count=selected; + if(maxnode) *maxnode=high; + return 0; +} + +/* `out` must be empty. On every failure it remains empty, which lets managed + * configuration fail closed without accidentally retaining a previous mask. */ +static int coli_idmask_parse(const char *spec,unsigned long max_id,ColiIdMask *out){ + const unsigned long word_bits=(unsigned long)(sizeof(unsigned long)*CHAR_BIT); + unsigned long maxnode=0; + if(!out){ errno=EINVAL; return -1; } + *out=(ColiIdMask){0}; + if(max_id==ULONG_MAX) max_id=ULONG_MAX-1; /* maxnode must represent id+1 */ + if(coli_idmask_parse_pass(spec,max_id,NULL,0,NULL,&maxnode)) return -1; + if(!maxnode){ errno=EINVAL; return -1; } + size_t nwords=(size_t)((maxnode-1)/word_bits)+1; + if(nwords>(size_t)-1/sizeof(unsigned long)){ errno=EOVERFLOW; return -1; } + unsigned long *words=(unsigned long*)calloc(nwords,sizeof(unsigned long)); + if(!words) return -1; + size_t count=0; + if(coli_idmask_parse_pass(spec,max_id,words,nwords,&count,&maxnode)){ + int saved=errno; free(words); errno=saved; return -1; + } + out->words=words; out->nwords=nwords; out->count=count; out->maxnode=maxnode; + return 0; +} + +static int coli_idmask_has(const ColiIdMask *m,unsigned long id){ + const unsigned long word_bits=(unsigned long)(sizeof(unsigned long)*CHAR_BIT); + size_t wi=(size_t)(id/word_bits); + return m && winwords && !!(m->words[wi]&(1UL<<(id%word_bits))); +} + +static int coli_idmask_is_subset(const ColiIdMask *selected,const ColiIdMask *domain){ + if(!selected || !domain) return 0; + for(size_t i=0;inwords;i++){ + unsigned long allowed=inwords?domain->words[i]:0; + if(selected->words[i]&~allowed) return 0; + } + return 1; +} + +static int coli_idmask_read_file(const char *path,unsigned long max_id,ColiIdMask *out){ + char buf[4096]; + FILE *fp=fopen(path,"r"); + if(!fp) return -1; + if(!fgets(buf,sizeof(buf),fp)){ + int saved=ferror(fp)?errno:EINVAL; fclose(fp); errno=saved; return -1; + } + size_t n=strlen(buf); + if(n==sizeof(buf)-1 && buf[n-1]!='\n' && !feof(fp)){ + fclose(fp); errno=EOVERFLOW; return -1; + } + if(fclose(fp)) return -1; + while(n && (buf[n-1]=='\n'||buf[n-1]=='\r')) buf[--n]=0; + return coli_idmask_parse(buf,max_id,out); +} + +static int coli_idmask_read_status_field(const char *field,unsigned long max_id, + ColiIdMask *out){ + char line[4096]; + size_t field_len=strlen(field); + FILE *fp=fopen("/proc/self/status","r"); + if(!fp) return -1; + while(fgets(line,sizeof(line),fp)){ + size_t n=strlen(line); + if(n==sizeof(line)-1 && line[n-1]!='\n' && !feof(fp)){ + fclose(fp); errno=EOVERFLOW; return -1; + } + if(strncmp(line,field,field_len) || line[field_len]!=':') continue; + char *spec=line+field_len+1; + while(*spec==' '||*spec=='\t') spec++; + while(n && (line[n-1]=='\n'||line[n-1]=='\r')) line[--n]=0; + if(fclose(fp)) return -1; + return coli_idmask_parse(spec,max_id,out); + } + { + int saved=ferror(fp)?(errno?errno:EIO):ENOENT; + fclose(fp); errno=saved; return -1; + } +} + +/* `/sys/.../online` is authoritative and already uses range-list syntax. The + * bounded directory scan is only the old-kernel fallback; unlike the former + * loop it does not stop at the first sparse node ID. */ +static int coli_numa_online_mask(ColiIdMask *out){ + if(!coli_idmask_read_file("/sys/devices/system/node/online", + COLI_IDMASK_MAX_ID,out)) return 0; + char spec[256]={0}; + size_t used=0; + int found=0; + for(int i=0;i<64;i++){ + char path[64]; + struct stat st; + snprintf(path,sizeof(path),"/sys/devices/system/node/node%d",i); + if(stat(path,&st)) continue; + int wrote=snprintf(spec+used,sizeof(spec)-used,"%s%d",found?",":"",i); + if(wrote<0 || (size_t)wrote>=sizeof(spec)-used){ errno=EOVERFLOW; return -1; } + used+=(size_t)wrote; found=1; + } + if(!found){ errno=ENOENT; return -1; } + return coli_idmask_parse(spec,63,out); +} + +/* Apply and then read back a managed CPU mask. sched_setaffinity may silently + * intersect a request with offline CPUs or a cgroup cpuset, so success alone is + * not an exact-placement guarantee; byte-for-byte readback is required. */ +static int coli_cpu_affinity_apply(const char *spec){ +#if defined(SYS_sched_setaffinity) && defined(SYS_sched_getaffinity) + const unsigned long word_bits=(unsigned long)(sizeof(unsigned long)*CHAR_BIT); + ColiIdMask possible={0},requested={0}; + unsigned long max_id=0; + size_t kernel_words=0; + int have_possible=!coli_idmask_read_file("/sys/devices/system/cpu/possible", + COLI_IDMASK_MAX_ID,&possible); + if(have_possible){ + if(!possible.maxnode){ errno=EINVAL; goto fail; } + max_id=possible.maxnode-1; + kernel_words=possible.nwords; + } else { + long ncpu=sysconf(_SC_NPROCESSORS_CONF); + if(ncpu<1){ errno=ENODEV; goto fail; } + max_id=(unsigned long)ncpu-1; + kernel_words=(size_t)(((unsigned long)ncpu-1)/word_bits)+1; + } + if(coli_idmask_parse(spec,max_id,&requested)) goto fail; + if(have_possible && !coli_idmask_is_subset(&requested,&possible)){ + errno=EINVAL; goto fail; + } + if(kernel_words>(size_t)-1/sizeof(unsigned long)){ errno=EOVERFLOW; goto fail; } + size_t bytes=kernel_words*sizeof(unsigned long); + unsigned long *wanted=(unsigned long*)calloc(kernel_words,sizeof(unsigned long)); + unsigned long *actual=(unsigned long*)calloc(kernel_words,sizeof(unsigned long)); + if(!wanted || !actual){ + int saved=errno; free(wanted); free(actual); errno=saved; goto fail; + } + memcpy(wanted,requested.words,requested.nwords*sizeof(unsigned long)); + if(syscall(SYS_sched_setaffinity,0,bytes,wanted)<0){ + int saved=errno; + free(wanted); free(actual); errno=saved; goto fail; + } + if(syscall(SYS_sched_getaffinity,0,bytes,actual)<0){ + int saved=errno; + free(wanted); free(actual); errno=saved; goto fail; + } + if(memcmp(wanted,actual,bytes)){ + int saved=EPERM; /* an offline CPU or cpuset silently narrowed the request */ + free(wanted); free(actual); errno=saved; goto fail; + } + free(wanted); free(actual); + coli_idmask_free(&requested); + coli_idmask_free(&possible); + return 0; +fail: + { + int saved=errno?errno:EINVAL; + coli_idmask_free(&requested); + coli_idmask_free(&possible); + errno=saved; + return -1; + } +#else + (void)spec; + errno=ENOTSUP; + return -1; +#endif +} + +#define COLI_MPOL_BIND 2 +#define COLI_MPOL_INTERLEAVE 3 +#define COLI_MPOL_F_STATIC_NODES (1<<15) +static int coli_numa_policy_mode(size_t selected_count,int explicit_nodes){ + int mode=selected_count==1 && explicit_nodes + ?COLI_MPOL_BIND:COLI_MPOL_INTERLEAVE; + return mode|(explicit_nodes?COLI_MPOL_F_STATIC_NODES:0); +} +static ColiIdMask g_numa_mask={0}; +static int g_numa_nodes=0; /* selected-node count; IDs live in g_numa_mask */ +static int g_numa_mbind_mode=COLI_MPOL_INTERLEAVE; static int g_numa_skip_bind=0; /* raised around the GPU-prefix pin load: those slabs are * upload staging, freed right after — binding them buys * nothing and costs ~2 transient VMAs each (#419) */ #endif -static void numa_slab_bind(void *p, size_t n){ +static int numa_slab_bind(void *p, size_t n){ #ifdef __linux__ - if(g_numa_nodes<2 || g_numa_skip_bind || !p || !n) return; - unsigned long mask=(1UL<(size_t)INT_MAX){ + coli_idmask_free(&selected); errno=EOVERFLOW; return -1; + } + /* Keep explicit physical node IDs stable if the task later crosses a + * cpuset boundary. Legacy autodetection keeps the historical remapping + * behavior. MPOL_F_STATIC_NODES is the Linux UAPI bit from mempolicy.h. */ + g_numa_mbind_mode=coli_numa_policy_mode(selected.count,selected_spec!=NULL); + g_numa_mask=selected; + g_numa_nodes=(int)selected.count; + if(g_numa_nodes<2 && !selected_spec){ + fprintf(stderr,"[NUMA] single selected node: COLI_NUMA ignored\n"); + coli_idmask_free(&g_numa_mask); + return 0; + } /* Probe mbind once so a constrained container degrades with a message * instead of silently losing the interleave. The probe page must be - * page-aligned (mbind rejects unaligned addresses with EINVAL) and only - * errno==EPERM disables — any other failure keeps NUMA on. */ + * page-aligned (mbind rejects unaligned addresses with EINVAL). Legacy + * autodetection retains its EPERM degradation; an explicit managed mask + * fails closed on every probe error. */ { void *pg=NULL; - if(!posix_memalign(&pg,4096,4096)){ - unsigned long mask=(1UL<=(size_t)1<<20) numa_slab_bind(p,n); /* resident dense weights too (#82: attention/shared stream from RAM every token) */ + if(n>=(size_t)1<<20 && numa_slab_bind(p,n)){ + int saved=errno?errno:EIO; + fprintf(stderr,"[NUMA] cannot apply reviewed policy to %.1f MiB slab: %s\n", + n/(double)(1<<20),strerror(saved)); + free(p); errno=saved; exit(2); + } /* resident dense weights too (#82: attention/shared stream from RAM every token) */ return p; } static float *qsalloc(int O){ return (float*)qalloc((size_t)O*sizeof(float)); } @@ -1370,7 +1762,8 @@ static const char *qt_name_by_fmt(int fmt){ * (expert_load_impl and friends) always pass NULL: this branch's repack * tool never stamps routed experts, so there is nothing for those paths to * consult. */ -static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs, const char *stamped_name){ +static int qt_resolve_fmt_impl(const char *name, int O, int I, int64_t nb, int64_t ns, + int *gs, const char *stamped_name, int fatal){ int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4); int64_t exp_i3=(int64_t)O*i3_rowbytes(I); /* int3-g64 (fmt=5): 24B per 64-input group */ /* fmt=6 (E8/IQ3, #452): scales live inside the 98B super-blocks, so the .qs @@ -1414,16 +1807,19 @@ static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns if(sf==1 && i8_row_also){ *gs=0; return 1; } /* sf==8 with only fp8_blk_ue8m0_also true falls through here too -- * see the comment above this block. */ - fprintf(stderr,"%s: [%d,%d] byte layout (nb=%lld ns=%lld) matches E8/IQ3 " - "(fmt=6, 4-byte tag)%s%s%s; refusing rather than guessing (untrusted " - "container, fmt=6 collision at I=98)%s\n", - name,O,I,(long long)nb,(long long)ns, - fp8_blk_f32_also ? " AND per-128x128-block FP8 f32 scales (fmt=8, single block)" : "", - fp8_blk_ue8m0_also ? " AND per-128x128-block FP8 ue8m0 scales (fmt=8, 4 blocks, recognized-not-implemented)" : "", - i8_row_also ? " AND plain int8 per-row (fmt=1, O=1)" : "", - stamped_name ? " -- metadata stamp present but names a format/encoding that doesn't resolve the ambiguity" - : ""); - exit(1); + if(fatal){ + fprintf(stderr,"%s: [%d,%d] byte layout (nb=%lld ns=%lld) matches E8/IQ3 " + "(fmt=6, 4-byte tag)%s%s%s; refusing rather than guessing (untrusted " + "container, fmt=6 collision at I=98)%s\n", + name,O,I,(long long)nb,(long long)ns, + fp8_blk_f32_also ? " AND per-128x128-block FP8 f32 scales (fmt=8, single block)" : "", + fp8_blk_ue8m0_also ? " AND per-128x128-block FP8 ue8m0 scales (fmt=8, 4 blocks, recognized-not-implemented)" : "", + i8_row_also ? " AND plain int8 per-row (fmt=1, O=1)" : "", + stamped_name ? " -- metadata stamp present but names a format/encoding that doesn't resolve the ambiguity" + : ""); + exit(1); + } + return -1; } *gs=0; return 6; } @@ -1433,8 +1829,13 @@ static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns * int3-g64 and grouped-int4-at-gs=64 carry the SAME scale cardinality O*ceil(I/64). */ int fmt = (nb==exp_i8)?1 : (nb==exp_i4)?2 : (nb==exp_i2)?3 : (nb==exp_i3)?5 : 0; if(!fmt){ - fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2/int3-g64/fp8 layout for [%d,%d], refusing (untrusted container)\n", - name,(long long)nb,O,I); exit(1); } + if(fatal){ + fprintf(stderr,"%s: quantized weight is %lld bytes — no int8/int4/int2/int3-g64/fp8 layout for [%d,%d], refusing (untrusted container)\n", + name,(long long)nb,O,I); + exit(1); + } + return -1; + } *gs=0; if(fmt==2){ int g=detect_group_size(O,I,ns); if(g>0){ fmt=4; *gs=g; } } /* fmt=1 vs fmt=8 (native FP8-e4m3 passthrough): THE DESIGN LANDMINE. Weight @@ -1542,12 +1943,15 @@ static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns if(sf==1 || sf==8){ fmt = sf; } else if(stamped_name){ - fprintf(stderr,"%s: [%d,%d] scale array is %lld bytes — matches BOTH per-row " - "int8 (fmt=1) and per-128x128-block FP8 (fmt=8) scale geometry; refusing " - "rather than guessing (untrusted container, THE DESIGN LANDMINE) -- metadata " - "stamp present but names a format that doesn't resolve the ambiguity\n", - name,O,I,(long long)ns); - exit(1); + if(fatal){ + fprintf(stderr,"%s: [%d,%d] scale array is %lld bytes — matches BOTH per-row " + "int8 (fmt=1) and per-128x128-block FP8 (fmt=8) scale geometry; refusing " + "rather than guessing (untrusted container, THE DESIGN LANDMINE) -- metadata " + "stamp present but names a format that doesn't resolve the ambiguity\n", + name,O,I,(long long)ns); + exit(1); + } + return -1; } /* else (no stamp at all): falls through to fmt=1, the INVERSION. */ } else if(is_blk_ue8m0){ @@ -1555,16 +1959,19 @@ static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns fmt = 1; /* stamp confirms this is genuinely plain int8, not an * unimplemented-encoding fp8 tensor -- safe to resolve. */ } else { - fprintf(stderr,"%s: [%d,%d] fp8-e4m3-b128 with ue8m0 scales recognized but not " - "implemented; only f32 block scales are supported in this build (nb=%lld " - "bytes matches raw e4m3 weight bytes, ns=%lld bytes matches %lld blocks x " - "1 byte/block)%s%s -- refusing rather than misreading the sidecar " - "(untrusted container)\n", - name,O,I,(long long)nb,(long long)ns,(long long)(nblkO*nblkI), - is_row ? " -- scale array ALSO matches per-row int8 (fmt=1)" : "", - stamped_name ? " -- a metadata stamp cannot grant this build a decoder it doesn't have" - : ""); - exit(1); + if(fatal){ + fprintf(stderr,"%s: [%d,%d] fp8-e4m3-b128 with ue8m0 scales recognized but not " + "implemented; only f32 block scales are supported in this build (nb=%lld " + "bytes matches raw e4m3 weight bytes, ns=%lld bytes matches %lld blocks x " + "1 byte/block)%s%s -- refusing rather than misreading the sidecar " + "(untrusted container)\n", + name,O,I,(long long)nb,(long long)ns,(long long)(nblkO*nblkI), + is_row ? " -- scale array ALSO matches per-row int8 (fmt=1)" : "", + stamped_name ? " -- a metadata stamp cannot grant this build a decoder it doesn't have" + : ""); + exit(1); + } + return -1; } } else if(is_blk && !is_row) fmt=8; } @@ -1573,11 +1980,28 @@ static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns : (fmt==8)? fp8_nblk(O)*fp8_nblk(I) : (int64_t)O; /* in FLOAT */ if(ns != exp_scale*4){ - fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", - name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); exit(1); } + if(fatal){ + fprintf(stderr,"%s: scale array is %lld bytes — expected %lld for [%d,%d] fmt=%d, refusing (untrusted container)\n", + name,(long long)ns,(long long)(exp_scale*4),O,I,fmt); + exit(1); + } + return -1; + } return fmt; } +/* Optional consumers (RAM-map eligibility) need the exact same byte geometry + * and collision policy without turning an ineligible optimization into a fatal + * model-load error. The ordinary container path keeps its fail-loud contract. */ +static int qt_try_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, + int *gs, const char *stamped_name){ + return qt_resolve_fmt_impl(name,O,I,nb,ns,gs,stamped_name,0); +} +static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, + int *gs, const char *stamped_name){ + return qt_resolve_fmt_impl(name,O,I,nb,ns,gs,stamped_name,1); +} + /* TRUST-VERIFY-REFUSE: if `stamped` (the tensor's __metadata__ format-NAME * stamp, or NULL if none -- see st_fmt_stamp/st_fmt_stamp_ingest in st.h) * is present, verify it agrees with `fmt` (qt_resolve_fmt's byte-arithmetic @@ -1739,11 +2163,15 @@ static void layer_cuda_shard_kvb(Layer *l,int H,int Q,int V){ } #endif -static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits){ +static void model_init(Model *m, const char *snap, const char *weights_dir, + int cap, int ebits, int dbits){ memset(m,0,sizeof(*m)); m->ebits=ebits; m->dbits=dbits; + /* Configuration and tokenizer identity always stay rooted at SNAP. Only the + * safetensors namespace is replaceable (for a staged tmpfs + SSD-symlink + * hybrid), and defaults to SNAP in main(). */ load_cfg(&m->c,snap); { const char *xd=getenv("COLI_MODEL_DIRS"); /* SPLIT: model shards spread across N drives */ - st_init_multi(&m->S,snap,(xd&&*xd)?xd:NULL); } + st_init_multi(&m->S,weights_dir,(xd&&*xd)?xd:NULL); } Cfg *c=&m->c; char nm[256]; int H=c->n_heads, D=c->hidden; /* embed e lm_head sono il confine I/O: tenerli ad alta precisione (come i quant dynamic * reali). A bf16 ~1.9GB su GLM reale: trascurabile. dbits>=8 -> qui f32; piu' basso -> dbits. */ @@ -1971,11 +2399,25 @@ static void *map_of_fd(int fd){ void *base=NULL; #if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) struct stat st; - if(g_nmaps<512 && fstat(fd,&st)==0){ - size_t len=((size_t)st.st_size+16383)&~(size_t)16383; + if(g_nmaps<512 && fstat(fd,&st)==0 && st.st_size>0 && (uint64_t)st.st_size<=SIZE_MAX){ + /* mmap accepts an exact, non-page-multiple file length and rounds the VMA + * internally. Linux RAMMAP keeps that bounded length; the Metal-specific + * branch below restores its required 16 KiB registration granularity with + * an explicit overflow guard. */ + size_t len=(size_t)st.st_size; +#ifdef COLI_METAL + /* Apple newBufferWithBytesNoCopy requires both base and length to use + * the 16 KiB VM-page granularity. Preserve the legacy rounded mapping + * for Metal while keeping the Linux RAM-map length exact. */ + if(len>SIZE_MAX-16383){ pthread_mutex_unlock(&g_map_mtx); return NULL; } + len=(len+16383)&~(size_t)16383; +#endif void *p=mmap(NULL,len,PROT_READ,MAP_SHARED,fd,0); if(p!=MAP_FAILED){ base=p; g_maps[g_nmaps].fd=fd; g_maps[g_nmaps].base=p; g_maps[g_nmaps].len=len; g_nmaps++; +#if defined(__linux__) && defined(MADV_HUGEPAGE) + if(g_rammap) (void)madvise(p,len,MADV_HUGEPAGE); +#endif #ifdef COLI_METAL if(g_metal_enabled) coli_metal_register(p,len); #endif @@ -1985,6 +2427,106 @@ static void *map_of_fd(int fd){ pthread_mutex_unlock(&g_map_mtx); return base; } +static ESlot *rammap_slot(Model *m, int layer, int eid){ + if(!m->rammap || layer<0 || layer>m->c.n_layers || eid<0 || eid>=m->c.n_experts) return NULL; + ESlot *s=&m->rammap[(int64_t)layer*m->c.n_experts+eid]; + return s->backing==ESLOT_BACKING_RAMMAP && s->eid==eid ? s : NULL; +} + +/* Bind one immutable direct expert. Eligibility is descriptor-based: a symlink + * in the staged namespace is direct only when its opened target is actually tmpfs. + * Format inference reuses qt_resolve_fmt's authoritative geometry and collision + * policy through its nonfatal wrapper, including grouped-int4, int3-g64, E8 and + * FP8. Any mismatch returns 0 so the expert simply falls back to the ordinary + * SSD/slab path — a tmpfs-ineligible expert is expected, not fatal. */ +static int64_t rammap_bind_one(Model *m, int layer, int eid, ESlot *s){ +#ifndef __linux__ + (void)m;(void)layer;(void)eid;(void)s; return 0; +#else + Cfg *c=&m->c; int I=c->moe_inter,D=c->hidden; + const char suf[3][16]={"gate_proj","up_proj","down_proj"}; + char nm[3][288],qn[300]; st_tensor *tw[3],*tq[3]; + int OO[3]={I,I,D},II[3]={D,D,I}; + for(int k=0;k<3;k++){ + snprintf(nm[k],sizeof(nm[k]),"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]); + tw[k]=st_find(&m->S,nm[k]); + /* GCC sees an indexed nm[3][] as the whole 864-byte array when it + * diagnoses snprintf. Construct the suffix with the same explicit + * bound used by the ordinary and io_uring expert loaders. */ + size_t name_len=strnlen(nm[k],sizeof(nm[k])); + if(name_len+3>=sizeof(qn)) return 0; + memcpy(qn,nm[k],name_len); memcpy(qn+name_len,".qs",4); + tq[k]=st_find(&m->S,qn); + if(!tw[k]||!tq[k] || tw[k]->dtype!=3 || tq[k]->dtype!=2 || + !st_fd_is_tmpfs(&m->S,tw[k]->fd) || !st_fd_is_tmpfs(&m->S,tq[k]->fd) || + (tw[k]->off&3) || (tq[k]->off&3) || tw[k]->nbytes<=0 || tq[k]->nbytes<=0 || + tw[k]->numel!=tw[k]->nbytes || tq[k]->numel!=tq[k]->nbytes/4) return 0; + } + void *bw[3],*bq[3]; QT *qt[3]={&s->g,&s->u,&s->d}; int64_t total=0; + for(int k=0;k<3;k++){ + int O=OO[k], In=II[k]; + int64_t nb=tw[k]->nbytes, ns=tq[k]->nbytes; + int gs=0; + int fmt=qt_try_resolve_fmt(tw[k]->name,O,In,nb,ns,&gs,NULL); + if(fmt<0) return 0; + bw[k]=map_of_fd(tw[k]->fd); bq[k]=map_of_fd(tq[k]->fd); + if(!bw[k]||!bq[k]) return 0; + memset(qt[k],0,sizeof(*qt[k])); + qt[k]->fmt=fmt; qt[k]->O=O; qt[k]->I=In; qt[k]->gs=gs; + qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); + qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off); + qt[k]->s=(float*)((char*)bq[k]+tq[k]->off); + total+=nb+ns; + } + s->eid=eid; s->backing=ESLOT_BACKING_RAMMAP; + return total; +#endif +} + +static void rammap_touch_qt(const QT *q){ + const char *w=q->fmt==1?(const char*)q->q8:(const char*)q->q4; + int64_t scale_b=qt_scale_bytes(q); + int64_t weight_b=qt_bytes(q)-scale_b; + volatile unsigned char acc=0; + for(int64_t i=0;i0) acc^=(unsigned char)w[weight_b-1]; + const char *sp=(const char*)q->s; + for(int64_t i=0;i0) acc^=(unsigned char)sp[scale_b-1]; + (void)acc; +} + +static void rammap_build(Model *m){ + if(!g_rammap) return; + Cfg *c=&m->c; int rows=c->n_layers+1; + m->rammap=calloc((size_t)rows*c->n_experts,sizeof(ESlot)); + if(!m->rammap){ fprintf(stderr,"[RAMMAP] metadata allocation failed\n"); exit(1); } + for(int64_t z=0;z<(int64_t)rows*c->n_experts;z++) m->rammap[z].eid=-1; + for(int l=0;l<=c->n_layers;l++){ + int sparse=(ln_layers&&m->L[l].sparse)||(l==c->n_layers&&m->has_mtp); + if(!sparse) continue; + for(int e=0;en_experts;e++){ + ESlot *s=&m->rammap[(int64_t)l*c->n_experts+e]; + int64_t nb=rammap_bind_one(m,l,e,s); + if(nb>0){ m->rammap_experts++; m->rammap_bytes+=nb; } + } + } + double t0=now_s(); + if(g_ram_prefault){ + int64_t n=(int64_t)rows*c->n_experts; + #pragma omp parallel for schedule(dynamic,8) + for(int64_t z=0;zrammap[z].backing==ESLOT_BACKING_RAMMAP){ + rammap_touch_qt(&m->rammap[z].g); rammap_touch_qt(&m->rammap[z].u); rammap_touch_qt(&m->rammap[z].d); + } + } + m->rammap_prefault_s=g_ram_prefault?now_s()-t0:0; + if(g_ram_prefault) + fprintf(stderr,"[RAMMAP] %d direct tmpfs experts, %.2f GB mapped; prefaulted in %.2fs\n", + m->rammap_experts,m->rammap_bytes/1e9,m->rammap_prefault_s); + else + fprintf(stderr,"[RAMMAP] %d direct tmpfs experts, %.2f GB mapped (prefault off)\n", + m->rammap_experts,m->rammap_bytes/1e9); +} /* ==================== MULTI-SSD: N model copies, N drives ==================== * COLI_MODEL_MIRROR=[;...] registers additional read-only copies of @@ -2179,9 +2721,10 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i qt_from_disk(m,nm[0],I,D,b,g_drop,&s->g); qt_from_disk(m,nm[1],I,D,b,g_drop,&s->u); qt_from_disk(m,nm[2],D,I,b,g_drop,&s->d); - atomic_fetch_add_explicit(&g_prof_io, - st_nbytes(&m->S,nm[0])+st_nbytes(&m->S,nm[1])+st_nbytes(&m->S,nm[2]),memory_order_relaxed); - s->eid=eid; return 0; + int64_t ssd=0; + for(int k=0;k<3;k++) ssd+=prof_ssd_tensor_bytes(&m->S,st_find(&m->S,nm[k])); + atomic_fetch_add_explicit(&g_prof_io,ssd,memory_order_relaxed); + s->eid=eid; s->backing=ESLOT_BACKING_OWNED; return 0; } st_tensor *tw[3], *tq[3]; for(int k=0;k<3;k++){ @@ -2239,11 +2782,12 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i * leak locked pages for every GPU-tier expert). See pin_wire() below: it wires * the final resident set only, after GPU release has already nulled out the * pointers for anything that isn't genuinely RAM-tier. */ - atomic_fetch_add_explicit(&g_prof_io,(int64_t)(n+nq),memory_order_relaxed); + int64_t ssd=prof_ssd_tensor_bytes(&m->S,tw[k])+prof_ssd_tensor_bytes(&m->S,tq[k]); + atomic_fetch_add_explicit(&g_prof_io,ssd,memory_order_relaxed); atomic_fetch_add_explicit(&g_mir_bytes[rep],tw[k]->nbytes+tq[k]->nbytes,memory_order_relaxed); } atomic_fetch_add_explicit(&g_mir_nread[rep],1,memory_order_relaxed); - s->eid=eid; return 0; + s->eid=eid; s->backing=ESLOT_BACKING_MMAP; return 0; } } int64_t wtot=tw[0]->nbytes+tw[1]->nbytes+tw[2]->nbytes; @@ -2263,7 +2807,13 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i compat_aligned_free(s->slab); if(posix_memalign((void**)&s->slab,4096,wtot+8192)){fprintf(stderr,"OOM slab\n"); if(fatal) exit(1); s->slab=NULL; s->slab_cap=0; return -1;} s->slab_cap=wtot+8192; - numa_slab_bind(s->slab,(size_t)s->slab_cap); + if(numa_slab_bind(s->slab,(size_t)s->slab_cap)){ + int saved=errno?errno:EIO; + fprintf(stderr,"[NUMA] cannot apply reviewed policy to expert slab: %s\n", + strerror(saved)); + compat_aligned_free(s->slab); s->slab=NULL; s->slab_cap=0; + errno=saved; if(fatal) exit(2); return -1; + } #endif } if(!s->fslab || ftot > s->fslab_cap){ @@ -2300,7 +2850,13 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i } } s->fslab_cap=ftot; - numa_slab_bind(s->fslab,(size_t)ftot*sizeof(float)); + if(numa_slab_bind(s->fslab,(size_t)ftot*sizeof(float))){ + int saved=errno?errno:EIO; + fprintf(stderr,"[NUMA] cannot apply reviewed policy to expert scales: %s\n", + strerror(saved)); + free(s->fslab); s->fslab=NULL; s->fslab_cap=0; + errno=saved; if(fatal) exit(2); return -1; + } #endif } /* DISK-CLASS: classify before the reads; computed unconditionally at dc_on sites so @@ -2361,10 +2917,12 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i if(dc_on) dc_wall_exit(dc_cls,now_s()); /* pair the enter on the non-fatal unwind */ return -1; } fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; } - atomic_fetch_add_explicit(&g_prof_io,wtot+fo*4,memory_order_relaxed); + int64_t ssd=0; + for(int k=0;k<3;k++) ssd+=prof_ssd_tensor_bytes(&m->S,tw[k])+prof_ssd_tensor_bytes(&m->S,tq[k]); + atomic_fetch_add_explicit(&g_prof_io,ssd,memory_order_relaxed); if(dc_on){ /* DISK-CLASS accounting, see dc_needed() */ double dc_t1=now_s(); /* one clock read for thread-ns AND the wall exit */ - int64_t bytes=wtot+fo*4; + int64_t bytes=ssd; /* matches g_prof_io: tmpfs-backed reads count 0 */ atomic_fetch_add_explicit(&g_dc_n[dc_cls],1,memory_order_relaxed); atomic_fetch_add_explicit(&g_dc_bytes[dc_cls],bytes,memory_order_relaxed); atomic_fetch_add_explicit(&g_dc_ns[dc_cls],(int64_t)((dc_t1-dc_t0)*1e9),memory_order_relaxed); @@ -2386,7 +2944,7 @@ static int expert_load_impl(Model *m, int layer, int eid, ESlot *s, int fatal, i qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->gs=gs; qt[k]->qf=NULL; qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k]; } - s->eid=eid; return 0; + s->eid=eid; s->backing=ESLOT_BACKING_OWNED; return 0; } /* Every expert read goes through here: time the whole load (pread/fault + * bookkeeping) on the thread that runs it, into the disk-service counter. */ @@ -2485,6 +3043,11 @@ static int uring_load_add(UringBatch *b,Model *m,int layer,int eid,ESlot *s,int if(posix_memalign((void**)&s->slab,4096,(size_t)wtot+8192)){ s->slab=NULL; s->slab_cap=0; return uring_load_error(l,ENOMEM,"io_uring expert slab"),li; } s->slab_cap=wtot+8192; + if(numa_slab_bind(s->slab,(size_t)s->slab_cap)){ + int saved=errno?errno:EIO; + compat_aligned_free(s->slab); s->slab=NULL; s->slab_cap=0; + return uring_load_error(l,saved,"io_uring expert slab NUMA policy"),li; + } #endif } if(!s->fslab || ftot>s->fslab_cap){ @@ -2498,6 +3061,11 @@ static int uring_load_add(UringBatch *b,Model *m,int layer,int eid,ESlot *s,int free(s->fslab); s->fslab=malloc((size_t)ftot*sizeof(float)); if(!s->fslab){ s->fslab_cap=0; return uring_load_error(l,ENOMEM,"io_uring expert scales"),li; } s->fslab_cap=ftot; + if(numa_slab_bind(s->fslab,(size_t)s->fslab_cap*sizeof(float))){ + int saved=errno?errno:EIO; + free(s->fslab); s->fslab=NULL; s->fslab_cap=0; + return uring_load_error(l,saved,"io_uring expert scales NUMA policy"),li; + } #endif } int ord[3]={0,1,2}; @@ -2786,6 +3354,10 @@ static void expert_host_release(Model *m, ESlot *s){ m->resident_bytes-=bytes; if(m->resident_bytes<0) m->resident_bytes=0; } static void expert_host_ensure(Model *m, int layer, ESlot *s){ + /* File-backed views are already valid host storage. A direct RAM-map expert or a + * legacy mmap view must never fall through the CUDA host-restore path and be copied + * into an anonymous slab. */ + if(s->backing==ESLOT_BACKING_RAMMAP || s->backing==ESLOT_BACKING_MMAP) return; if(s->slab) return; if(s->aslab){ s->slab=s->aslab; s->fslab=s->afslab; } /* re-attach the arena slice; caps survived release */ /* re-materializing a GPU-resident expert's host copy, not a routing miss: demand=0 */ @@ -2806,6 +3378,7 @@ static void expert_host_ensure(Model *m, int layer, ESlot *s){ * EN: under O_DIRECT the weights bypass the page cache, so their WILLNEED is wasted; * the .qs scales are always buffered, so keep theirs. Advisory hint -> output-preserving. */ static void expert_prefetch(Model *m, int layer, int eid){ + if(expert_is_resident(m,layer,eid)) return; /* RAM map, pin, or ecache: nothing to prefetch */ char nm[300]; int rep=expert_route(layer,eid); const char *suf[3]={"gate_proj.weight","up_proj.weight","down_proj.weight"}; for(int k=0;k<3;k++){ @@ -3715,12 +4288,30 @@ static void *vk2_issue_worker(void *p){ } #endif -/* pin ∪ LRU residency probe (used by CACHE_ROUTE max-rank fill). */ -static int expert_is_resident(Model *m, int layer, int eid){ +/* One precedence rule for every routed-expert consumer. Direct tmpfs views are + * immutable and win over any stale/duplicate pin or LRU entry. `touch` is used + * only by the actual MoE resolver: probes for routing/prefetch do not alter LRU + * recency or call telemetry. */ +static ESlot *expert_resident_slot(Model *m, int layer, int eid, int touch){ + if(layer<0 || layer>m->c.n_layers || eid<0 || eid>=m->c.n_experts) return NULL; + ESlot *r=rammap_slot(m,layer,eid); + if(r){ if(touch) m->rammap_calls++; return r; } ESlot *P=m->pin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid) return 1; + for(int z=0;znpin[layer];z++) if(P[z].eid==eid) return &P[z]; ESlot *Sl=m->ecache[layer]; - for(int z=0;zecn[layer];z++) if(Sl[z].eid==eid) return 1; + for(int z=0;zecn[layer];z++) if(Sl[z].eid==eid){ + if(touch) Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); + return &Sl[z]; + } + return NULL; +} +static int expert_is_resident(Model *m, int layer, int eid){ + return expert_resident_slot(m,layer,eid,0)!=NULL; +} +static int expert_slot_is_pinned(const Model *m, int layer, const ESlot *slot){ + if(!slot || !m->pin || !m->npin || layer<0 || layer>m->c.n_layers) return 0; + const ESlot *P=m->pin[layer]; + for(int z=0;znpin[layer];z++) if(slot==&P[z]) return 1; return 0; } @@ -4033,16 +4624,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int int e=idxs[(int64_t)s*K+kk]; for(int j=0;jpin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ found=1; break; } - if(!found){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; m->hit_vk++; continue; } } #endif - ESlot *P=m->pin[layer]; - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ m->hits++; m->hit_pin++; use[j]=&P[z]; break; } - if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zhits++; m->hit_ecache++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } } - if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; + use[j]=expert_resident_slot(m,layer,eid,1); /* RAM map -> pin -> LRU; touch bumps recency/telemetry */ + if(use[j]){ + m->hits++; + /* preserve dev's #336 tier split: a direct tmpfs view is its own tier + * (counted in m->rammap_calls inside expert_resident_slot), pin/ecache + * stay distinguishable for the PROF tier breakdown. */ + if(use[j]->backing!=ESLOT_BACKING_RAMMAP){ + if(expert_slot_is_pinned(m,layer,use[j])) m->hit_pin++; + else m->hit_ecache++; + } + } else { qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; if(g_disk_split){ if(m->ld_ctx==1) m->miss_draft++; else if(m->ld_ctx==2) m->miss_absorb++; } } } int metal_done=0; @@ -4253,15 +4843,11 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int * questo — il kernel legge in background, le pread dopo trovano cache calda */ if(base+64pin[layer]; + for(int j=0;jnpin[layer] && !found;z++) if(P[z].eid==eid) found=1; - ESlot *Sl=m->ecache[layer]; - for(int z=0;zecn[layer] && !found;z++) if(Sl[z].eid==eid) found=1; - if(!found) expert_prefetch(m,layer,eid); + if(!served) expert_prefetch(m,layer,eid); } } #ifdef COLI_CUDA @@ -4509,13 +5095,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int uint8_t vcls[64]; for(int c2=0;c2=0) vcls[c2] = pipe_ready(cqof[c2]) ? 0 : 2; /* loaded : in flight */ - else vcls[c2] = ce[c2]->slab ? 0 : 1; /* resident : sync miss */ + else vcls[c2] = expert_host_ready(ce[c2]) ? 0 : 1; /* resident : sync miss */ } int vord[64], no=0; for(uint8_t k2=0;k2<3;k2++) for(int c2=0;c2=0){ double tw=now_s(); pipe_wait(cqof[c2]); m->t_ewait += now_s()-tw; } - if(!e->slab) expert_load(m,layer,e->eid,e,1,1); /* demand=1: moe miss path (FASE A snapshot valid) */ + if(!expert_host_ready(e)) expert_load(m,layer,e->eid,e,1,1); /* demand=1: moe miss path (FASE A snapshot valid) */ for(int r=0;rg,&e->u,nr); @@ -4888,7 +5474,15 @@ static void la_predict(Model *m, int target, const float *h, int kind){ } Layer *sl = &m->L[src_layer]; int sI = c->moe_inter * c->n_shared; - float *snrm = falloc(D), *sg = falloc(sI), *su = falloc(sI); + /* falloc deliberately returns uninitialized hot-path storage. rmsnorm + * overwrites D values before sh_gate consumes them, but GCC 15.2 + * cannot join that extent to the separately stored QT.I model-shape + * invariant and emits -Wmaybe-uninitialized below. This optional + * LOOKA scratch is cold and small; initializing its source state makes + * the invariant explicit. rmsnorm still overwrites it, so valid-model + * arithmetic and output bytes are unchanged. */ + float *snrm = xzalloc((size_t)D*sizeof(*snrm), "LOOKA normalization"); + float *sg = falloc(sI), *su = falloc(sI); float *sout = falloc(D), *hc = falloc(D); rmsnorm(snrm, h, sl->post_ln, D, c->eps); matmul_qt(sg, snrm, &sl->sh_gate, 1); @@ -4947,13 +5541,14 @@ static void pilot_realload(Model *m, int layer, int eid){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); /* fuori range (come il ramo URING) o main gia' su questo layer */ pthread_mutex_unlock(&g_pilot_mx); return; } - ESlot *P=m->pin[layer]; /* gia' residente (pin o ecache)? skip */ #ifdef COLI_VULKAN if(vk_reg_served(layer,eid)){ pthread_mutex_unlock(&g_pilot_mx); return; } /* VK-tier-served: no load */ #endif - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){ pthread_mutex_unlock(&g_pilot_mx); return; } + if(expert_is_resident(m,layer,eid)){ /* RAM map, pin, or ecache: nothing to load */ + pthread_mutex_unlock(&g_pilot_mx); return; + } ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; /* dedup contro residenti E prenotazioni in volo -(eid+2) */ - for(int z=0;z1): scegli lo slot sotto lock e MARCALO prenotato prima di * rilasciarlo, cosi' gli altri worker non lo scelgono come vittima ne' ricaricano * lo stesso eid. Stesso schema del ramo URING (prenotazione visibile -(eid+2), @@ -5022,13 +5617,13 @@ static void pilot_uring_batch(Model *m){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); pthread_mutex_unlock(&g_pilot_mx); continue; } - int found=0; ESlot *P=m->pin[layer]; + int found=0; #ifdef COLI_VULKAN if(vk_reg_served(layer,eid)) found=1; /* VK-tier-served: no load */ #endif - for(int z=0;znpin[layer];z++) if(P[z].eid==eid){found=1;break;} + if(!found) found=expert_is_resident(m,layer,eid); ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer]; - for(int z=0;zecap){ slot=nn; m->ecn[layer]=nn+1; } @@ -5202,16 +5797,15 @@ static void couple_prefetch(Model *m, int layer, const int *idx, int Ke){ for(int e=0;ebv){bv=sc[e];best=e;} if(best<0) break; sc[best]=0; - int found=0; /* residency scan, same locking as pilot */ + int found=0; /* residency scan, same locking as pilot */ pthread_mutex_lock(&g_pilot_mx); - ESlot *P=m->pin[lt]; #ifdef COLI_VULKAN if(vk_reg_served(lt,best)) found=1; /* VK-tier-served: no load */ #endif - for(int z=0;znpin[lt] && !found;z++) if(P[z].eid==best) found=1; + if(!found) found=expert_is_resident(m,lt,best); ESlot *Sl=m->ecache[lt]; for(int z=0;zecn[lt] && !found;z++) - if(Sl[z].eid==best || Sl[z].eid==-(best+2)) found=1; + if(Sl[z].eid==-(best+2)) found=1; /* URING sentinel (==best moved into expert_is_resident) */ pthread_mutex_unlock(&g_pilot_mx); if(!found){ unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED); @@ -5269,14 +5863,13 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S){ * lock anyway, making a racing redundant enqueue harmless. */ int found=0; pthread_mutex_lock(&g_pilot_mx); - ESlot *P=m->pin[lnext]; #ifdef COLI_VULKAN if(vk_reg_served(lnext,best)) found=1; /* VK-tier-served: no load */ #endif - for(int z=0;znpin[lnext] && !found;z++) if(P[z].eid==best) found=1; + if(!found) found=expert_is_resident(m,lnext,best); ESlot *Sl=m->ecache[lnext]; for(int z=0;zecn[lnext] && !found;z++) - if(Sl[z].eid==best || Sl[z].eid==-(best+2)) found=1; + if(Sl[z].eid==-(best+2)) found=1; /* URING sentinel (==best moved into expert_is_resident) */ pthread_mutex_unlock(&g_pilot_mx); if(!found){ unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED); @@ -6163,10 +6756,11 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo typedef struct { int *dst; int n; } EmitStore; static void emit_store(int t, void *ud){ EmitStore *e=(EmitStore*)ud; e->dst[e->n++]=t; } /* emit callback: detokenizza e stampa in streaming (chat/run), con heartbeat */ -typedef struct { Tok *T; Model *m; double t0; int count; int quiet; } EmitStream; +typedef struct { Tok *T; Model *m; double t0; int count; int quiet; double first_s; } EmitStream; static void emit_stream(int t, void *ud){ EmitStream *e=(EmitStream*)ud; char dec[64]; int dn=tok_decode(e->T,&t,1,dec,63); dec[dn]=0; fputs(dec,stdout); fflush(stdout); + if(e->first_s<0) e->first_s=now_s()-e->t0; if(!e->quiet && ++e->count%16==0){ double tt=e->m->hits+e->m->miss; if(g_cache_route && e->m->route_slots){ double swap=100.0*e->m->route_swaps/e->m->route_slots; @@ -6465,6 +7059,7 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, } } int64_t io=atomic_load_explicit(&g_prof_io,memory_order_relaxed)-b->io; + int64_t physical_io=prof_physical_read_delta(b); uint64_t dh=m->hits-b->hits, dm=m->miss-b->miss, dq=m->ereq-b->ereq; double hitp=(dh+dm)?100.0*dh/(dh+dm):100.0; double eb=(double)expert_bytes_probe(m,m->ebits); @@ -6473,12 +7068,22 @@ static void prof_report(Model *m, const ProfBase *b, double elapsed, int tokens, double io_w=m->t_ewait-b->ewait; /* stall the compute thread felt */ double io_svc=edisk_s()-b->edisk; /* read service on the loading threads (overlaps compute) */ uint64_t dhp=m->hit_pin-b->hit_pin, dhe=m->hit_ecache-b->hit_ecache; /* split #336 */ - fprintf(f,"[PROF] expert I/O: %.3f GB fetched (%.1f MB/token, %.2f GB/s over the run%s) | " + fprintf(f,"[PROF] SSD-backed expert requests: %.3f GB (%.1f MB/token, %.2f GB/s over the run%s) | " "hit %.1f%% (%llu pin + %llu lru / %llu load) | %.1f loads/token | %.1fs read service / %.1fs felt wait\n", io/1e9, tokens>0?io/1e6/tokens:0.0, io/1e9/elapsed, g_mmap?"; COLI_MMAP=1: page cache may serve part":"", hitp,(unsigned long long)dhp,(unsigned long long)dhe,(unsigned long long)dm, tokens>0?(double)dq/tokens:0.0, io_svc,io_w); + if(physical_io>=0) + fprintf(f,"[PROF] physical SSD reads: %.3f GB (%.1f MB/token; Linux /proc/self/io read_bytes, process-wide)\n", + physical_io/1e9,tokens>0?physical_io/1e6/tokens:0.0); + else + fprintf(f,"[PROF] physical SSD reads: unavailable on this platform/kernel\n"); + if(m->rammap_experts){ + uint64_t rcalls=m->rammap_calls-b->rammap_calls; + fprintf(f,"[PROF] RAM map: %d experts / %.3f GB direct | %llu calls this window | zero slab reads\n", + m->rammap_experts,m->rammap_bytes/1e9,(unsigned long long)rcalls); + } /* DISK-CLASS: per-load cold/warm classification vs. which fd ACTUALLY served it. * Three per-class rates, labeled to keep the units unambiguous (ambiguous units * mislead -- measured lesson): GB/s-thread = bytes / thread-seconds (per-read @@ -6628,18 +7233,19 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ profile_reset(m); ProfBase pb; prof_base(m,&pb); double t=now_s(); - EmitStream es={&T,m,t,0,0}; + EmitStream es={&T,m,t,0,0,-1}; grammar_reset(&g_grd); int produced=spec_decode(m,all,np,ngen,eos,logit,emit_stream,&es,NULL,NULL); double dt=now_s()-t; double tot=m->hits+m->miss; int nsp=0; for(int i=0;in_layers;i++) if(m->L[i].sparse) nsp++; printf("\n---\nprefill %d tokens in %.2fs | decode %d tokens in %.2fs (%.2f tok/s) | " - "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%%s) | RSS %.2f GB", /* split #336 (+VK VRAM tier) */ + "expert hit rate %.1f%% (pin %.1f%% + lru %.1f%%%s) | RSS %.2f GB | TTFT %.3fs", /* split #336 (+VK VRAM tier) */ np,prefill_t,produced,dt,produced/dt,tot?100.0*m->hits/tot:0.0, tot?100.0*m->hit_pin/tot:0.0, tot?100.0*m->hit_ecache/tot:0.0, m->hit_vk?({ static char vkb[40]; snprintf(vkb,sizeof(vkb)," + vk %.1f%%",tot?100.0*m->hit_vk/tot:0.0); vkb; }):"", - rss_gb()); + rss_gb(), + prefill_t+(es.first_s>=0?es.first_s:dt)); if(g_cache_route && m->route_slots) printf(" | swap %.1f%% (%llu/%llu)", 100.0*m->route_swaps/m->route_slots, @@ -6749,8 +7355,24 @@ static int repin_pick(Model *m, RepinCand *out, int maxc){ ESlot *P=m->pin[l]; int ids[4096], zp, eu; long g; int np=m->npin[l]; if(np>4096) np=4096; for(int z=0;zeheat[l],m->elast[l],m->eaccess_clock, - c->n_experts,ids,np,&zp,&eu,&g)) continue; + const uint32_t *heat=m->eheat[l],*last=m->elast[l]; + uint32_t *fh=NULL,*fl=NULL; + if(m->rammap_experts){ /* mask direct tmpfs experts out of admission */ + fh=malloc((size_t)c->n_experts*sizeof(*fh)); + fl=malloc((size_t)c->n_experts*sizeof(*fl)); + if(fh&&fl){ + memcpy(fh,heat,(size_t)c->n_experts*sizeof(*fh)); + memcpy(fl,last,(size_t)c->n_experts*sizeof(*fl)); + for(int e=0;en_experts;e++) if(rammap_slot(m,l,e)){ + fh[e]=0; fl[e]=m->eaccess_clock-255u; /* never a REPIN admission candidate */ + } + heat=fh; last=fl; + } + } + int picked=tier_pick_lfru(heat,last,m->eaccess_clock, + c->n_experts,ids,np,&zp,&eu,&g); + free(fh); free(fl); + if(!picked || rammap_slot(m,l,eu)) continue; /* defensive: never repin a direct expert */ if(nbout[w].gain) out[w]=(RepinCand){g,l,zp,eu,0}; } @@ -6917,15 +7539,15 @@ static void repin_pass_limit(Model *m,int limit){ typedef struct { KVState kv; int *hist, len, first; } ServeCtx; static double kv_pool_bytes(Model *m, int max_ctx); -static void serve_ctx_init(Model *m, ServeCtx *s, const char *snap, int slot, int maxctx){ +static void serve_ctx_init(Model *m, ServeCtx *s, const char *state_dir, int slot, int maxctx){ s->kv.kv_start=calloc(m->c.n_layers+1,sizeof(int)); if(m->has_mtp) s->kv.kv_start[m->c.n_layers]=-1; kv_bind(m,&s->kv); kv_alloc(m,maxctx); s->hist=malloc(maxctx*sizeof(int)); if(!s->hist){ fprintf(stderr,"OOM serve_ctx_init hist\n"); exit(1); } s->first=1; - if(slot==0) snprintf(s->kv.disk_path,sizeof(s->kv.disk_path),"%s/.coli_kv",snap); - else snprintf(s->kv.disk_path,sizeof(s->kv.disk_path),"%s/.coli_kv.%d",snap,slot); + if(slot==0) snprintf(s->kv.disk_path,sizeof(s->kv.disk_path),"%s/.coli_kv",state_dir); + else snprintf(s->kv.disk_path,sizeof(s->kv.disk_path),"%s/.coli_kv.%d",state_dir,slot); s->len=kv_disk_load(m,s->hist,maxctx); if(s->len>0) s->first=0; } @@ -6946,10 +7568,10 @@ typedef struct { emitted token sits at hist[len], not yet forwarded */ unsigned long long id; float temp, top_p; - double started; + double started, request_started, first_s; uint64_t hits0, miss0; - ProfBase pb; /* phase-time window start (same convention as hits0): - feeds the PROF protocol line and the PROF=1 report */ + ProfBase pb, request_pb; /* pb: phase-time window start (same convention as hits0); + request_pb: full-request I/O window for PROF physical.bytes */ } ServeReq; static void mux_data(Tok *T, unsigned long long id, int token){ @@ -7018,6 +7640,10 @@ static void mux_spec_emit(int t, void *ud){ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){ double dt=now_s()-r->started; if(dt<1e-6) dt=1e-6; double dh=(double)(m->hits-r->hits0), dm=(double)(m->miss-r->miss0); + /* Close the request I/O window before persistence/dashboard work below can + * introduce unrelated process reads. */ + ProfPhysicalWire physical=prof_physical_wire( + prof_physical_read_delta(&r->request_pb)); hwinfo_emit(m); usage_save(m); /* la cache che impara non deve aspettare l'uscita */ tiers_emit(m); @@ -7030,11 +7656,25 @@ static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){ * in a wall-time breakdown. With KV_SLOTS>1 concurrent slots share the * batched forwards, so the shares describe the whole engine over the * window, not the single request (same convention as the STAT hit% below). */ - printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu\n",dt, + double fp50=-1,fp99=-1; + uint64_t nw=g_prof_nlat-r->pb.nlat; if(nw>PROF_LAT_CAP) nw=PROF_LAT_CAP; + if(nw){ + double *values=malloc((size_t)nw*sizeof(double)); + if(values){ + for(uint64_t i=0;ifirst_s>=0?r->first_s:now_s()-r->request_started)*1e3; + printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %llu %.3f %.3f %lld %d %lld %.3f %.3f %d\n",dt, r->prompt_tokens,r->emitted, edisk_s()-r->pb.edisk,m->t_ewait-r->pb.ewait,m->t_emm-r->pb.emm, m->t_attn-r->pb.attn,m->t_head-r->pb.head, - (unsigned long long)(m->n_fw-r->pb.n_fw)); + (unsigned long long)(m->n_fw-r->pb.n_fw),fp50,fp99, + (long long)physical.bytes,m->rammap_experts,(long long)m->rammap_bytes, + ttft_ms,m->rammap_prefault_s,physical.valid); printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted, r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(), r->prompt_tokens,r->length_limited); @@ -7082,6 +7722,7 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g } ColiSubmit sub; int valid=coli_submit_parse(line,&sub); if(!valid){ printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0; } + double request_started=now_s(); char *raw=malloc((size_t)sub.bytes+1); if(!raw){ fprintf(stderr,"OOM multiplex payload\n"); exit(1); } if(fread(raw,1,(size_t)sub.bytes,stdin)!=(size_t)sub.bytes){ free(raw); free(line); return -1; } @@ -7189,12 +7830,15 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g if(add>0) memcpy(sc->hist+sc->len,tmp+sc->len,(size_t)add*sizeof(int)); fprintf(stderr,"[API] KV slot %d prefix %d/%d token, prefill %d\n",sub.slot,sc->len,nt,add); free(tmp); + ProfBase request_pb; prof_base(m,&request_pb); float *logit = add>0 ? step(m,sc->hist+sc->len,add,sc->len) : step(m,sc->hist+sc->len-1,1,sc->len-1); sc->len+=add; sc->first=0; ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r)); r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p; - r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss; + r->prompt_tokens=nt; r->started=now_s(); r->request_started=request_started; r->first_s=-1; + r->hits0=m->hits; r->miss0=m->miss; + r->request_pb=request_pb; prof_base(m,&r->pb); /* a few loads: cheap enough to always track */ /* Clamp to the KV room WITHOUT flagging: length_limited must mean "the * limit is what stopped us", not "the request asked for more than the @@ -7219,13 +7863,14 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g if(r->maximum<=0){ r->length_limited=1; mux_done(m,sc,r); return 1; } /* no room at all */ if(next==eos || is_stop(next)){ mux_done(m,sc,r); return 1; } r->pending=next; r->emitted=1; r->active=1; sc->hist[sc->len]=next; m->n_emit++; + r->first_s=now_s()-r->request_started; if(grd[sub.slot].on){ grammar_reset(&grd[sub.slot]); gr_feed(&grd[sub.slot],next); } mux_data(T,r->id,next); if(r->emitted>=r->maximum){ r->length_limited=1; mux_done(m,sc,r); } return 1; } -static void run_serve_mux(Model *m, const char *snap){ +static void run_serve_mux(Model *m, const char *snap, const char *state_dir){ char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap); Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm_tok(&m->c,eos,&T); int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096; @@ -7244,7 +7889,7 @@ static void run_serve_mux(Model *m, const char *snap){ KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req)); GrDraft *grd=calloc(nctx,sizeof(*grd)); /* per-slot request grammars (SUBMIT 7th field) */ - for(int i=0;ikv=NULL; m->Lc=m->Rc=m->Ic=NULL; m->kv_start=NULL; m->max_t=0; } -static void run_serve(Model *m, const char *snap){ +static void run_serve(Model *m, const char *snap, const char *state_dir){ /* Serve mode speaks a byte protocol over BOTH stdout and stdin: * stdout: \x01\x01READY\x01\x01\n, STAT lines, \x01\x01END\x01\x01\n * stdin: text lines plus \x02RESET / \x02MORE control bytes. @@ -7418,7 +8063,7 @@ static void run_serve(Model *m, const char *snap){ if(nctx<1||nctx>16){ fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); exit(2); } KVState *initial=m->kv; free(initial->kv_start); free(initial); ServeCtx *ctx=calloc(nctx,sizeof(ServeCtx)); - for(int i=0;ikv); fprintf(stderr,"[KV] context slots: %d x %d tokens, projected pool %.2f GB\n", nctx,maxctx,kv_pool_bytes(m,maxctx)/1e9); @@ -7442,7 +8087,7 @@ static void run_serve(Model *m, const char *snap){ uint64_t h0=m->hits, ms0=m->miss; double tt0=now_s(); ProfBase pb; if(g_prof) prof_base(m,&pb); float *logit=step(m,hist+len-1,1,len-1); - EmitStream es={&T,m,now_s(),0,1}; + EmitStream es={&T,m,now_s(),0,1,-1}; int prod=0; if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len,NULL); else free(logit); @@ -7524,7 +8169,7 @@ static void run_serve(Model *m, const char *snap){ float *logit; if(k>0){ logit=step(m,hist+len,k,len); len+=k; } else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */ - EmitStream es={&T,m,now_s(),0,1}; + EmitStream es={&T,m,now_s(),0,1,-1}; int prod=0; grammar_reset(&g_grd); /* nuova risposta = nuovo documento (MORE invece continua) */ if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len,NULL); @@ -7940,7 +8585,7 @@ static int pin_rec_cmp(const void *a,const void *b){ * its alloc branch never fires); aslab marks arena ownership for the * release/ensure paths. Arena-OOM just leaves the slots on the individual path. */ static void pin_arena_bind(Model *m, PinRec *r, int *slot_of, int from, int to){ - if(g_numa_nodes<2 || g_mmap || from>=to) return; + if(g_numa_nodes<1 || g_mmap || from>=to) return; Cfg *c=&m->c; int NR=c->n_layers+1; int *cnt=calloc((size_t)NR,sizeof(int)); int *first=malloc((size_t)NR*sizeof(int)); if(!cnt||!first){ free(cnt); free(first); return; } @@ -7963,8 +8608,13 @@ static void pin_arena_bind(Model *m, PinRec *r, int *slot_of, int from, int to){ uint8_t *aw=NULL; float *af=NULL; if(posix_memalign((void**)&aw,4096,(size_t)cnt[l]*ws)) continue; if(posix_memalign((void**)&af,4096,(size_t)cnt[l]*fs)){ free(aw); continue; } - numa_slab_bind(aw,(size_t)cnt[l]*ws); - numa_slab_bind(af,(size_t)cnt[l]*fs); + if(numa_slab_bind(aw,(size_t)cnt[l]*ws) || + numa_slab_bind(af,(size_t)cnt[l]*fs)){ + int saved=errno?errno:EIO; + fprintf(stderr,"[NUMA] cannot apply reviewed policy to layer-%d pin arena: %s\n", + l,strerror(saved)); + free(aw); free(af); errno=saved; exit(2); + } int i=0; for(int a=from;ac; int cap=(c->n_layers+1)*c->n_experts; PinRec *r=malloc((size_t)cap*sizeof(PinRec)); int n=0; unsigned char *seen=calloc((size_t)(c->n_layers+1)*c->n_experts,1); + /* Direct tmpfs experts are already immutable residents. Mark them before + * parsing both the profile and PIN_FILL so pinning cannot copy them into + * anonymous slabs or spend the hot-store budget on duplicates. */ + for(int li=0;li<=c->n_layers;li++){ + int sparse=(lin_layers&&m->L[li].sparse)||(li==c->n_layers&&m->has_mtp); + if(sparse) for(int ei=0;ein_experts;ei++) + if(rammap_slot(m,li,ei)) seen[(int64_t)li*c->n_experts+ei]=1; + } /* A named file is what the identity refusal tells the user to pass, so honouring it * here is what makes that message true. Dimensions and format version still apply: * those refusals never offered a way past them. */ @@ -8784,7 +9442,79 @@ static int coli_env_on(const char *name) strcmp(v,"off")==0 || strcmp(v,"no")==0); } +/* Managed engines give each replica a stable durable state root. Create an + * explicitly requested root like `mkdir -p` with private permissions, and fail + * before loading the model if any component is not a directory or the result is + * not writable. The implicit SNAP default keeps the legacy behavior exactly. */ +static int state_path_sep(char c){ +#ifdef _WIN32 + return c=='/' || c=='\\'; +#else + return c=='/'; +#endif +} +static int state_mkdir_private(const char *path){ +#ifdef _WIN32 + return _mkdir(path); +#else + return mkdir(path,0700); +#endif +} +static int state_dir_prepare(const char *path){ + char p[2048]; size_t n=strlen(path); + /* ServeCtx appends the longest slot suffix (/.coli_kv.15) into the same + * 2048-byte class of buffer. Reject here instead of silently truncating a + * durable state path later in snprintf(). */ + static const char suffix[]="/.coli_kv.15"; + if(!n || n+sizeof(suffix)>sizeof(((KVState*)0)->disk_path) || n>=sizeof(p)){ + errno=ENAMETOOLONG; return -1; + } + memcpy(p,path,n+1); + char *start=p+1; +#ifdef _WIN32 + if(n>=2 && p[1]==':'){ + start=p+2; if(state_path_sep(*start)) start++; + } else if(n>=2 && state_path_sep(p[0]) && state_path_sep(p[1])){ + /* Preserve the \\server\\share root and begin mkdir-p below it. */ + char *q=p+2; + while(*q && !state_path_sep(*q)) q++; + if(*q) q++; + while(*q && !state_path_sep(*q)) q++; + start=*q?q+1:q; + } +#endif + for(char *q=start;;q++){ + if(!state_path_sep(*q) && *q) continue; + char save=*q; *q=0; + if(*p){ + struct stat st; + if(state_mkdir_private(p) && errno!=EEXIST){ *q=save; return -1; } + if(stat(p,&st) || !S_ISDIR(st.st_mode)){ *q=save; errno=ENOTDIR; return -1; } + } + *q=save; if(!save) break; + } +#ifdef _WIN32 + return access(p,W_OK); +#else + return access(p,W_OK|X_OK); +#endif +} + int main(int argc, char **argv){ +#ifdef __linux__ + /* The managed placement contract is independent of OMP tuning/re-exec. + * Apply and read it back on every entry, including COLI_NO_OMP_TUNE=1, + * COLI_OMP_TUNED=1, and explicit CPU-only COLI_CUDA=0 launches. */ + const char *managed_cpu_affinity=getenv("COLI_CPU_AFFINITY"); + if(managed_cpu_affinity){ + if(coli_cpu_affinity_apply(managed_cpu_affinity)){ + fprintf(stderr,"[CPU] invalid or unavailable COLI_CPU_AFFINITY=%s: %s\n", + managed_cpu_affinity,strerror(errno)); + return 2; + } + fprintf(stderr,"[CPU] managed affinity: %s\n",managed_cpu_affinity); + } +#endif /* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are * tiny and back-to-back; with the default passive wait policy libgomp parks * the worker team between regions and the re-wake latency dominates. Keeping @@ -8803,7 +9533,9 @@ int main(int argc, char **argv){ * and COLI_NO_OMP_TUNE=1 is a documented kill-switch that disables the whole * re-exec + tuning path (distinct from the internal COLI_OMP_TUNED sentinel). * - * Must remain the FIRST statement in main(): argv is passed verbatim to execv(). */ + * This must remain the first tuning block in main(): argv is passed verbatim + * to execv(). Managed CPU-contract validation above is intentionally outside + * the optional tuning path. */ /* COLI_OMP_TUNED e COLI_NO_OMP_TUNE sono KILL-SWITCH: presence-based e' voluto * (c/coli lo documenta: "impostarla a qualsiasi valore, anche 0, disattiva"). * COLI_CUDA e COLI_METAL invece sono STATO, e uno 0 significa "niente GPU": @@ -8830,13 +9562,16 @@ int main(int argc, char **argv){ setenv("OMP_DYNAMIC","FALSE",0); /* fixed team size: no per-region thread-count churn */ setenv("COLI_OMP_TUNED","1",1); #ifdef __linux__ - fprintf(stderr,"[OMP] hot-thread tuning: re-exec once (COLI_NO_OMP_TUNE=1 to skip)\n"); /* #471: execv PRESERVES the CPU affinity mask. If the user exported * OMP_PROC_BIND/OMP_PLACES, libgomp's constructor already bound THIS thread to * place 0 (one core's SMT siblings) before main() ran; the re-exec'd image would * inherit that 1-core mask, enumerate OMP_PLACES=cores inside it, and jail the - * whole team on one core (measured ~20x slowdown). Reset to all online CPUs so - * the fresh libgomp binds from the full set — the user's OMP_* env still wins. */ + * whole team on one core (measured ~20x slowdown). A managed server launch + * supplies COLI_CPU_AFFINITY and gets that exact mask (including sparse CPU + * IDs); apply+readback fails closed if a cpuset/offline CPU narrows it. + * Unmanaged launches retain the legacy reset to all online CPUs so the fresh + * libgomp binds from the full set — the user's OMP_* env still wins. */ + if(!managed_cpu_affinity){ /* CPU_SETSIZE is only exposed when _GNU_SOURCE was defined before the first * system header. The standalone engine build defines it at the top of this * file so the reset is active where it matters; test TUs that #include this @@ -8850,6 +9585,8 @@ int main(int argc, char **argv){ if(sched_setaffinity(0, sizeof(all), &all) != 0) perror("[OMP] sched_setaffinity pre-reexec (continuing)"); } #endif + } + fprintf(stderr,"[OMP] hot-thread tuning: re-exec once (COLI_NO_OMP_TUNE=1 to skip)\n"); execv("/proc/self/exe", argv); /* returns only on failure -> fall through and run untuned */ perror("[OMP] execv self-reexec failed, running untuned"); #endif @@ -8875,12 +9612,30 @@ int main(int argc, char **argv){ } #endif const char *snap=getenv("SNAP"); if(!snap){fprintf(stderr,"SNAP=\n");return 1;} + const char *weights_dir=getenv("COLI_WEIGHTS_DIR"); + const char *state_env=getenv("COLI_STATE_DIR"); + const char *state_dir=state_env; + int state_explicit=state_env&&*state_env; + if(!weights_dir||!*weights_dir) weights_dir=snap; + if(!state_dir||!*state_dir) state_dir=snap; g_nopack = getenv("NOPACK")?1:0; g_drop = getenv("DROP")?1:0; g_prefetch = getenv("PREFETCH")?atoi(getenv("PREFETCH")):0; g_mmap = getenv("COLI_MMAP")?atoi(getenv("COLI_MMAP")):0; + g_rammap = getenv("COLI_RAMMAP")?atoi(getenv("COLI_RAMMAP")):0; + g_ram_prefault = getenv("COLI_RAM_PREFAULT")?atoi(getenv("COLI_RAM_PREFAULT")):0; + if(rammap_modes_conflict(g_mmap,g_rammap)){ + fprintf(stderr,"COLI_MMAP=1 and COLI_RAMMAP=1 are mutually exclusive\n"); return 2; + } +#ifndef __linux__ + if(g_rammap){ fprintf(stderr,"COLI_RAMMAP=1 is supported only on Linux\n"); return 2; } +#endif + if(state_explicit && state_dir_prepare(state_dir)){ + fprintf(stderr,"COLI_STATE_DIR=%s is not a writable directory: %s\n",state_dir,strerror(errno)); + return 2; + } if(g_mmap) fprintf(stderr,"[MMAP] expert = viste zero-copy nei file (page cache = cache)\n"); - numa_init(); /* COLI_NUMA=1: expert-slab interleave (#82) */ + if(numa_init()) return 2; /* exact managed NUMA masks fail closed */ g_topk = getenv("TOPK")?atoi(getenv("TOPK")):0; g_topp = getenv("TOPP")?atof(getenv("TOPP")):0; /* EXPERT_BUDGET e' sotto quarantena: la finestra operativa e' misurata VUOTA. @@ -9169,7 +9924,8 @@ int main(int argc, char **argv){ fprintf(stderr,"METAL: fast SSD (%.1f GB/s) — page cache favored, expert cache minimal (cap 1); override with --cap\n", coli_ssd_gbs); printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits); g_mem_avail_boot = mem_available_gb(); - Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits); + Model m; double t0=now_s(); model_init(&m,snap,weights_dir,cap,ebits,dbits); + rammap_build(&m); /* immutable direct tier precedes PIN/LRU */ if(!g_direct_heat_explicit){ /* COLI_DISKCLASS_WINDOW default, needs m.c (topk/n_layers) */ /* CURRENT-STATE CALIBRATION: the "8" multiplier (recency window ~= the last 8 * tokens' worth of routing) is a measured-config constant, not a derived truth. @@ -9226,16 +9982,21 @@ int main(int argc, char **argv){ "not ragged-safe across KV slots. Single-slot serve (KV_SLOTS=1) keeps MTP.\n"); else fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", eff_draft); -#ifdef __linux__ - { /* Only warn for a GENUINE 9p mount (WSL Windows drives, magic 0x01021997), where - * fadvise is a no-op. The old check was `snap` starting with "/mnt/", which - * false-positives on native-Linux ZFS/ext4/xfs/NFS mounts that also live under /mnt. */ - struct statfs sfb; - if(statfs(snap,&sfb)==0 && (unsigned long)sfb.f_type==0x01021997UL) - fprintf(stderr,"WARNING: the model is on %s (9p/Windows filesystem; fadvise is ineffective).\n" - " Keep it on a native Linux fs (ext4/xfs/zfs) for memory efficiency and speed.\n", snap); + { /* Inspect the opened weight descriptors rather than the SNAP pathname. This + * preserves the genuine-filesystem check without false-positive warnings for + * native mounts under /mnt, and also covers hybrid RAMMAP namespaces whose SSD + * fallback shards may live on 9p/DrvFS outside SNAP. */ + int slow_9p=0; + for(int i=0;i nessun pin (AUTOPIN piu' sotto resta escluso: PIN e' settato). * EN: prefer the live usage history over the frozen one-shot profile, so each * reload's pin placement follows the accumulated real workload. */ - snprintf(pauto,sizeof(pauto),"%s/.coli_usage",snap); + snprintf(pauto,sizeof(pauto),"%s/.coli_usage",state_dir); FILE *pf=fopen(pauto,"rb"); long psz=0; if(pf){ fseek(pf,0,SEEK_END); psz=ftell(pf); fclose(pf); } if(psz<=0){ snprintf(pauto,sizeof(pauto),"%s/stats.txt",snap); pf=fopen(pauto,"rb"); psz=0; if(pf){ fseek(pf,0,SEEK_END); psz=ftell(pf); fclose(pf); } } if(psz>0){ pin=pauto; fprintf(stderr,"[PIN] auto: seeding from %s\n",pauto); } - else { pin=NULL; fprintf(stderr,"[PIN] auto: no .coli_usage or stats.txt in %s yet (no pin this run)\n",snap); } + else { pin=NULL; fprintf(stderr,"[PIN] auto: no .coli_usage in %s or stats.txt in %s yet (no pin this run)\n",state_dir,snap); } } if(pin){ const char *pin_gb=getenv("PIN_GB"); @@ -9284,7 +10045,7 @@ int main(int argc, char **argv){ * conosce la TUA storia, la LRU si adatta alla sessione). AUTOPIN=0 disattiva. */ { double ram_env = getenv("RAM_GB")?atof(getenv("RAM_GB")):0.0; int est_ctx = getenv("CTX")?atoi(getenv("CTX")):4096; /* stesso default di run_serve */ - snprintf(g_usage_path,sizeof(g_usage_path),"%s/.coli_usage",snap); + snprintf(g_usage_path,sizeof(g_usage_path),"%s/.coli_usage",state_dir); int64_t hist = usage_load(&m,g_usage_path); if(hist>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",(long long)hist,g_usage_path); int autopin = getenv("AUTOPIN")?atoi(getenv("AUTOPIN")):1; @@ -9318,8 +10079,8 @@ int main(int argc, char **argv){ /* modo serve persistente per la CLI 'coli': SERVE=1 */ if(getenv("SERVE")){ - if(getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH"))) run_serve_mux(&m,snap); - else run_serve(&m,snap); + if(getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH"))) run_serve_mux(&m,snap,state_dir); + else run_serve(&m,snap,state_dir); if(stats) stats_dump(&m,stats); return 0; } diff --git a/c/compat.h b/c/compat.h index 61d55fd6b..91466fc92 100644 --- a/c/compat.h +++ b/c/compat.h @@ -438,13 +438,20 @@ static inline int coli_stdin_readable(void) * Sta QUI e non copiato in ogni motore: e' esattamente cosi' che era sparito. * * No-op su Linux/macOS. */ -static inline void coli_serve_binary_mode(void) +static inline void coli_serve_binary_mode_stream(FILE *stream) { #ifdef _WIN32 - _setmode(_fileno(stdin), _O_BINARY); - _setmode(_fileno(stdout), _O_BINARY); - setvbuf(stdout, NULL, _IONBF, 0); + _setmode(_fileno(stream), _O_BINARY); + setvbuf(stream, NULL, _IONBF, 0); +#else + (void)stream; #endif } +static inline void coli_serve_binary_mode(void) +{ + coli_serve_binary_mode_stream(stdin); + coli_serve_binary_mode_stream(stdout); +} + #endif /* COMPAT_H */ diff --git a/c/openai_server.py b/c/openai_server.py index c129714b3..c1978c9c8 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -41,6 +41,7 @@ def default_engine(): READY = b"\x01\x01READY\x01\x01\n" MAX_BODY = 4 << 20 PROFILE_TURNS = 120 # rolling window of per-turn PROF snapshots kept for /profile +ENGINE_READY_TIMEOUT = 7200.0 DEFAULT_CORS_ORIGINS = ( "http://127.0.0.1:8000", "http://localhost:8000", @@ -1369,18 +1370,69 @@ def cap_for_arch(arch, cap): class Engine: + @staticmethod + def _terminate_process(process): + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + @classmethod + def _wait_until_ready(cls, process, timeout): + outcome = queue.Queue(maxsize=1) + + def read_ready(): + try: + read_engine_turn(process.stdout, READY, lambda _: None) + except BaseException as error: + outcome.put(error) + else: + outcome.put(None) + + reader = threading.Thread(target=read_ready, name="colibri-ready", daemon=True) + reader.start() + try: + try: + error = outcome.get(timeout=timeout) + except queue.Empty: + raise RuntimeError( + "colibri engine did not become ready within %.3g seconds" % timeout + ) + if error is not None: + raise error + except BaseException: + cls._terminate_process(process) + reader.join(timeout=5) + raise + reader.join() + # cap=None = "not explicitly set": a glm-arch model's engine resolves the # 0 sentinel (8 historically, 1 on Metal+darwin+fast SSD -- colibri.c # coli_resolve_cap, #379), non-glm arches get the legacy 8, via # cap_for_arch above. Same convention as the --cap flags in coli and # main() below, so programmatic callers that never pass cap get the same # auto behavior as the CLI; an explicit int (0 included) is verbatim. - def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1): + def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_slots=1, + command_prefix=None, stderr=None): child_env = dict(env or os.environ, SNAP=str(model), SERVE="1", SERVE_BATCH="1", NGEN=str(max_tokens), KV_SLOTS=str(kv_slots)) + try: + ready_timeout = float(child_env.get("COLI_ENGINE_READY_TIMEOUT", + ENGINE_READY_TIMEOUT)) + except (TypeError, ValueError): + raise ValueError("COLI_ENGINE_READY_TIMEOUT must be numeric") + if not math.isfinite(ready_timeout) or not 0 < ready_timeout <= 86400: + raise ValueError("COLI_ENGINE_READY_TIMEOUT must be between 0 and 86400 seconds") + command = list(command_prefix or ()) + [ + str(executable), str(cap_for_arch(model_arch(model), cap)) + ] self.process = subprocess.Popen( - [str(executable), str(cap_for_arch(model_arch(model), cap))], env=child_env, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0, + command, env=child_env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=stderr, bufsize=0, ) self.write_lock = threading.Lock() self.pending_lock = threading.Lock() @@ -1391,12 +1443,16 @@ def __init__(self, executable, model, cap=None, max_tokens=1024, env=None, kv_sl self.kv_slots = kv_slots self.tiers = None self.hwinfo = None + self.gpus = [] + self.gpus_seq = 0 self.emap = None self.hits = None self.hits_seq = 0 # latest "TIERS" snapshot from the engine self.profile = collections.deque(maxlen=PROFILE_TURNS) # per-turn phase timings self.profile_seq = 0 - read_engine_turn(self.process.stdout, READY, lambda _: None) + self._pending_profile = None + self._pending_done_profile = None + self._wait_until_ready(self.process, ready_timeout) self.dispatcher = threading.Thread(target=self._dispatch_stdout, name="colibri-stdout", daemon=True) self.dispatcher.start() @@ -1414,6 +1470,16 @@ def _stats(fields): "length_limited": bool(int(fields[6])) if len(fields) > 6 else False, } + def _publish_profile(self, profile, stats): + profile.update({ + "tokens_per_second": stats["tokens_per_second"], + "cache_hit_percent": stats["cache_hit_percent"], + "rss_gb": stats["rss_gb"], + "length_limited": stats["length_limited"], + }) + self.profile.append(profile) + self.profile_seq += 1 + def _fail_pending(self, error): with self.pending_lock: requests = list(self.pending.values()) @@ -1466,6 +1532,17 @@ def _dispatch_stdout(self): elif kind == "DONE" and len(fields) >= 7: request_id = fields[1] stats = self._stats(fields[2:]) + profile = self._pending_profile + self._pending_profile = None + if profile is not None: + self._pending_done_profile = None + self._publish_profile(profile, stats) + else: + # Inkling emits DONE immediately before PROF, while + # the mux engine emits PROF immediately before DONE. + # Retain one adjacent DONE snapshot so both producers + # publish the same /profile schema. + self._pending_done_profile = dict(stats) with self.pending_lock: events = self.pending.pop(request_id, None) if events is not None: @@ -1483,8 +1560,9 @@ def _dispatch_stdout(self): self.hits = fields[3] self.hits_seq += 1 elif kind == "PROF" and len(fields) >= 10: - # per-turn phase timings: where the engine spent this turn's wall time - self.profile.append({ + # PROF has no request id. The mux engine emits it immediately + # before DONE and Inkling emits it immediately after DONE. + profile = { "wall_s": float(fields[1]), "prompt_tokens": int(fields[2]), "completion_tokens": int(fields[3]), @@ -1494,21 +1572,127 @@ def _dispatch_stdout(self): "attention_s": float(fields[7]), "lm_head_s": float(fields[8]), "forwards": int(fields[9]), - }) - self.profile_seq += 1 + } + if len(fields) >= 17: + physical_bytes = int(fields[12]) + if len(fields) >= 18: + physical_valid = bool(int(fields[17])) + else: + # Legacy producers used zero both for a measured + # zero and for unsupported accounting. A positive + # legacy count is known-valid; zero is unknown. + physical_valid = True if physical_bytes > 0 else None + profile.update({ + "forward_p50_ms": None if float(fields[10]) < 0 else float(fields[10]), + "forward_p99_ms": None if float(fields[11]) < 0 else float(fields[11]), + "physical_ssd_bytes": physical_bytes if physical_valid else None, + "physical_ssd_valid": physical_valid, + "rammap_experts": int(fields[13]), + "rammap_bytes": int(fields[14]), + "ttft_ms": float(fields[15]), + "prefault_seconds": float(fields[16]), + }) + stats = self._pending_done_profile + self._pending_done_profile = None + if stats is not None: + self._pending_profile = None + self._publish_profile(profile, stats) + else: + self._pending_profile = profile elif kind == "TIERS" and len(fields) >= 6: self.tiers = {"vram": int(fields[1]), "ram": int(fields[2]), "disk": int(fields[3]), "vram_gb": float(fields[4]), "ram_gb": float(fields[5])} + elif kind == "GPUS" and len(fields) >= 2: + # Legacy advisory telemetry: triples are card-wide used/total + # decimal GB and Colibri resident expert count. Preserve it + # for older engines while GPUDETAIL supplies byte-exact data. + count = int(fields[1]) + if count < 0 or len(fields) != 2 + count * 3: + raise RuntimeError(f"invalid engine GPUS: {' '.join(fields)}") + devices = [] + for index in range(count): + offset = 2 + index * 3 + used_gb = float(fields[offset]) + total_gb = float(fields[offset + 1]) + expert_count = int(fields[offset + 2]) + if (not math.isfinite(used_gb) or not math.isfinite(total_gb) or + used_gb < 0 or total_gb < 0 or expert_count < 0): + raise RuntimeError(f"invalid engine GPUS: {' '.join(fields)}") + devices.append({ + "device": index, + "identity": None, + "used_gb": used_gb, + "total_gb": total_gb, + "expert_count": expert_count, + }) + self.gpus = devices + self.gpus_seq += 1 + elif kind == "GPUDETAIL" and len(fields) >= 3: + version = int(fields[1]) + count = int(fields[2]) + if count < 0: + raise RuntimeError(f"invalid engine GPUDETAIL: {' '.join(fields)}") + if version != 1: + # A newer advisory schema is not protocol corruption. Its + # versioned body is deliberately opaque to this server. + continue + if len(fields) != 3 + count * 8: + raise RuntimeError(f"invalid engine GPUDETAIL: {' '.join(fields)}") + devices = [] + seen_devices = set() + for index in range(count): + offset = 3 + index * 8 + values = [int(value) for value in fields[offset + 2:offset + 8]] + if any(value < 0 for value in values): + raise RuntimeError( + f"invalid engine GPUDETAIL: {' '.join(fields)}" + ) + total_bytes, free_bytes, model_bytes, expert_bytes, \ + nonexpert_bytes, expert_count = values + device = int(fields[offset]) + if ( + device < 0 + or device in seen_devices + or free_bytes > total_bytes + or model_bytes > total_bytes + or expert_bytes + nonexpert_bytes != model_bytes + ): + raise RuntimeError( + f"invalid engine GPUDETAIL: {' '.join(fields)}" + ) + seen_devices.add(device) + devices.append({ + "device": device, + "identity": None if fields[offset + 1] == "-" else fields[offset + 1], + "total_bytes": total_bytes, + "free_bytes": free_bytes, + "used_bytes": max(0, total_bytes - free_bytes), + "model_bytes": model_bytes, + "expert_bytes": expert_bytes, + "nonexpert_bytes": nonexpert_bytes, + "expert_count": expert_count, + }) + self.gpus = devices + self.gpus_seq += 1 elif kind == "ERROR" and len(fields) >= 2: request_id = fields[1] message = " ".join(fields[2:]) or "engine request failed" + self._pending_profile = None + self._pending_done_profile = None with self.pending_lock: events = self.pending.pop(request_id, None) if events is not None: events.put(("error", _engine_error(fields[2:], message))) - else: + elif kind in { + "DATA", "ACCEPT", "DONE", "HWINFO", "EMAP", "HITS", + "PROF", "TIERS", "GPUS", "GPUDETAIL", "ERROR", + }: + # Unknown advisory kinds are intentionally ignored, but a + # malformed frame of a kind this server understands is fatal. raise RuntimeError(f"invalid engine response: {' '.join(fields)}") + else: + continue except Exception as error: if not self.closed: self.dispatcher_error = error @@ -1615,13 +1799,7 @@ def close(self): return self.closed = True self._fail_pending(RuntimeError("colibri engine is shutting down")) - if self.process.poll() is None: - self.process.terminate() - try: - self.process.wait(timeout=5) - except subprocess.TimeoutExpired: - self.process.kill() - self.process.wait(timeout=5) + self._terminate_process(self.process) if self.dispatcher is not threading.current_thread(): self.dispatcher.join(timeout=5) @@ -1630,6 +1808,31 @@ def model_object(model_id, created): return {"id": model_id, "object": "model", "created": created, "owned_by": "colibri"} +def _engine_health_error(engine): + """Return a stable public reason when the serving engine is unavailable.""" + if engine is None: + return "engine-unavailable" + if getattr(engine, "dispatcher_error", None) is not None: + return "dispatcher-error" + if getattr(engine, "closed", False): + return "engine-closed" + sentinel = object() + process = getattr(engine, "process", sentinel) + if process is sentinel: + # Lightweight in-process engines used by embedders/tests do not have a + # subprocess. Their generate() implementation is the serving engine. + return None + if process is None or not callable(getattr(process, "poll", None)): + return "process-status-unavailable" + try: + returncode = process.poll() + except (OSError, subprocess.SubprocessError): + return "process-status-unavailable" + if returncode is not None: + return "process-exited" + return None + + class APIServer(ThreadingHTTPServer): daemon_threads = True @@ -1848,6 +2051,15 @@ def do_GET(self): self._check_host() path = urlsplit(self.path).path if path == "/health": + engine = self.server.engine + engine_error = _engine_health_error(engine) + if engine_error is not None: + self.send_json( + 503, + {"status": "error", "reason": engine_error}, + request_id, + ) + return # Liveness is always public; hardware/scheduler internals only when a # request is authed (or no key set), so a configured key isn't leaked # past a bare 200 to an unauthenticated probe. (#SEC-8) @@ -1855,10 +2067,12 @@ def do_GET(self): if self._is_authed(): payload["scheduler"] = self.server.scheduler.snapshot() payload["kv_slots"] = self.server.kv_slots - tiers = getattr(self.server.engine, "tiers", None) if self.server.engine else None + tiers = getattr(engine, "tiers", None) if engine else None if tiers: payload["tiers"] = tiers - hwinfo = getattr(self.server.engine, "hwinfo", None) if self.server.engine else None + hwinfo = getattr(engine, "hwinfo", None) if engine else None if hwinfo: payload["hwinfo"] = hwinfo + payload["gpus"] = list(getattr(engine, "gpus", ()) or ()) if engine else [] + payload["gpus_seq"] = getattr(engine, "gpus_seq", 0) if engine else 0 self.send_json(200, payload, request_id) return if path == "/experts": diff --git a/c/resource_plan.py b/c/resource_plan.py index f0c6f2530..374588eee 100644 --- a/c/resource_plan.py +++ b/c/resource_plan.py @@ -13,6 +13,12 @@ GB = 1_000_000_000 EXPERT_RE = re.compile(r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.") +_DARWIN_RECLAIMABLE_PAGE_KEYS = ( + "Pages free", + "Pages inactive", + "Pages speculative", + "Pages purgeable", +) def _tensor_sizes(path): @@ -76,6 +82,78 @@ def analyze_model(model): } +def _parse_vm_stat(text, fallback_page_size): + """Return reclaimable macOS bytes from one ``vm_stat`` response.""" + if not isinstance(text, str): + return 0 + page_match = re.search(r"page size of (\d+) bytes", text) + page_size = ( + int(page_match.group(1)) + if page_match + else fallback_page_size + ) + if not isinstance(page_size, int) or page_size <= 0: + return 0 + pages = 0 + for key in _DARWIN_RECLAIMABLE_PAGE_KEYS: + match = re.search( + rf"^{re.escape(key)}:\s+(\d+)\.\s*$", + text, + re.MULTILINE, + ) + if match: + pages += int(match.group(1)) + return pages * page_size if pages else 0 + + +def _darwin_memory_available(*, run=None, fallback_page_size=None): + """Return reclaimable Darwin memory without depending on ``PATH``. + + Nix build sandboxes do not expose Apple's system utilities through PATH, + even though the native tools remain available at their stable system + locations. Injecting the command runner keeps parsing tests hermetic. + """ + run = subprocess.run if run is None else run + if fallback_page_size is None: + try: + fallback_page_size = os.sysconf("SC_PAGE_SIZE") + except (OSError, ValueError, AttributeError): + fallback_page_size = 0 + try: + result = run( + ["/usr/bin/vm_stat"], + text=True, + capture_output=True, + timeout=5, + ) + if result.returncode == 0: + available = _parse_vm_stat( + result.stdout, + fallback_page_size, + ) + if available: + return available + except (OSError, subprocess.SubprocessError): + pass + + # Preserve the existing conservative fallback to installed RAM, but use + # the absolute Apple tool path so it also works inside Nix builds. + try: + result = run( + ["/usr/sbin/sysctl", "-n", "hw.memsize"], + text=True, + capture_output=True, + timeout=5, + ) + if result.returncode == 0: + total = int(result.stdout.strip()) + if total > 0: + return total + except (OSError, subprocess.SubprocessError, ValueError): + pass + return 0 + + def memory_available(): # Linux (and MSYS2/Git-Bash CPython where /proc exists): MemAvailable. try: @@ -116,30 +194,11 @@ class MEMORYSTATUSEX(ctypes.Structure): return total_kb.value * 1024 except OSError: pass - # macOS: no /proc and not win32. Sum the reclaimable pages reported by vm_stat - # (free + inactive + speculative + purgeable) — the same "reclaimable without swapping" - # definition the C engine's compat_meminfo uses. Fall back to total RAM (never 0 on a Mac). + # macOS: no /proc and not win32. Sum the reclaimable pages reported by + # vm_stat (free + inactive + speculative + purgeable) — the same + # "reclaimable without swapping" definition the C engine uses. if sys.platform == "darwin": - try: - out = subprocess.run(["vm_stat"], text=True, capture_output=True, timeout=5).stdout - page_match = re.search(r"page size of (\d+) bytes", out) - page = int(page_match.group(1)) if page_match else os.sysconf("SC_PAGE_SIZE") - pages = 0 - for key in ("Pages free", "Pages inactive", "Pages speculative", "Pages purgeable"): - match = re.search(rf"{key}:\s+(\d+)\.", out) - if match: - pages += int(match.group(1)) - if pages: - return pages * page - except (OSError, subprocess.SubprocessError, ValueError): - pass - try: - total = subprocess.run(["sysctl", "-n", "hw.memsize"], text=True, - capture_output=True, timeout=5).stdout.strip() - if total: - return int(total) - except (OSError, subprocess.SubprocessError, ValueError): - pass + return _darwin_memory_available() return 0 diff --git a/c/st.h b/c/st.h index fd20e6346..56c601656 100644 --- a/c/st.h +++ b/c/st.h @@ -17,9 +17,25 @@ #include #include #include +#ifdef __linux__ +#include +#endif #include "json.h" #include "compat.h" +/* Keep the filesystem probes dependency-free. linux/magic.h is not available + * in every libc/sysroot used by the portable build, so define the two values we + * need when the platform headers did not provide them. */ +#ifndef TMPFS_MAGIC +#define TMPFS_MAGIC 0x01021994 +#endif +#ifndef NINEP_SUPER_MAGIC +#define NINEP_SUPER_MAGIC 0x01021997 +#endif +#ifndef WSLFS_MAGIC +#define WSLFS_MAGIC 0x53464846 +#endif + /* tetto sulla dimensione dell'header safetensors: gli header reali sono piccoli * (KB..pochi MB). Un file crafted che dichiara un hlen enorme causerebbe una * malloc gigante prima ancora di leggere: lo respingiamo. */ @@ -40,6 +56,8 @@ typedef struct { int fds[512]; int dfds[512]; /* gemelli O_DIRECT (aperti pigramente): -2 = non ancora provato */ char *paths[512]; + long fs_magic[512]; /* fstatfs result for the opened descriptor (Linux) */ + unsigned char is_tmpfs[512]; /* descriptor backing, not pathname spelling/symlink */ int nfd; #define ST_MAX_MIR 4 /* extra read replicas beyond the primary (multi-SSD) */ int mfds[ST_MAX_MIR][512]; /* MIRROR: fds of replica copy r+1 (multi-SSD), -1 = absent */ @@ -134,15 +152,29 @@ static inline float f16_to_f32(uint16_t h) { static int st_open_fd(shards *S, const char *path) { for (int i = 0; i < S->nfd; i++) if (!strcmp(S->paths[i], path)) return S->fds[i]; + if (S->nfd >= ST_MAX_SHARDS) { + fprintf(stderr, "too many open shards (>%d)\n", ST_MAX_SHARDS); exit(1); + } int fd = open(path, COMPAT_O_RDONLY); if (fd < 0) { perror(path); exit(1); } - S->paths[S->nfd] = strdup(path); S->fds[S->nfd] = fd; + int si=S->nfd; + S->paths[si] = strdup(path); S->fds[si] = fd; +#ifdef __linux__ + struct statfs sfs; + if(fstatfs(fd,&sfs)==0){ + S->fs_magic[si]=(long)sfs.f_type; + S->is_tmpfs[si]=((unsigned long)sfs.f_type==(unsigned long)TMPFS_MAGIC); + } +#endif #ifdef O_DIRECT - S->dfds[S->nfd] = open(path, COMPAT_O_RDONLY | O_DIRECT); /* eager: lookup poi thread-safe */ + /* O_DIRECT has no value for tmpfs and may fail with EINVAL on older kernels. + * More importantly, the RAM-map path must not manufacture a second descriptor + * whose semantics suggest physical storage for an in-memory shard. */ + S->dfds[si] = S->is_tmpfs[si] ? -1 : open(path, COMPAT_O_RDONLY | O_DIRECT); /* eager: lookup poi thread-safe */ #elif defined(__APPLE__) || defined(_WIN32) - S->dfds[S->nfd] = compat_open_direct(path); /* macOS: F_NOCACHE; Windows: NO_BUFFERING */ + S->dfds[si] = compat_open_direct(path); /* macOS: F_NOCACHE; Windows: NO_BUFFERING */ #else - S->dfds[S->nfd] = -1; /* niente equivalente: solo buffered */ + S->dfds[si] = -1; /* niente equivalente: solo buffered */ #endif S->nfd++; return fd; @@ -158,6 +190,17 @@ static int st_direct_fd(shards *S, int fd) { int i = st_fidx(S, fd); return i < 0 ? -1 : S->dfds[i]; } +static int st_fd_slot(const shards *S, int fd) { + for(int i=0;infd;i++) if(S->fds[i]==fd) return i; + return -1; +} +static int st_fd_is_tmpfs(const shards *S, int fd) { + int i=st_fd_slot(S,fd); return i>=0 && S->is_tmpfs[i]; +} +static long st_fd_fs_magic(const shards *S, int fd) { + int i=st_fd_slot(S,fd); return i>=0 ? S->fs_magic[i] : 0; +} + /* ---- MIRROR (multi-SSD): read-only copies of the model on other drives ---- * st_fd_rep/st_direct_fd_rep: fd of replica `rep` (0 = primary, 1..nrep = * mirrors) for the SAME file identified by its primary fd. -1 if absent. */ diff --git a/c/telemetry.h b/c/telemetry.h index 53725f3f9..4182f29e2 100644 --- a/c/telemetry.h +++ b/c/telemetry.h @@ -4,6 +4,11 @@ #ifndef TELEMETRY_H #define TELEMETRY_H +/* PR #377: forward decl — rammap_slot() is defined in colibri.c below the point + * this header is #included, but emap_emit/tiers_emit consult it for the tmpfs + * direct tier. telemetry.h is included after Model/ESlot are defined. */ +static ESlot *rammap_slot(Model *m, int layer, int eid); + static int64_t tbytes(int O,int I,int bits){ if(bits>=16) return (int64_t)O*I*4; if(bits>=5) return (int64_t)O*I + (int64_t)O*4; @@ -90,6 +95,93 @@ static void hw_probe(char *cpu, size_t cn, int *cores, double *ram_total, double #endif } +/* Per-device CUDA placement. Keep the documented GPUS line for older + * dashboards, then publish an integer-byte, versioned record for control-plane + * consumers that need to distinguish model tensors from card-wide VRAM use. + * + * GPUDETAIL v1 record (eight fields per device): + * + * + * + * The CUDA backend currently exposes ordinals but not PCI/UUID identity, so + * identity is "-". The reserved token makes adding a backend identity query + * wire-compatible later. */ +static void gpus_emit(Model *m){ + int ndev=0,valid_count=0; +#ifdef COLI_CUDA + if(g_cuda_enabled) ndev=g_cuda_ndev; + uint64_t expert_bytes[COLI_CUDA_MAX_DEVICES]={0}; + int expert_count[COLI_CUDA_MAX_DEVICES]={0}; + size_t free_bytes[COLI_CUDA_MAX_DEVICES]={0}; + size_t total_bytes[COLI_CUDA_MAX_DEVICES]={0}; + size_t model_bytes[COLI_CUDA_MAX_DEVICES]={0}; + unsigned char detail_valid[COLI_CUDA_MAX_DEVICES]={0}; + if(ndev){ + Cfg *c=&m->c; + for(int li=0;li<=c->n_layers;li++) for(int z=0;znpin[li];z++){ + ESlot *s=&m->pin[li][z]; + if(!s->g.cuda && !s->u.cuda && !s->d.cuda) continue; + int device=s->g.cuda?s->g.cuda_device: + s->u.cuda?s->u.cuda_device:s->d.cuda_device; + int di=-1; + for(int i=0;ig.cuda) + +(uint64_t)coli_cuda_tensor_bytes(s->u.cuda) + +(uint64_t)coli_cuda_tensor_bytes(s->d.cuda); + } + for(int i=0;i0 + && free_bytes[i]<=total_bytes[i] + && model_bytes[i]<=total_bytes[i] + && expert_bytes[i]<=(uint64_t)model_bytes[i]){ + detail_valid[i]=1; + valid_count++; + } + } + } +#else + (void)m; +#endif + /* GPUS has implicit ordinal positions, so a partial sample cannot be + * represented safely. Fail that legacy advisory closed as an empty set. */ + int legacy_count=valid_count==ndev?ndev:0; + printf("GPUS %d",legacy_count); +#ifdef COLI_CUDA + if(legacy_count) for(int i=0;ic; (void)c; char cpu[256]; int cores; double ram_total,ram_avail; @@ -108,6 +200,7 @@ static void hwinfo_emit(Model *m){ printf("HWINFO %d %.1f %.1f %d %.1f %s|%s\n", cores,ram_total,ram_avail,ngpu,vram_total,cpu,gpu_name); fflush(stdout); + gpus_emit(m); } static void tiers_emit(Model *m){ @@ -120,10 +213,12 @@ static void tiers_emit(Model *m){ #ifdef COLI_CUDA vram=m->gpu_expert_count; vram_gb=m->gpu_expert_bytes/1e9; #endif - int ram=pinned-vram+lru; if(ram<0) ram=0; + int anon_ram=pinned-vram+lru; if(anon_ram<0) anon_ram=0; + int ram=anon_ram+m->rammap_experts; int disk=total-vram-ram; if(disk<0) disk=0; double eb=(double)expert_bytes_probe(m,m->ebits); - printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram*eb/1e9); + double ram_gb=anon_ram*eb/1e9+m->rammap_bytes/1e9; + printf("TIERS %d %d %d %.2f %.2f\n",vram,ram,disk,vram_gb,ram_gb); fflush(stdout); } @@ -139,9 +234,9 @@ static void emap_emit(Model *m){ int is_row = (in_layers && m->L[i].sparse) || (i==c->n_layers && has_mtp); if(!is_row) continue; for(int e=0;epin[i]; - for(int z=0;znpin[i];z++) if(P[z].eid==e){ + for(int z=0;!tier && znpin[i];z++) if(P[z].eid==e){ #ifdef COLI_CUDA tier = P[z].g.cuda?2:1; #else diff --git a/c/tests/test_backend_cuda.cu b/c/tests/test_backend_cuda.cu index 9829db9f8..478e6bbc1 100644 --- a/c/tests/test_backend_cuda.cu +++ b/c/tests/test_backend_cuda.cu @@ -156,6 +156,10 @@ static int test_fmt6(int dev) { ColiCudaTensor *t6 = nullptr; int ok = coli_cuda_matmul(&t6, got, x, q, nullptr, 6, S, I, O, dev, 0); if (!ok) { std::fprintf(stderr,"fmt=6 matmul rejected\n"); return 0; } + if (coli_cuda_tensor_bytes(t6) != (size_t)O*nb*T6_BB) { + std::fprintf(stderr,"fmt=6 tensor bytes include an unallocated scale buffer\n"); + return 0; + } if (!relative_rms(got, want, S*O, 1e-4f)) { std::fprintf(stderr,"fmt=6 matmul mismatch\n"); return 0; } /* --- expert MLP: gate/up, silu, the device-side down rotation, down --- @@ -199,6 +203,12 @@ static int test_fmt6(int dev) { !coli_cuda_tensor_upload(&td6,qd,nullptr,6,O,I,dev)) { std::fprintf(stderr,"fmt=6 expert upload failed\n"); return 0; } + if (coli_cuda_tensor_bytes(tg6) != (size_t)O*nb*T6_BB || + coli_cuda_tensor_bytes(tu6) != (size_t)O*nb*T6_BB || + coli_cuda_tensor_bytes(td6) != (size_t)I*nbd*T6_BB) { + std::fprintf(stderr,"fmt=6 expert tensor byte accounting mismatch\n"); + return 0; + } float *got_e = (float*)std::malloc((size_t)S*I*sizeof(float)); if (!coli_cuda_expert_mlp(tg6,tu6,td6,got_e,x,S)) { std::fprintf(stderr,"fmt=6 expert_mlp rejected\n"); return 0; diff --git a/c/tests/test_fp8_e2e_repack_load.py b/c/tests/test_fp8_e2e_repack_load.py index c38cd4d7b..e4a394df9 100644 --- a/c/tests/test_fp8_e2e_repack_load.py +++ b/c/tests/test_fp8_e2e_repack_load.py @@ -43,7 +43,13 @@ def _cc_flags(): (falls back to single-threaded -- exactly like the Makefile's own OMPDIR probe -- if it's not, rather than failing the build). Returns (cc, cflags, ldflags) or (None, None, None) if no compiler is found.""" - cc = shutil.which("cc") or shutil.which("clang") or shutil.which("gcc") + # Mirror the Makefile's compiler choice: Apple Clang on Darwin, GCC on + # every other supported target. A generic `cc` may be Clang without + # libomp even when the production GCC toolchain is installed. + if sys.platform == "darwin": + cc = shutil.which("clang") or shutil.which("cc") + else: + cc = shutil.which("gcc") if not cc: return None, None, None cflags = ["-O3", "-Wall", "-Wextra", "-Wno-unused-parameter", diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py index 77225a6ea..647b3e2e5 100644 --- a/c/tests/test_openai_server.py +++ b/c/tests/test_openai_server.py @@ -1,8 +1,10 @@ +import contextlib import http.client import io import json import math import socket +import subprocess import tempfile import threading import unittest @@ -15,7 +17,7 @@ DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler, READY, Engine, InklingStreamSplit, StopFilter, ThinkingStreamSplit, _engine_error, cap_for_arch, conversation_cache_slot, model_arch, - generation_options, parse_tool_calls, read_engine_turn, + default_engine, generation_options, parse_tool_calls, read_engine_turn, render_chat, render_chat_kimi, serve, split_thinking_reply, stop_policy) @@ -233,6 +235,11 @@ def test_strict_mode_still_stops_on_a_leading_match(self): class ProtocolTest(unittest.TestCase): + def test_default_engine_uses_current_binary_name(self): + with patch("openai_server.Path.exists", autospec=True, + side_effect=lambda candidate: candidate.name == "colibri.exe"): + self.assertEqual(default_engine().name, "colibri.exe") + def test_reads_payload_and_extended_status(self): stream = io.BytesIO(b"hello" + END + b"STAT 2 3.5 44 1.2 7 1\n") chunks = [] @@ -412,6 +419,80 @@ def kill(self): self.terminate() +class StartupProcess: + def __init__(self, output=b"", ignore_terminate=False): + self.stdout = BlockingStream(output) + self.stdin = io.BytesIO() + self.returncode = None + self.ignore_terminate = ignore_terminate + self.terminate_calls = 0 + self.kill_calls = 0 + self.wait_calls = [] + + def poll(self): + return self.returncode + + def terminate(self): + self.terminate_calls += 1 + if not self.ignore_terminate: + self.returncode = 0 + self.stdout.close() + + def wait(self, timeout=None): + self.wait_calls.append(timeout) + if self.returncode is None: + raise subprocess.TimeoutExpired("glm", timeout) + return self.returncode + + def kill(self): + self.kill_calls += 1 + self.returncode = -9 + self.stdout.close() + + +class EngineStartupTest(unittest.TestCase): + def test_readiness_parse_failure_reaps_stubborn_child_before_reraising(self): + process = StartupProcess(READY + b"BROKEN\n", ignore_terminate=True) + with patch("openai_server.subprocess.Popen", return_value=process): + with self.assertRaisesRegex(RuntimeError, "invalid engine status: BROKEN"): + Engine("glm", "model") + + self.assertEqual(process.terminate_calls, 1) + self.assertEqual(process.kill_calls, 1) + self.assertEqual(process.wait_calls, [5, 5]) + self.assertEqual(process.returncode, -9) + + def test_readiness_timeout_terminates_child_without_changing_constructor_api(self): + process = StartupProcess() + errors = [] + + def construct(): + try: + Engine( + "glm", + "model", + env={"COLI_ENGINE_READY_TIMEOUT": "0.01"}, + ) + except BaseException as error: + errors.append(error) + + with patch("openai_server.subprocess.Popen", return_value=process): + thread = threading.Thread(target=construct, daemon=True) + thread.start() + thread.join(timeout=0.5) + finished_within_bound = not thread.is_alive() + if thread.is_alive(): + process.terminate() + thread.join(timeout=1) + + self.assertTrue(finished_within_bound, "Engine readiness wait was unbounded") + self.assertEqual(len(errors), 1) + self.assertRegex(str(errors[0]), "did not become ready within") + self.assertEqual(process.terminate_calls, 1) + self.assertEqual(process.kill_calls, 0) + self.assertEqual(process.wait_calls, [5]) + + class DispatcherTest(unittest.TestCase): def test_dispatches_interleaved_requests_by_id(self): submitted = [] @@ -542,8 +623,196 @@ def respond(process, frame): "wall_s": 2.5, "prompt_tokens": 7, "completion_tokens": 12, "expert_disk_s": 0.4, "expert_wait_s": 0.1, "expert_matmul_s": 0.9, "attention_s": 0.6, "lm_head_s": 0.2, "forwards": 15, + "tokens_per_second": 4.8, "cache_hit_percent": 0.0, + "rss_gb": 1.0, "length_limited": False, }]) + def test_records_inkling_done_then_prof_profile_order(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed( + b"DONE " + request_id + + b" STAT 12 4.8 25 1.0 7 0\n" + ) + process.stdout.feed( + b"PROF 2.500 7 12 0.400 0.100 0.900 " + b"0.600 0.200 15\n" + ) + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 16, 0.7, 0.9, lambda _: None) + for _ in range(100): + if engine.profile_seq: + break + threading.Event().wait(0.01) + engine.close() + + self.assertEqual(engine.profile_seq, 1) + profile = list(engine.profile)[0] + self.assertEqual(profile["prompt_tokens"], 7) + self.assertEqual(profile["completion_tokens"], 12) + self.assertEqual(profile["tokens_per_second"], 4.8) + self.assertEqual(profile["cache_hit_percent"], 25.0) + + def test_records_legacy_and_detailed_gpu_telemetry(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed( + b"GPUS 2 0.600 1.000 3 1.300 2.000 6\n" + b"GPUDETAIL 1 2 " + b"2 - 1000 400 500 300 200 3 " + b"5 GPU-abc 2000 700 900 600 300 6\n" + b"DONE " + request_id + b" STAT 1 2.5 0 1.0 4 0\n" + ) + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 4, 0.7, 0.9, lambda _: None) + engine.close() + + self.assertEqual(engine.gpus_seq, 2) + self.assertEqual(engine.gpus, [ + { + "device": 2, "identity": None, + "total_bytes": 1000, "free_bytes": 400, "used_bytes": 600, + "model_bytes": 500, "expert_bytes": 300, "nonexpert_bytes": 200, + "expert_count": 3, + }, + { + "device": 5, "identity": "GPU-abc", + "total_bytes": 2000, "free_bytes": 700, "used_bytes": 1300, + "model_bytes": 900, "expert_bytes": 600, "nonexpert_bytes": 300, + "expert_count": 6, + }, + ]) + + def test_legacy_gpu_telemetry_remains_available_without_detail_frame(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed(b"GPUS 1 1.250 24.000 7\n") + process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 4 0\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 4, 0.7, 0.9, lambda _: None) + engine.close() + self.assertEqual(engine.gpus_seq, 1) + self.assertEqual(engine.gpus, [{ + "device": 0, "identity": None, "used_gb": 1.25, + "total_gb": 24.0, "expert_count": 7, + }]) + + def test_empty_gpu_detail_reports_cpu_only_engine(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed(b"GPUS 0\nGPUDETAIL 1 0\n") + process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 4 0\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 4, 0.7, 0.9, lambda _: None) + engine.close() + self.assertEqual(engine.gpus, []) + self.assertEqual(engine.gpus_seq, 2) + + def test_ignores_unknown_advisory_telemetry(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed(b"FUTURE_TELEMETRY any shape is advisory\n") + process.stdout.feed(b"DONE " + request_id + b" STAT 1 2.5 0 1.0 4 0\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + stats = engine.generate("hello", 4, 0.7, 0.9, lambda _: None) + engine.close() + self.assertEqual(stats["tokens_per_second"], 2.5) + + def test_malformed_known_gpu_telemetry_stops_dispatcher(self): + def respond(process, _frame): + process.stdout.feed(b"GPUDETAIL 1 1 0 - 100\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + with self.assertRaisesRegex(RuntimeError, "GPUDETAIL"): + engine.generate("hello", 4, 0.7, 0.9, lambda _: None) + engine.close() + + def test_gpu_detail_rejects_duplicate_and_impossible_device_rows(self): + frames = { + "duplicate ordinal": ( + b"GPUDETAIL 1 2 " + b"0 - 1000 400 500 300 200 3 " + b"0 - 2000 700 900 600 300 6\n" + ), + "free exceeds total": ( + b"GPUDETAIL 1 1 0 - 1000 1001 500 300 200 3\n" + ), + "model exceeds total": ( + b"GPUDETAIL 1 1 0 - 1000 400 1001 600 401 6\n" + ), + } + for label, telemetry in frames.items(): + with self.subTest(label=label): + def respond(process, _frame, telemetry=telemetry): + process.stdout.feed(telemetry) + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + with self.assertRaisesRegex(RuntimeError, "GPUDETAIL"): + engine.generate("hello", 4, 0.7, 0.9, lambda _: None) + engine.close() + + def test_records_extended_persistent_benchmark_telemetry(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed(b"DATA " + request_id + b" 2\nok\n") + process.stdout.feed( + b"PROF 2.500 7 32 0.400 0.100 0.900 0.600 0.200 15 " + b"12.500 44.000 4096 8 65536 123.000 1.250\n" + ) + process.stdout.feed(b"DONE " + request_id + b" STAT 32 12.8 0 1.0 7 1\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 32, 0.0, 1.0, lambda _: None) + engine.close() + profile = list(engine.profile)[0] + self.assertEqual(profile["forward_p50_ms"], 12.5) + self.assertEqual(profile["forward_p99_ms"], 44.0) + self.assertEqual(profile["physical_ssd_bytes"], 4096) + self.assertIs(profile["physical_ssd_valid"], True) + self.assertEqual(profile["rammap_experts"], 8) + self.assertEqual(profile["rammap_bytes"], 65536) + self.assertEqual(profile["ttft_ms"], 123.0) + self.assertEqual(profile["prefault_seconds"], 1.25) + + def test_extended_profile_distinguishes_unavailable_physical_io(self): + def respond(process, frame): + request_id = frame.split()[1] + process.stdout.feed( + b"PROF 2.500 7 32 0.400 0.100 0.900 0.600 0.200 15 " + b"12.500 44.000 0 8 65536 123.000 1.250 0\n" + ) + process.stdout.feed(b"DONE " + request_id + b" STAT 32 12.8 0 1.0 7 1\n") + + process = FakeProcess(respond) + with patch("openai_server.subprocess.Popen", return_value=process): + engine = Engine("glm", "model") + engine.generate("hello", 32, 0.0, 1.0, lambda _: None) + engine.close() + profile = list(engine.profile)[0] + self.assertIsNone(profile["physical_ssd_bytes"]) + self.assertIs(profile["physical_ssd_valid"], False) + def test_cancels_generation_after_consumer_disconnects(self): request_id = None @@ -708,6 +977,53 @@ def test_health_reports_scheduler_and_kv_slots(self): self.assertEqual(scheduler["max_queue"], 8) self.assertIn("queued", scheduler) self.assertEqual(health["kv_slots"], 2) + self.assertEqual(health["gpus"], []) + self.assertEqual(health["gpus_seq"], 0) + + def test_health_exposes_gpu_details_only_to_authenticated_callers(self): + gpu = { + "device": 2, "identity": "GPU-abc", + "total_bytes": 1000, "free_bytes": 400, "used_bytes": 600, + "model_bytes": 500, "expert_bytes": 300, "nonexpert_bytes": 200, + "expert_count": 3, + } + self.engine.gpus = [gpu] + self.engine.gpus_seq = 7 + try: + with self.request("/health") as response: + health = json.load(response) + self.assertEqual(health["gpus"], [gpu]) + self.assertEqual(health["gpus_seq"], 7) + + with urlopen(self.base + "/health", timeout=2) as response: + public = json.load(response) + self.assertEqual(public, {"status": "ok"}) + finally: + del self.engine.gpus, self.engine.gpus_seq + + def test_health_is_unavailable_after_dispatcher_or_child_failure(self): + failures = ( + ("dispatcher-error", RuntimeError("engine reader failed"), None), + ("process-exited", None, 17), + ) + for reason, dispatcher_error, returncode in failures: + with self.subTest(reason=reason): + self.engine.dispatcher_error = dispatcher_error + self.engine.process = type( + "ProcessState", + (), + {"poll": lambda _self, value=returncode: value}, + )() + try: + with self.assertRaises(HTTPError) as caught: + urlopen(self.base + "/health", timeout=2) + self.assertEqual(caught.exception.code, 503) + self.assertEqual( + json.load(caught.exception), + {"status": "error", "reason": reason}, + ) + finally: + del self.engine.dispatcher_error, self.engine.process def test_profile_reports_recent_turns_without_auth(self): with urlopen(self.base + "/profile", timeout=2) as response: @@ -890,6 +1206,12 @@ def test_static_root_stays_inside_dist_directory(self): urlopen(self.base + "/%2e%2e/dist-private/secret.txt", timeout=2) self.assertEqual(caught.exception.code, 404) + def test_health_without_configured_key_exposes_gpu_telemetry_shape(self): + with urlopen(self.base + "/health", timeout=2) as response: + health = json.load(response) + self.assertEqual(health["gpus"], []) + self.assertEqual(health["gpus_seq"], 0) + class SchedulerHTTPTest(unittest.TestCase): def setUp(self): @@ -1543,11 +1865,20 @@ def test_anthropic_stream_announces_close_framing(self): def test_engine_failure_after_commit_does_not_splice_a_second_response(self): """Once the 200 is out, a 500 status line would land inside the event stream.""" server = self._server(_ExplodingEngine()) - raw = self._raw(server, self._request_bytes(dict(self.CHAT, stream=True))) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + raw = self._raw( + server, + self._request_bytes(dict(self.CHAT, stream=True)), + ) self.assertEqual(raw.count("HTTP/1."), 1, "a second HTTP response was spliced into the committed SSE stream") self.assertIn("partial", raw) # the events sent before the failure survive self.assertNotIn("", raw) + self.assertIn( + "request failed: engine died mid-stream", + stderr.getvalue(), + ) def test_non_streaming_response_still_reuses_the_connection(self): """The fix must not turn every response into a close: plain JSON stays persistent.""" diff --git a/c/tests/test_openai_tools_e2e.py b/c/tests/test_openai_tools_e2e.py index c6c43e693..ac447075c 100644 --- a/c/tests/test_openai_tools_e2e.py +++ b/c/tests/test_openai_tools_e2e.py @@ -23,8 +23,7 @@ # Mock engine: replies are keyed on the prompt so one process covers every case. # Prompts received are appended to MOCK_LOG for assertions on the rendering. -MOCK_ENGINE = r'''#!/usr/bin/env python3 -import sys, os +MOCK_ENGINE = r'''import sys, os out, inp = sys.stdout.buffer, sys.stdin.buffer out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush() @@ -81,7 +80,7 @@ class ToolCallingE2E(unittest.TestCase): def setUpClass(cls): cls.tmp = tempfile.TemporaryDirectory() mock = Path(cls.tmp.name) / "mock_engine.py" - mock.write_text(MOCK_ENGINE) + mock.write_text("#!%s\n%s" % (sys.executable, MOCK_ENGINE)) mock.chmod(0o755) cls.mock_log = Path(cls.tmp.name) / "prompts.log" cls.mock_log.touch() diff --git a/c/tests/test_rammap.c b/c/tests/test_rammap.c new file mode 100644 index 000000000..474d157e8 --- /dev/null +++ b/c/tests/test_rammap.c @@ -0,0 +1,292 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif +#include + +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +#ifdef __linux__ +static int fail(const char *what){ fprintf(stderr,"FAIL: %s\n",what); return 1; } + +static int expert_greedy_token(ESlot *slot,const float x[4],float out[4]){ + float gate[3],up[3],hidden[3]; + matmul_qt(gate,x,&slot->g,1); matmul_qt(up,x,&slot->u,1); + for(int i=0;i<3;i++) hidden[i]=siluf(gate[i])*up[i]; + matmul_qt(out,hidden,&slot->d,1); + int token=0; for(int i=1;i<4;i++) if(out[i]>out[token]) token=i; + return token; +} + +static int make_regular_temp(char *path,size_t cap){ + const char *roots[]={"/var/tmp","/tmp","."}; + for(size_t i=0;iS.t[m->S.n++]; + *w=(st_tensor){strdup(name),tfd,wo,wb[k],3,wb[k]}; wo+=wb[k]; + size_t n=strlen(name); memcpy(name+n,".qs",4); + st_tensor *q=&m->S.t[m->S.n++]; + int fd=(expert==1 && k==2)?mixed_fd:tfd; + *q=(st_tensor){strdup(name),fd,so,sb[k],2,sb[k]/4}; so+=sb[k]; + } +} + +static int64_t add_quant_expert(Model *m,int fd,long fs_magic,int fmt, + int hidden,int moe_inter,int64_t base){ + static const char *proj[3]={"gate_proj","up_proj","down_proj"}; + int OO[3]={moe_inter,moe_inter,hidden},II[3]={hidden,hidden,moe_inter}; + memset(m,0,sizeof(*m)); m->c.hidden=hidden; m->c.moe_inter=moe_inter; + m->S.cap=6; m->S.t=calloc(6,sizeof(st_tensor)); + m->S.fds[0]=fd; m->S.fs_magic[0]=fs_magic; m->S.is_tmpfs[0]=1; m->S.nfd=1; + int64_t off=base,total=0; + for(int k=0;k<3;k++){ + int O=OO[k],I=II[k]; + int64_t wb=fmt==6 ? (int64_t)O*e8_rowbytes(I) : (int64_t)O*I; + int64_t sb=fmt==6 ? 4 : fp8_nblk(O)*fp8_nblk(I)*4; + char name[300]; + snprintf(name,sizeof(name),"model.layers.0.mlp.experts.0.%s.weight",proj[k]); + m->S.t[m->S.n++]=(st_tensor){strdup(name),fd,off,wb,3,wb}; off+=wb; + size_t n=strlen(name); memcpy(name+n,".qs",4); + m->S.t[m->S.n++]=(st_tensor){strdup(name),fd,off,sb,2,sb/4}; off+=sb; + total+=wb+sb; + } + return total; +} + +static void free_quant_expert(Model *m){ + for(int i=0;iS.n;i++) free(m->S.t[i].name); + free(m->S.t); +} + +static int check_direct_quant_format(int fd,long fs_magic,int fmt,int hidden,int moe_inter, + int64_t base,int64_t expected_total){ + Model m; int64_t fixture_total=add_quant_expert(&m,fd,fs_magic,fmt,hidden,moe_inter,base); + ESlot slot={0}; int64_t got=rammap_bind_one(&m,0,0,&slot); + QT *q[3]={&slot.g,&slot.u,&slot.d}; + int OO[3]={moe_inter,moe_inter,hidden},II[3]={hidden,hidden,moe_inter}; + int bad=fixture_total!=expected_total || got!=expected_total || + slot.eid!=0 || slot.backing!=ESLOT_BACKING_RAMMAP; + for(int k=0;k<3;k++){ + int64_t want_weight=fmt==6 ? (int64_t)OO[k]*e8_rowbytes(II[k]) + : (int64_t)OO[k]*II[k]; + int64_t want_scale=fmt==6 ? 4 : fp8_nblk(OO[k])*fp8_nblk(II[k])*4; + if(q[k]->fmt!=fmt || q[k]->O!=OO[k] || q[k]->I!=II[k] || q[k]->gs!=0 || + qt_scale_bytes(q[k])!=want_scale || qt_bytes(q[k])-want_scale!=want_weight || + (fmt==6 ? q[k]->q4==NULL : q[k]->q8==NULL)) bad=1; + } + free_quant_expert(&m); + return bad ? fail(fmt==6 ? "direct E8 binding / geometry" : + "direct FP8 binding / geometry") : 0; +} +#endif + +int main(void){ +#ifndef __linux__ + puts("test_rammap: skipped (Linux only)"); return 0; +#else + uint8_t owned_byte=0; + ESlot no_host={0}, released_owned={.backing=ESLOT_BACKING_OWNED}; + ESlot owned={.slab=&owned_byte,.backing=ESLOT_BACKING_OWNED}; + ESlot rammap={.backing=ESLOT_BACKING_RAMMAP}; + ESlot mmap_slot={.backing=ESLOT_BACKING_MMAP}; + if(expert_host_ready(&no_host) || expert_host_ready(&released_owned) || + !expert_host_ready(&owned) || !expert_host_ready(&rammap) || + !expert_host_ready(&mmap_slot)) + return fail("expert host-ready backing classification"); + QT grouped={.fmt=4,.O=2,.I=129,.gs=64}; + QT int3={.fmt=5,.O=2,.I=65}; + QT e8={.fmt=6,.O=2,.I=257}; + if(qt_scale_bytes(&grouped)!=24 || qt_scale_bytes(&int3)!=16 || + qt_bytes(&int3)-qt_scale_bytes(&int3)!=96 || + qt_scale_bytes(&e8)!=4 || qt_bytes(&e8)-qt_scale_bytes(&e8)!=392) + return fail("grouped/int3/E8 scale and weight byte geometry"); + ProfPhysicalWire unavailable=prof_physical_wire(-1); + ProfPhysicalWire measured_zero=prof_physical_wire(0); + ProfPhysicalWire measured_bytes=prof_physical_wire(4096); + if(unavailable.valid || unavailable.bytes!=0 || + !measured_zero.valid || measured_zero.bytes!=0 || + !measured_bytes.valid || measured_bytes.bytes!=4096) + return fail("physical-read wire validity distinguishes unavailable from zero"); + uint64_t read_bytes; + if(prof_physical_read_bytes(&read_bytes)){ + ProfBase sample={.physical_read_bytes=read_bytes,.physical_read_valid=1}; + if(prof_physical_read_delta(&sample)<0) + return fail("monotonic /proc/self/io read_bytes sampling"); + } + char tpath[]="/dev/shm/coli-rammap-XXXXXX"; + int tfd=mkstemp(tpath); + if(tfd<0){ printf("test_rammap: skipped (/dev/shm: %s)\n",strerror(errno)); return 0; } + struct statfs tfs; + if(fstatfs(tfd,&tfs) || (unsigned long)tfs.f_type!=(unsigned long)TMPFS_MAGIC){ + close(tfd); unlink(tpath); puts("test_rammap: skipped (/dev/shm is not tmpfs)"); return 0; + } + shards probe={0}; int probe_fd=st_open_fd(&probe,tpath); + if(!st_fd_is_tmpfs(&probe,probe_fd) || probe.dfds[0]!=-1) + return fail("st_open_fd tmpfs detection / O_DIRECT suppression"); + close(probe_fd); free(probe.paths[0]); + char rpath[256]; int rfd=make_regular_temp(rpath,sizeof(rpath)); + int untracked_mixed=0; + if(rfd<0){ + char fallback[]="/dev/shm/coli-rammap-mixed-XXXXXX"; + rfd=mkstemp(fallback); if(rfd<0){ close(tfd); unlink(tpath); return fail("mixed fixture"); } + snprintf(rpath,sizeof(rpath),"%s",fallback); + untracked_mixed=1; /* still proves all six descriptors must be verified */ + } + unsigned char data[152]={0}; + for(int expert=0;expert<2;expert++){ + int base=expert*76; + for(int i=0;i<36;i++) data[base+i]=(unsigned char)(1+(expert*17+i)%31); + float scales[10]; for(int i=0;i<10;i++) scales[i]=0.01f*(float)(i+1); + memcpy(data+base+36,scales,sizeof(scales)); + } + if(pwrite(tfd,data,sizeof(data),0)!=(ssize_t)sizeof(data) || + pwrite(rfd,data,sizeof(data),0)!=(ssize_t)sizeof(data)){ + close(tfd); close(rfd); unlink(tpath); unlink(rpath); return fail("fixture write"); + } + if(ftruncate(tfd,1<<20)) return fail("extended quantized fixture"); + if(check_direct_quant_format(tfd,(long)tfs.f_type,6,384,256,4096,137996)) return 1; + if(check_direct_quant_format(tfd,(long)tfs.f_type,8,384,256,262144,294984)) return 1; + + /* At I=98 the E8 weight/tag geometry collides with FP8 (and, for O=1, + * int8). Direct mapping must decline an unstamped ambiguous expert and + * leave the ordinary slab/SSD loader in charge, never guess a decoder. */ + Model collision; add_quant_expert(&collision,tfd,(long)tfs.f_type,6,98,64,600000); + ESlot collision_slot={0}; + if(rammap_bind_one(&collision,0,0,&collision_slot)!=0 || + collision_slot.backing==ESLOT_BACKING_RAMMAP) + return fail("ambiguous E8/FP8 direct binding fails closed"); + free_quant_expert(&collision); + + Model m={0}; m.c.n_layers=1; m.c.n_experts=2; m.c.hidden=4; m.c.moe_inter=3; + m.c.first_dense=0; m.ebits=8; m.L=calloc(1,sizeof(Layer)); m.L[0].sparse=1; + m.S.cap=12; m.S.t=calloc(12,sizeof(st_tensor)); + m.S.fds[0]=tfd; m.S.fs_magic[0]=(long)tfs.f_type; m.S.is_tmpfs[0]=1; m.S.nfd=1; + if(!untracked_mixed){ + struct statfs rfs; if(fstatfs(rfd,&rfs)){ return fail("regular fstatfs"); } + m.S.fds[1]=rfd; m.S.fs_magic[1]=(long)rfs.f_type; + m.S.is_tmpfs[1]=((unsigned long)rfs.f_type==(unsigned long)TMPFS_MAGIC); m.S.nfd=2; + } + add_expert(&m,0,tfd,tfd); add_expert(&m,1,tfd,rfd); + m.pin=calloc(2,sizeof(ESlot*)); m.npin=calloc(2,sizeof(int)); + m.ecache=calloc(2,sizeof(ESlot*)); m.ecn=calloc(2,sizeof(int)); + + g_mmap=0; g_rammap=1; g_ram_prefault=0; + rammap_build(&m); + if(m.rammap_experts!=1 || m.rammap_bytes!=76 || !rammap_slot(&m,0,0) || rammap_slot(&m,0,1)) + return fail("full-six-tensor tmpfs eligibility / mixed fallback"); + if(!rammap_modes_conflict(1,1) || rammap_modes_conflict(1,0) || rammap_modes_conflict(0,1)) + return fail("COLI_MMAP/COLI_RAMMAP conflict"); + + ESlot staged={0}; atomic_store_explicit(&g_prof_io,0,memory_order_relaxed); + if(expert_load_impl(&m,0,0,&staged,1,0) || + atomic_load_explicit(&g_prof_io,memory_order_relaxed)!=0) + return fail("tmpfs slab path has zero SSD-backed expert requests"); + ESlot *mapped=rammap_slot(&m,0,0); float x[4]={.25f,-.5f,.75f,1.f}; + QT *direct_qt[3]={&mapped->g,&mapped->u,&mapped->d}; + QT *slab_qt[3]={&staged.g,&staged.u,&staged.d}; + float in3[3]={.2f,-.4f,.6f}; + for(int k=0;k<3;k++){ + float direct_y[4]={0},slab_y[4]={0}; + const float *input=k<2?x:in3; int outputs=k<2?3:4; + matmul_qt(direct_y,input,direct_qt[k],1); matmul_qt(slab_y,input,slab_qt[k],1); + if(direct_qt[k]->fmt!=slab_qt[k]->fmt || direct_qt[k]->gs!=slab_qt[k]->gs || + memcmp(direct_y,slab_y,(size_t)outputs*sizeof(float))) + return fail("all direct and tmpfs-slab projections are equivalent"); + } + compat_aligned_free(staged.slab); free(staged.fslab); + + char state_root[]="/tmp/coli-rammap-state-XXXXXX"; + if(!mkdtemp(state_root)) return fail("state fixture"); + char state_leaf[320]; snprintf(state_leaf,sizeof(state_leaf),"%s/node/engine",state_root); + struct stat state_st; + if(state_dir_prepare(state_leaf) || stat(state_leaf,&state_st) || !S_ISDIR(state_st.st_mode)) + return fail("explicit state directory creation"); + char state_too_long[2048]; memset(state_too_long,'x',sizeof(state_too_long)-1); state_too_long[0]='/'; + state_too_long[sizeof(state_too_long)-1]=0; + if(state_dir_prepare(state_too_long)==0 || errno!=ENAMETOOLONG) + return fail("state directory suffix length guard"); + + char profile[]="/tmp/coli-rammap-profile-XXXXXX"; int pfd=mkstemp(profile); + if(pfd<0 || dprintf(pfd,"0 0 100\n0 1 50\n")<0){ return fail("pin profile"); } + close(pfd); pin_load(&m,profile,0.001,1); unlink(profile); + if(m.npin[0]!=1 || m.pin[0][0].eid!=1) return fail("PIN excludes direct experts"); + if(expert_resident_slot(&m,0,1,0)!=&m.pin[0][0]) return fail("hybrid SSD fallback residency"); + + /* The same expert bytes are available on both fixtures. Exercise three + * complete paths and compare a synthetic greedy token: all-SSD slabs, + * full tmpfs RAM-map, and five tmpfs tensors + one SSD fallback tensor. */ + st_tensor *expert1[6]; int expert1_fd[6],nexpert1=0; + for(int i=0;ifd=rfd; + atomic_store_explicit(&g_prof_io,0,memory_order_relaxed); + if(expert_load_impl(&m,0,1,&ssd,1,0)) return fail("SSD expert load"); + if(!untracked_mixed && atomic_load_explicit(&g_prof_io,memory_order_relaxed)!=76) + return fail("all-SSD requested expert byte telemetry"); + for(int i=0;i<6;i++) expert1[i]->fd=tfd; + if(rammap_bind_one(&m,0,1,&full)!=76) return fail("full direct expert binding"); + for(int i=0;i<6;i++) expert1[i]->fd=expert1_fd[i]; + atomic_store_explicit(&g_prof_io,0,memory_order_relaxed); + if(expert_load_impl(&m,0,1,&mixed,1,0)) return fail("hybrid expert load"); + if(!untracked_mixed && atomic_load_explicit(&g_prof_io,memory_order_relaxed)!=16) + return fail("descriptor-classified requested expert byte telemetry"); + float y_ssd[4],y_full[4],y_mixed[4]; + int tok_ssd=expert_greedy_token(&ssd,x,y_ssd); + int tok_full=expert_greedy_token(&full,x,y_full); + int tok_mixed=expert_greedy_token(&mixed,x,y_mixed); + if(tok_ssd!=tok_full || tok_ssd!=tok_mixed || + memcmp(y_ssd,y_full,sizeof(y_ssd)) || memcmp(y_ssd,y_mixed,sizeof(y_ssd))) + return fail("SSD/full-RAM/hybrid greedy token identity"); + compat_aligned_free(ssd.slab); free(ssd.fslab); + compat_aligned_free(mixed.slab); free(mixed.fslab); + + m.pin[0]=realloc(m.pin[0],2*sizeof(ESlot)); memset(&m.pin[0][1],0,sizeof(ESlot)); + m.pin[0][1].eid=0; m.pin[0][1].backing=ESLOT_BACKING_OWNED; m.npin[0]=2; + m.ecache[0]=calloc(1,sizeof(ESlot)); m.ecache[0][0].eid=0; m.ecache[0][0].used=123; m.ecn[0]=1; + atomic_store_explicit(&g_prof_io,777,memory_order_relaxed); m.rammap_calls=0; + ESlot *direct=rammap_slot(&m,0,0); + if(!expert_slot_is_pinned(&m,0,&m.pin[0][0]) || + expert_slot_is_pinned(&m,0,direct) || + expert_slot_is_pinned(&m,0,&m.ecache[0][0])) + return fail("pin tier classification uses slot identity"); + if(expert_resident_slot(&m,0,0,1)!=direct || m.rammap_calls!=1 || + m.ecache[0][0].used!=123 || atomic_load_explicit(&g_prof_io,memory_order_relaxed)!=777) + return fail("direct-map precedence / LRU and I/O exclusion / telemetry"); + if(!expert_is_resident(&m,0,0) || m.rammap_calls!=1) return fail("residency probe telemetry isolation"); + + compat_aligned_free(m.pin[0][0].slab); free(m.pin[0][0].fslab); free(m.pin[0]); + free(m.ecache[0]); free(m.pin); free(m.npin); free(m.ecache); free(m.ecn); + free(m.rammap); free(m.L); + for(int i=0;i /proc/self/io read_bytes + stays ~0, reported as `[PROF] physical SSD reads: 0.000 GB`. + +Gating: needs distinct canonical and staged GLM-compatible int4 namespaces plus +the built colibri binary. The canonical namespace must be block-backed and have +its safetensor shards deliberately hidden after copying; the staged namespace +must be tmpfs-backed and complete. This proves the engine used +COLI_WEIGHTS_DIR rather than silently falling back to SNAP. The live test runs +only when both variables below are set; otherwise it skips honestly. + + COLI_RAMMAP_E2E_CANONICAL=/path/on/disk/glm_i4 \ + COLI_RAMMAP_E2E_STAGED=/dev/shm/glm_i4 \ + python3 -m pytest tests/test_rammap_e2e.py -v + +The parse logic itself is covered by ProfParseTest below, which runs everywhere +and pins the exact colibri.c PROF emission strings. +""" + +import os +import re +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +ENGINE = ROOT / "colibri" + +# `[PROF] RAM map: experts / GB direct | ...` (colibri.c prof_report, +# printed only when COLI_RAMMAP bound at least one expert -> tmpfs-backed). +RAM_MAP_RE = re.compile(r"\[PROF\] RAM map: (\d+) experts / ([0-9.]+) GB direct") +# `[PROF] physical SSD reads: GB (...)` -- the live /proc/self/io read_bytes +# delta. The "unavailable" branch (non-Linux, or no /proc/self/io) has no number. +PHYSICAL_RE = re.compile(r"\[PROF\] physical SSD reads: ([0-9.]+) GB") +UNAVAILABLE = "physical SSD reads: unavailable" + +# A fully-tmpfs model does zero block-device reads during the decode window, so +# read_bytes stays ~0. 50 MB is far above any kernel-accounting noise and far +# below the GBs real SSD streaming would report -- it discriminates the only +# regressions that matter here (model not on tmpfs, or RAMMAP silently bypassed +# in favor of a block-backed path). tmpfs slab fallback still reads ~0, which is +# correct: tmpfs is not a block device. +MAX_PHYSICAL_GB = 0.05 +# REPLAY drives the forward from these token ids -- no tokenizer.json needed, so +# the test runs on the CI-generated bench fixture (random weights, no tokenizer) +# as well as on a real model. REF_FORCE bypasses the vocab-size sanity check. +REF = ROOT / "ref_glm.json" + + +def parse_prof(output): + """Return (rammap_experts, rammap_gb, physical_gb_or_None) from glm PROF output. + + physical_gb is None when the engine reports accounting as unavailable. + """ + rm = RAM_MAP_RE.search(output) + pm = PHYSICAL_RE.search(output) + experts = int(rm.group(1)) if rm else 0 + rammap_gb = float(rm.group(2)) if rm else 0.0 + physical = float(pm.group(1)) if pm else None + return experts, rammap_gb, physical + + +@unittest.skipUnless( + os.environ.get("COLI_RAMMAP_E2E_CANONICAL") + and os.environ.get("COLI_RAMMAP_E2E_STAGED"), + "set distinct COLI_RAMMAP_E2E_CANONICAL and COLI_RAMMAP_E2E_STAGED dirs", +) +class RammapE2ETest(unittest.TestCase): + """Live Colibri on tmpfs: asserts the zero-physical-SSD-read contract holds.""" + + def setUp(self): + self.canonical = Path( + os.environ["COLI_RAMMAP_E2E_CANONICAL"] + ).resolve() + self.staged = Path(os.environ["COLI_RAMMAP_E2E_STAGED"]).resolve() + self.assertTrue(ENGINE.exists(), "colibri binary not built -- run `make colibri`") + self.assertTrue( + self.canonical.is_dir(), + "canonical model dir missing: %s" % self.canonical, + ) + self.assertTrue( + self.staged.is_dir(), + "staged model dir missing: %s" % self.staged, + ) + self.assertTrue(REF.exists(), "missing %s -- run from a repo checkout" % REF) + self.assertNotEqual(self.canonical, self.staged) + self.assertNotEqual( + self.canonical.stat().st_dev, + self.staged.stat().st_dev, + "canonical and staged fixtures share a backing device", + ) + canonical_fs = self._filesystem_type(self.canonical) + staged_fs = self._filesystem_type(self.staged) + self.assertIn( + canonical_fs, + ("ext4", "xfs"), + "canonical fixture is not block-backed: %s" % canonical_fs, + ) + self.assertEqual(staged_fs, "tmpfs") + self.assertFalse( + any(self.canonical.glob("*.safetensors")), + "canonical shards must be hidden after staging", + ) + self.assertTrue( + any(self.staged.glob("*.safetensors")), + "staged tmpfs namespace has no safetensor shards", + ) + + def _filesystem_type(self, path): + result = subprocess.run( + ["findmnt", "-T", str(path), "-n", "-o", "FSTYPE"], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual( + result.returncode, + 0, + "cannot identify filesystem for %s: %s" % (path, result.stderr), + ) + return result.stdout.strip() + + def _engine_environment(self, *, staged): + env = dict(os.environ) + env.update( + SNAP=str(self.canonical), + COLI_RAMMAP="1", + PROF="1", + REPLAY="1", + REF_FORCE="1", + REF=str(REF), + COLI_NO_OMP_TUNE="1", + ) + if staged: + env["COLI_WEIGHTS_DIR"] = str(self.staged) + else: + env.pop("COLI_WEIGHTS_DIR", None) + env.pop("COLI_MODEL_DIRS", None) + env.pop("COLI_MODEL_MIRROR", None) + env.pop("COLI_STATE_DIR", None) + return env + + def test_canonical_namespace_cannot_launch_without_staged_redirect(self): + proc = subprocess.run( + [str(ENGINE)], + env=self._engine_environment(staged=False), + capture_output=True, + text=True, + timeout=30, + ) + self.assertNotEqual( + proc.returncode, + 0, + "engine unexpectedly found poisoned canonical shards; the positive " + "test would not prove COLI_WEIGHTS_DIR redirection", + ) + + def test_zero_physical_ssd_reads_on_tmpfs_rammap(self): + # REPLAY mode drives a real forward (MoE expert loading -> RAMMAP -> I/O) + # from the repo's ref_glm.json token ids. It needs no tokenizer.json, so it + # works on the CI-generated bench fixture as well as on a real model. The + # physical-read accounting is identical to the PROMPT/serve path: the same + # prof_report emits the [PROF] lines this test parses. + proc = subprocess.run( + [str(ENGINE)], + env=self._engine_environment(staged=True), + capture_output=True, + text=True, + timeout=180, + ) + output = proc.stdout + proc.stderr + self.assertEqual( + proc.returncode, 0, "colibri exited %d:\n%s" % (proc.returncode, output[-2000:]) + ) + self.assertNotIn( + UNAVAILABLE, + output, + "physical SSD accounting unavailable -- not a Linux tmpfs backing?", + ) + experts, _, physical = parse_prof(output) + self.assertGreater( + experts, + 0, + "COLI_RAMMAP bound no experts -- model is not tmpfs-backed, so this " + "run exercised the SSD path and proves nothing", + ) + # The None branch is the "unavailable" case (already ruled out above); the + # `if ... is None: fail()` narrows physical to float for the assertLess. + if physical is None: + self.fail("missing [PROF] physical SSD reads line") + self.assertLess( + physical, + MAX_PHYSICAL_GB, + "tmpfs + RAMMAP model still read %.3f GB from SSD during decode -- " + "zero-SSD-read contract broken" % physical, + ) + + +class ProfParseTest(unittest.TestCase): + """Pins parse_prof against the exact colibri.c PROF strings -- runs everywhere.""" + + def test_parses_bound_tmpfs_output(self): + sample = ( + "[PROF] RAM map: 4096 experts / 12.345 GB direct | 99 calls this window | zero slab reads\n" + "[PROF] physical SSD reads: 0.000 GB (0.0 MB/token; Linux /proc/self/io read_bytes, process-wide)\n" + ) + experts, rammap_gb, physical = parse_prof(sample) + self.assertEqual(experts, 4096) + self.assertAlmostEqual(rammap_gb, 12.345) + self.assertEqual(physical, 0.000) + + def test_parses_nonzero_ssd_output(self): + sample = "[PROF] physical SSD reads: 5.500 GB (0.2 MB/token; ...)\n" + _, _, physical = parse_prof(sample) + self.assertEqual(physical, 5.500) + + def test_parses_unavailable_as_none(self): + sample = "[PROF] physical SSD reads: unavailable on this platform/kernel\n" + _, _, physical = parse_prof(sample) + self.assertIsNone(physical) + + def test_detects_ssd_regression_threshold(self): + # A real SSD-streaming regression reports GBs, well above the threshold. + sample = "[PROF] physical SSD reads: 5.500 GB (...)\n" + _, _, physical = parse_prof(sample) + if physical is None: + self.fail("physical SSD reads line did not parse") + self.assertGreaterEqual(physical, MAX_PHYSICAL_GB) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_resource_masks.c b/c/tests/test_resource_masks.c new file mode 100644 index 000000000..9cc5ad2ae --- /dev/null +++ b/c/tests/test_resource_masks.c @@ -0,0 +1,172 @@ +/* Managed server placement needs exact Linux range-list masks. Keep this test + * independent of sysfs and machine topology: it exercises the production + * parser/mask representation directly, including sparse IDs beyond one + * unsigned-long word. */ +#define _GNU_SOURCE +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#define main coli_glm_main_unused +#include "../colibri.c" +#undef main + +static int failures; + +#define CHECK(cond, fmt, ...) do { \ + if(!(cond)){ \ + fprintf(stderr, "FAIL: " fmt "\n", ##__VA_ARGS__); \ + failures++; \ + } \ +} while(0) + +#ifdef __linux__ +static void expect_invalid(const char *spec, unsigned long max_id){ + ColiIdMask mask={0}; + errno=0; + CHECK(coli_idmask_parse(spec,max_id,&mask)<0, + "accepted invalid range list %s",spec?spec:"(null)"); + CHECK(mask.words==NULL && mask.nwords==0 && mask.maxnode==0 && mask.count==0, + "invalid parse left a partial mask for %s",spec?spec:"(null)"); + coli_idmask_free(&mask); +} +#endif + +int main(void){ +#ifndef __linux__ + puts("test_resource_masks: skipped (Linux only)"); + return 0; +#else + const unsigned long word_bits=(unsigned long)(sizeof(unsigned long)*CHAR_BIT); + ColiIdMask sparse={0}; + CHECK(coli_idmask_parse("0,2,65,130-132",4096,&sparse)==0, + "sparse range list did not parse"); + CHECK(sparse.count==6,"selected count=%zu, expected 6",sparse.count); + CHECK(sparse.maxnode==133,"maxnode=%lu, expected 133",sparse.maxnode); + CHECK(sparse.nwords==(133+word_bits-1)/word_bits, + "nwords=%zu does not cover maxnode 133",sparse.nwords); + CHECK(coli_numa_policy_mode(1,1)== + (COLI_MPOL_BIND|COLI_MPOL_F_STATIC_NODES), + "explicit one-node policy is not a static bind"); + CHECK(coli_numa_policy_mode(2,1)== + (COLI_MPOL_INTERLEAVE|COLI_MPOL_F_STATIC_NODES), + "explicit multi-node policy is not static interleave"); + CHECK(coli_numa_policy_mode(1,0)==COLI_MPOL_INTERLEAVE, + "legacy implicit policy unexpectedly changed"); + { + const unsigned long present[]={0,2,65,130,131,132}; + const unsigned long absent[]={1,64,66,129,133}; + for(size_t i=0;i0,"Mems_allowed_list parsed as empty"); + + expect_invalid(NULL,4096); + expect_invalid("",4096); + expect_invalid(" ",4096); + expect_invalid(",0",4096); + expect_invalid("0,",4096); + expect_invalid("0,,1",4096); + expect_invalid("-1",4096); + expect_invalid("+1",4096); + expect_invalid("1-",4096); + expect_invalid("2-1",4096); + expect_invalid("1--2",4096); + expect_invalid("1 2",4096); + expect_invalid("1, 2",4096); + expect_invalid("1,1",4096); + expect_invalid("1-3,3-5",4096); + expect_invalid("1",0); + expect_invalid("4097",4096); + expect_invalid("999999999999999999999999999999",ULONG_MAX-1); + + /* Exercise the production apply+readback path in a child so this test does + * not alter the test runner's own affinity. sched_getcpu() necessarily + * returns a CPU allowed to this process, making the case topology-neutral. */ + pid_t child=fork(); + CHECK(child>=0,"fork failed: %s",strerror(errno)); + if(child==0){ + int cpu=sched_getcpu(); + char spec[32]; + if(cpu<0) _exit(10); + snprintf(spec,sizeof(spec),"%d",cpu); + _exit(coli_cpu_affinity_apply(spec)?11:0); + } else if(child>0){ + int status=0; + CHECK(waitpid(child,&status,0)==child,"waitpid failed: %s",strerror(errno)); + CHECK(WIFEXITED(status) && WEXITSTATUS(status)==0, + "managed CPU apply/readback child status=%d",status); + } + + /* The engine entry point must honor the managed mask even when every OMP + * tuning/re-exec gate is disabled. This was previously nested inside that + * optional branch and silently bypassed by server-style overrides. */ + cpu_set_t inherited; + CPU_ZERO(&inherited); + if(!sched_getaffinity(0,sizeof(inherited),&inherited) && + CPU_COUNT(&inherited)>=2){ + int target=-1; + for(int cpu=0;cpu=0,"affinity-gate fork failed: %s",strerror(errno)); + if(gated==0){ + char spec[32]; + char *argv[]={(char*)"colibri",NULL}; + snprintf(spec,sizeof(spec),"%d",target); + setenv("COLI_CPU_AFFINITY",spec,1); + setenv("COLI_NO_OMP_TUNE","1",1); + setenv("COLI_CUDA","0",1); + unsetenv("SNAP"); + int rc=coli_glm_main_unused(1,argv); + cpu_set_t applied; + CPU_ZERO(&applied); + if(rc!=1 || sched_getaffinity(0,sizeof(applied),&applied) || + CPU_COUNT(&applied)!=1 || !CPU_ISSET(target,&applied)) + _exit(12); + _exit(0); + } else if(gated>0){ + int status=0; + CHECK(waitpid(gated,&status,0)==gated, + "affinity-gate waitpid failed: %s",strerror(errno)); + CHECK(WIFEXITED(status) && WEXITSTATUS(status)==0, + "OMP-gated managed affinity child status=%d",status); + } + } + + coli_idmask_free(&outside); + coli_idmask_free(&allowed); + coli_idmask_free(&allowed_mems); + coli_idmask_free(&sparse); + if(failures) return 1; + puts("OK exact CPU/NUMA range-list masks"); + return 0; +#endif +} diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py index c5a3fac9c..e462b4028 100644 --- a/c/tests/test_resource_plan.py +++ b/c/tests/test_resource_plan.py @@ -10,6 +10,8 @@ from resource_plan import ( GB, + _darwin_memory_available, + _parse_vm_stat, analyze_model, build_plan, cpu_socket_count, @@ -67,10 +69,12 @@ def test_analyzes_dense_and_expert_storage(self): self.assertEqual(info["expert_count"], 2) self.assertEqual(info["per_cap_bytes"], 60) - def test_memory_available_is_positive(self): + def test_native_memory_available_is_positive(self): # Regression: on native Windows CPython, /proc/meminfo does not exist, # so the Linux-only path returned 0 and the expert cache was sized to # 0 slots/layer. The value must be a sane positive number of bytes. + if sys.platform == "darwin": + self.skipTest("macOS has a separate native probe smoke") self.assertGreater(memory_available(), 0) def test_cpu_socket_count_is_positive(self): @@ -510,5 +514,101 @@ def test_zero_logical_cores_warns_and_returns_one(self): self.assertEqual(physical_cpu_count(), 1) +class DarwinMemoryProbeTest(unittest.TestCase): + """Hermetic coverage for the macOS command protocol and parser.""" + + VM_STAT = """Mach Virtual Memory Statistics: (page size of 16384 bytes) +Pages free: 100. +Pages active: 999. +Pages inactive: 200. +Pages speculative: 30. +Pages throttled: 7. +Pages purgeable: 40. +""" + + @staticmethod + def _runner(results, calls): + def run(command, **kwargs): + calls.append((command, kwargs)) + result = results.pop(0) + if isinstance(result, BaseException): + raise result + return subprocess.CompletedProcess(command, *result) + + return run + + def test_parse_vm_stat_sums_only_reclaimable_pages(self): + self.assertEqual( + _parse_vm_stat(self.VM_STAT, fallback_page_size=4096), + (100 + 200 + 30 + 40) * 16384, + ) + + def test_parse_vm_stat_uses_injected_fallback_page_size(self): + output = "Pages free: 2.\nPages inactive: 3.\n" + self.assertEqual( + _parse_vm_stat(output, fallback_page_size=4096), + 5 * 4096, + ) + + def test_darwin_probe_prefers_vm_stat_without_calling_sysctl(self): + calls = [] + run = self._runner([(0, self.VM_STAT, "")], calls) + self.assertEqual( + _darwin_memory_available(run=run, fallback_page_size=4096), + (100 + 200 + 30 + 40) * 16384, + ) + self.assertEqual([call[0] for call in calls], [["/usr/bin/vm_stat"]]) + + def test_darwin_probe_uses_checked_absolute_sysctl_fallback(self): + calls = [] + run = self._runner( + [ + (1, "", "vm_stat unavailable"), + (0, "68719476736\n", ""), + ], + calls, + ) + self.assertEqual( + _darwin_memory_available(run=run, fallback_page_size=4096), + 68_719_476_736, + ) + self.assertEqual( + [call[0] for call in calls], + [ + ["/usr/bin/vm_stat"], + ["/usr/sbin/sysctl", "-n", "hw.memsize"], + ], + ) + + def test_darwin_probe_returns_zero_when_commands_or_output_fail(self): + cases = [ + [FileNotFoundError(), FileNotFoundError()], + [(0, "not vm_stat", ""), (0, "not-an-integer\n", "")], + [(0, "Pages free: 0.\n", ""), (1, "68719476736\n", "denied")], + [(1, self.VM_STAT, "denied"), (0, "-1\n", "")], + ] + for results in cases: + with self.subTest(results=results): + self.assertEqual( + _darwin_memory_available( + run=self._runner(list(results), []), + fallback_page_size=4096, + ), + 0, + ) + + def test_native_probe_smoke(self): + if sys.platform != "darwin": + self.skipTest("native macOS only") + # A Nix Darwin build sandbox is not a native-host probe. The hermetic + # parser/command cases above stay active there; this smoke remains in + # the native macOS job and on developer Macs. + if os.environ.get("NIX_BUILD_TOP"): + self.skipTest( + "native macOS memory probing is unavailable in a Nix build sandbox" + ) + self.assertGreater(memory_available(), 0) + + if __name__ == "__main__": unittest.main() diff --git a/c/tests/test_serve_sentinel.c b/c/tests/test_serve_sentinel.c index 50328f626..cb1d177e1 100644 --- a/c/tests/test_serve_sentinel.c +++ b/c/tests/test_serve_sentinel.c @@ -47,13 +47,12 @@ int main(void) FILE *f = fopen(path, "w"); /* deliberately TEXT mode: the bug's condition */ if (!f) { perror("fopen"); remove(path); return 2; } - /* Same call the serve loops make before emitting the handshake. On Windows it - * flips the stream to binary; elsewhere it is a no-op. We point it at stdout, - * so exercise the same primitive here on our own stream. */ - coli_serve_binary_mode(); -#ifdef _WIN32 - _setmode(_fileno(f), _O_BINARY); -#endif + /* Exercise the real serve binary-mode primitive on the SAME stream we + * assert against, so a Windows no-op regression actually fails this test + * (#748). coli_serve_binary_mode() applies this to stdin/stdout in + * production;coli_serve_binary_mode_stream() exposes that exact logic to a + * chosen stream. */ + coli_serve_binary_mode_stream(f); fputs(SENTINEL, f); fprintf(f, "STAT 0 0.0 0.0 %.2f 0 0\n", 1.0); diff --git a/c/tests/test_uring.c b/c/tests/test_uring.c index 625c79bbb..cd218bcb6 100644 --- a/c/tests/test_uring.c +++ b/c/tests/test_uring.c @@ -11,6 +11,40 @@ static int fail(const char *s){ fprintf(stderr,"FAIL: %s\n",s); return 1; } +static int uring_resource_unavailable(int err){ + return err==ENOMEM || err==EPERM || err==ENOSYS || err==EACCES; +} + +static void free_tensor_metadata(Model *m){ + if(!m || !m->S.t) return; + for(int i=0;iS.n;i++) free(m->S.t[i].name); + free(m->S.t); m->S.t=NULL; m->S.n=m->S.cap=0; +} + +/* Metadata finalization is independent of the kernel ring. Exercise it even + * when io_uring_setup is blocked by the test sandbox: grouped-int4 scales must + * produce fmt=4/gs=16, exactly like the buffered expert loader. */ +static int test_grouped_finalize_metadata(void){ + Model m={0}; ESlot slot={0}; UringBatch batch={0}; + m.c.hidden=64; m.c.moe_inter=64; + st_tensor weights[3]={{0}}, scales[3]={{0}}; + int64_t wbytes=64*32, sbytes=64*4*4; + if(posix_memalign((void**)&slot.slab,4096,(size_t)wbytes*3+4096)) slot.slab=NULL; + slot.fslab=calloc((size_t)sbytes*3/4,sizeof(float)); + if(!slot.slab||!slot.fslab) return fail("grouped finalize allocation"); + UringLoad *load=&batch.load[0]; load->m=&m; load->s=&slot; + load->eid=9; load->done=1; + for(int k=0;k<3;k++){ + weights[k].nbytes=wbytes; scales[k].nbytes=sbytes; + load->tw[k]=&weights[k]; load->tq[k]=&scales[k]; load->pos[k]=(int64_t)k*wbytes; + } + int rc=uring_finalize_load(&batch,0,1); + int bad=rc || slot.eid!=9 || slot.g.fmt!=4 || slot.u.fmt!=4 || slot.d.fmt!=4 + || slot.g.gs!=16 || slot.u.gs!=16 || slot.d.gs!=16; + compat_aligned_free(slot.slab); free(slot.fslab); + return bad?fail("grouped-int4 io_uring finalization"):0; +} + static int test_expert_layout(int fd){ Model m={0}; ESlot slot={0}; UringBatch batch={0}; m.c.hidden=4; m.c.moe_inter=3; m.ebits=8; @@ -31,7 +65,15 @@ static int test_expert_layout(int fd){ size_t n=strlen(name); memcpy(name+n,".qs",4); m.S.t[3+k]=(st_tensor){strdup(name),fd,so,sbytes[k],2,sbytes[k]/4}; so+=sbytes[k]; } - if(uring_batch_init(&batch)){ free(m.S.t); return fail("expert ring init"); } + if(uring_batch_init(&batch)){ + int err=errno; + free_tensor_metadata(&m); + if(uring_resource_unavailable(err)){ + printf("test_uring: expert batch skipped (%s)\n",strerror(err)); + return 0; + } + return fail("expert ring init"); + } uring_batch_reset(&batch); int li=uring_load_add(&batch,&m,1,7,&slot,1); if(li!=0 || uring_submit_batch(&batch) || uring_finalize_load(&batch,li,1)){ @@ -50,7 +92,19 @@ static int test_expert_layout(int fd){ m.ecache[1]=calloc(2,sizeof(ESlot)); if(!m.pin||!m.npin||!m.ecache||!m.ecn||!m.ecache[1]) return fail("pilot fixture allocation"); - if(uring_batch_init(&g_ub_pilot)) return fail("pilot ring init"); + if(uring_batch_init(&g_ub_pilot)){ + int err=errno; + /* slot.slab/slot.fslab already freed at the end of the expert-batch + * section above; only the pilot fixture allocations need cleanup here. */ + free(m.ecache[1]); + free(m.pin); free(m.npin); free(m.ecache); free(m.ecn); + free_tensor_metadata(&m); + if(uring_resource_unavailable(err)){ + printf("test_uring: pilot batch skipped (%s)\n",strerror(err)); + return 0; + } + return fail("pilot ring init"); + } memset(g_pilot_inflight,0,sizeof(g_pilot_inflight)); atomic_store(&g_cur_moe_layer,-1); atomic_store(&g_pilot_loads,0); atomic_store(&g_pilot_drops,0); pilot_r=0; pilot_w=1; pilot_q[0].l=1; pilot_q[0].e=7; @@ -61,12 +115,12 @@ static int test_expert_layout(int fd){ compat_aligned_free(m.ecache[1][0].slab); free(m.ecache[1][0].fslab); free(m.ecache[1]); free(m.pin); free(m.npin); free(m.ecache); free(m.ecn); - for(int i=0;i HWINFO | +GPUS ( )×n +GPUDETAIL 1 ( )×n TIERS EMAP ``` -The server must not send requests before `READY`. `HWINFO`/`TIERS`/`EMAP` are -telemetry (see below) and may grow — **servers must ignore line kinds they do not -recognize**; that is the protocol's forward-compatibility rule. +The server must not send requests before `READY`. The remaining records are +telemetry (see below) and may grow — **servers must ignore line kinds they do +not recognize**; that is the protocol's forward-compatibility rule. ## Requests (server → engine) @@ -72,9 +74,11 @@ Errors replace the stream: `ERROR ` with codes `BAD_FRAME`, `BAD_REQU `CANCELLED`. A `CANCEL` is acknowledged by `ERROR CANCELLED` after the slot's KV is persisted. -Immediately before each `DONE` the engine emits a telemetry block for the finished -turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats below). -`.coli_usage` is persisted at every turn end, not only at exit. +Immediately before each `DONE` the engine emits a telemetry block for the +finished turn: `HWINFO`, legacy `GPUS`, `GPUDETAIL`, `TIERS`, `EMAP`, `HITS`, +and `PROF` (formats below). `GPUS` and `GPUDETAIL` are therefore emitted both +during startup and before every `DONE`. `.coli_usage` is persisted at every +turn end, not only at exit. ## Telemetry lines @@ -84,13 +88,28 @@ turn: `HWINFO`, `PERF`, `ENTROPY`, `GPUS`, `TIERS`, `EMAP`, `HITS` (formats belo | `HWINFO` | `HWINFO \|` | host snapshot (GBs are floats) | | `EMAP` | `EMAP ` | one byte per expert, row-major over `rows×cols` (sparse layers +MTP × experts): `byte = (tier<<6) \| heat` — 2-bit tier (0 disk / 1 RAM / 2 VRAM), 6-bit log₂-bucketed usage heat | | `HITS` | `HITS ` | 1 bit per expert, experts routed since the previous `HITS` | +| `PROF` | `PROF [ ]` | Per-turn phase metrics. On Linux, `physical_ssd_bytes` is the request-window delta of `/proc/self/io` `read_bytes` (process-wide). `physical_ssd_valid=0` distinguishes unavailable accounting from a measured zero; JSON clients expose unavailable bytes as `null`. The bracketed additive fields power persistent RAM-disk scorecards; older ten- and seventeen-field producers remain accepted, but a legacy zero is treated as unverified. | | `PERF` | `PERF
` | this turn's PROFILO deltas, seconds | | `ENTROPY` | `ENTROPY

…` | per-sparse-layer routing entropy of the turn, bits | | `GPUS` | `GPUS ( )×n` | per-device VRAM + resident expert count (CUDA builds) | +| `GPUDETAIL` | `GPUDETAIL 1 ( )×n` | versioned, integer-byte per-device placement. `ordinal` is the logical index in the engine's visible CUDA device list; `identity` is `-` when the backend cannot provide a UUID/PCI identity; `model_bytes = expert_bytes + nonexpert_bytes`. | | `TOPK` | `TOPK 5 ( )×5` | token text hex-encoded so the line stays line-shaped | | `REPIN` | `REPIN ` | one line per hot-store swap (`REPIN=n` mode) | +`GPUS` remains on the wire for older consumers. New control-plane consumers +should prefer `GPUDETAIL` for model-resident bytes and treat card-wide live +utilization/process memory as a separate sampler concern. + +The emitter publishes legacy `GPUS 0` when any selected device lacks a valid +memory sample, because that format has only implicit positions. `GPUDETAIL` +can retain valid devices because each row carries an explicit ordinal; omitted +selected ordinals make the control-plane snapshot incomplete and stale. +Consumers must reject duplicate ordinals, negative counts, `free_bytes` or +`model_bytes` above `total_bytes`, and a model/component sum mismatch. + All telemetry is advisory: servers render what they know and skip the rest. +An unknown telemetry kind or unsupported `GPUDETAIL` version is ignored. A +malformed record of a known, supported kind is a protocol error. ## HTTP surface (`openai_server.py`) @@ -98,6 +117,12 @@ All telemetry is advisory: servers render what they know and skip the rest. SSE frame `data: {"colibri": {stats, perf, topk, entropy, gpus, repin}}` immediately before `data: [DONE]`; non-streaming responses attach the same object as a `"colibri"` field. +- `GET /health` — public liveness; returns 503/`status: error` if the dispatcher + or engine child has failed. Authenticated requests (or servers with no + configured API key) also receive scheduler, tier, hardware, `gpus`, and + `gpus_seq` telemetry. +- `GET /profile` — the rolling completed-turn `PROF` records enriched with + `DONE` throughput, cache, RSS, and length-limit fields. - `GET /experts` — the latest `EMAP`/`HITS` state: `{rows, cols, map, hits, seq, gpus, entropy, repin}`. - `GET /*` — static hosting of `web/dist` (SPA fallback, path-traversal-safe), plus From 0d8722ad46f45ffdbe9eb41ad89eea1a6c3e2991 Mon Sep 17 00:00:00 2001 From: Ben Colsey Date: Mon, 3 Aug 2026 22:08:36 -0400 Subject: [PATCH 2/2] reconstruct PR2: headless planning/staging/mounts/recovery + tokenized CLI from donor 5f6f31a Co-Authored-By: Claude --- .github/workflows/check.yml | 118 +- .github/workflows/ci.yml | 63 + .github/workflows/release.yml | 24 +- .gitignore | 23 + README.md | 66 +- c/__init__.py | 5 + c/coli | 319 +- c/ramdisk.py | 1446 +++++ c/ramdisk_support/__init__.py | 1 + c/ramdisk_support/accelerator.py | 572 ++ c/ramdisk_support/cli.py | 517 ++ c/ramdisk_support/common.py | 175 + c/ramdisk_support/discovery.py | 705 +++ c/ramdisk_support/lifecycle.py | 3691 +++++++++++++ c/ramdisk_support/linux_ops.py | 2418 +++++++++ c/ramdisk_support/model.py | 306 ++ c/ramdisk_support/mounts.py | 1014 ++++ c/ramdisk_support/planning.py | 1228 +++++ c/ramdisk_support/platform_ops.py | 85 + c/ramdisk_support/presentation.py | 1434 +++++ c/ramdisk_support/presets.py | 385 ++ c/ramdisk_support/processes.py | 889 ++++ c/ramdisk_support/state.py | 2477 +++++++++ c/tests/platform_test_support.py | 350 ++ c/tests/ramdisk_test_support.py | 251 + c/tests/test_ramdisk.py | 11 + c/tests/test_ramdisk_accelerator.py | 411 ++ c/tests/test_ramdisk_cli.py | 204 + c/tests/test_ramdisk_cli_module.py | 423 ++ c/tests/test_ramdisk_cli_smoke.py | 188 + c/tests/test_ramdisk_facade.py | 227 + c/tests/test_ramdisk_integration.py | 114 + c/tests/test_ramdisk_model_planning.py | 704 +++ c/tests/test_ramdisk_mounts.py | 1278 +++++ c/tests/test_ramdisk_packaging.py | 715 +++ c/tests/test_ramdisk_planning_module.py | 324 ++ c/tests/test_ramdisk_platform.py | 3822 ++++++++++++++ c/tests/test_ramdisk_presentation_module.py | 88 + c/tests/test_ramdisk_presets.py | 548 ++ c/tests/test_ramdisk_processes.py | 2321 ++++++++ c/tests/test_ramdisk_state_lifecycle.py | 4655 +++++++++++++++++ c/tools/clean.py | 46 +- docs/ENVIRONMENT.md | 41 +- docs/SETTINGS.md | 80 +- docs/api.md | 67 + docs/quickstart.md | 2 +- .../2026-08-01-pr-377-stabilization-design.md | 140 + flake.nix | 38 +- pyproject.toml | 20 +- 49 files changed, 34942 insertions(+), 87 deletions(-) create mode 100644 c/__init__.py create mode 100644 c/ramdisk.py create mode 100644 c/ramdisk_support/__init__.py create mode 100644 c/ramdisk_support/accelerator.py create mode 100644 c/ramdisk_support/cli.py create mode 100644 c/ramdisk_support/common.py create mode 100644 c/ramdisk_support/discovery.py create mode 100644 c/ramdisk_support/lifecycle.py create mode 100644 c/ramdisk_support/linux_ops.py create mode 100644 c/ramdisk_support/model.py create mode 100644 c/ramdisk_support/mounts.py create mode 100644 c/ramdisk_support/planning.py create mode 100644 c/ramdisk_support/platform_ops.py create mode 100644 c/ramdisk_support/presentation.py create mode 100644 c/ramdisk_support/presets.py create mode 100644 c/ramdisk_support/processes.py create mode 100644 c/ramdisk_support/state.py create mode 100644 c/tests/platform_test_support.py create mode 100644 c/tests/ramdisk_test_support.py create mode 100644 c/tests/test_ramdisk.py create mode 100644 c/tests/test_ramdisk_accelerator.py create mode 100644 c/tests/test_ramdisk_cli.py create mode 100644 c/tests/test_ramdisk_cli_module.py create mode 100644 c/tests/test_ramdisk_cli_smoke.py create mode 100644 c/tests/test_ramdisk_facade.py create mode 100644 c/tests/test_ramdisk_integration.py create mode 100644 c/tests/test_ramdisk_model_planning.py create mode 100644 c/tests/test_ramdisk_mounts.py create mode 100644 c/tests/test_ramdisk_packaging.py create mode 100644 c/tests/test_ramdisk_planning_module.py create mode 100644 c/tests/test_ramdisk_platform.py create mode 100644 c/tests/test_ramdisk_presentation_module.py create mode 100644 c/tests/test_ramdisk_presets.py create mode 100644 c/tests/test_ramdisk_processes.py create mode 100644 c/tests/test_ramdisk_state_lifecycle.py create mode 100644 docs/superpowers/specs/2026-08-01-pr-377-stabilization-design.md diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 59968985c..7714c7227 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,6 +1,7 @@ # CI: run the repo's own dependency-free gate (`make check` = clean + portable # CPU build + C unit suites + Python stdlib tests) on the three claimed -# platforms. No model downloads, no CUDA, no external deps — by design (#140). +# platforms, then validate the locked Nix package on Linux and macOS. No model +# downloads or full-model inference — by design (#140). name: check on: @@ -10,6 +11,37 @@ on: branches: [main, dev] jobs: + committed-range: + name: Committed range whitespace + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Reject whitespace errors in the exact committed range + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + if [[ "$EVENT_NAME" == "pull_request" ]]; then + range="${PR_BASE_SHA}...${PR_HEAD_SHA}" + echo "checking pull-request range ${range}" + git diff --check "$range" + elif [[ "$PUSH_BEFORE_SHA" =~ ^0+$ ]]; then + empty_tree="$(git hash-object -t tree /dev/null)" + echo "checking new-history range ${empty_tree}..${PUSH_HEAD_SHA}" + git diff --check "$empty_tree" "$PUSH_HEAD_SHA" + else + range="${PUSH_BEFORE_SHA}..${PUSH_HEAD_SHA}" + echo "checking push range ${range}" + git diff --check "$range" + fi + linux: runs-on: ubuntu-latest steps: @@ -17,6 +49,67 @@ jobs: - name: make check run: make -C c check + wheel-backend: + name: Wheel backend (Linux, ${{ matrix.label }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - label: setuptools 77.0.1 minimum + requirement: setuptools==77.0.1 + expected: 77.0.1 + - label: current setuptools + requirement: setuptools + expected: current + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + # This is the lane's only dependency resolution step. The packaging test + # itself builds and installs without package-index access, using + # --no-index and --no-build-isolation. + - name: Install selected wheel backend + run: python -m pip install --upgrade "${{ matrix.requirement }}" + - name: Verify selected backend + env: + EXPECTED_SETUPTOOLS: ${{ matrix.expected }} + run: | + python - <<'PY' + import os + import setuptools + + actual = setuptools.__version__ + expected = os.environ["EXPECTED_SETUPTOOLS"] + print("setuptools", actual) + if expected != "current" and actual != expected: + raise SystemExit( + "expected setuptools %s, got %s" % (expected, actual) + ) + PY + - name: Build and smoke-test wheel without index access + run: >- + python -m unittest -v + c.tests.test_ramdisk_packaging.RamdiskPackagingTest.test_wheel_contains_runnable_ramdisk_control_plane + + wheel-isolated: + name: Wheel backend (Linux, default isolated PEP 517) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Report packaging frontend + run: python -m pip --version + - name: Build, install, and smoke-test with default PEP 517 isolation + env: + COLIBRI_TEST_ISOLATED_PEP517: '1' + run: >- + python -m unittest -v + c.tests.test_ramdisk_packaging.RamdiskPackagingTest.test_default_isolated_pep517_wheel_installs_and_runs + windows: # The job that would have caught #68/#137 pre-merge: native MinGW-w64 # (MSYS2/UCRT64), the exact toolchain the README's Windows port targets. @@ -55,3 +148,26 @@ jobs: run: brew install libomp - name: make check run: make -C c check + + nix: + name: Nix flake (${{ matrix.name }}) + strategy: + fail-fast: false + matrix: + include: + - name: Linux + os: ubuntu-latest + - name: macOS + os: macos-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Install Nix + # cachix/install-nix-action v31.10.7, pinned to its verified release commit. + uses: cachix/install-nix-action@a49548c11d9846ad46ecc0115273879b045f001c + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Validate locked, model-free flake + run: | + nix flake check --no-update-lock-file --print-build-logs + nix build --no-link --print-build-logs .#colibri diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 578ac41ab..404b5352f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -257,5 +257,68 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Install supported TUI dependency + run: python3 -m pip install -r c/requirements-tui.txt - name: Python test suite run: cd c && python3 -m unittest discover -s tests -p 'test_*.py' + + ramdisk-integration: + # P0.1: the only Python test that mounts a REAL tmpfs (prepare/status/destroy, + # swap-before/after, durable-state survival) is gated on COLI_RAMDISK_INTEGRATION=1 + # and otherwise never runs -- so those documented invariants were unverified in CI. + # Run the whole test as root in a private mount namespace. --kill-child and + # namespace teardown contain mounts even if the test or workflow is interrupted. + name: RAM-disk integration (real tmpfs lifecycle) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Real-tmpfs prepare/status/destroy + run: | + cd c + sudo unshare --mount --fork --kill-child --propagation private \ + env COLI_RAMDISK_INTEGRATION=1 \ + /usr/bin/python3 -m unittest discover \ + -s tests -p 'test_ramdisk_integration.py' -v + + ramdisk-e2e: + # P0.2: launch the REAL engine on a tmpfs int4 model and assert the live PROF + # output reports ~0 physical SSD reads (the zero-SSD-read contract the unit + # suite fakes via FakeEngine). This is the first CI job to generate and run a + # model, so it leans on the HF GLM modeling code in tools/make_glm_bench_model + # staying importable. This is a blocking contract: a physical-read regression + # must fail the pull request. + name: RAM-disk e2e (real engine, zero-SSD-read) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install fixture deps + run: | + pip install --no-input torch --index-url https://download.pytorch.org/whl/cpu + pip install --no-input transformers safetensors + - name: Build colibri + run: cd c && make colibri + - name: Generate canonical int4 fixture and stage it onto tmpfs + run: | + cd c + canonical="${RUNNER_TEMP}/glm_i4" + staged="/dev/shm/glm_i4" + python3 tools/make_glm_bench_model.py --fp8 \ + --output "${RUNNER_TEMP}/glm_fp8" --device cpu + python3 tools/convert_fp8_to_int4.py \ + --indir "${RUNNER_TEMP}/glm_fp8" --outdir "${canonical}" \ + --ebits 4 --group-size 128 --min-free-gb 1 + mkdir -p "${staged}" + cp -a --reflink=never "${canonical}/." "${staged}/" + findmnt -T "${canonical}" -n -o FSTYPE | grep -Ex 'ext4|xfs' + findmnt -T "${staged}" -n -o FSTYPE | grep -Fx 'tmpfs' + test "$(stat -c %d "${canonical}")" != "$(stat -c %d "${staged}")" + find "${canonical}" -maxdepth 1 -type f -name '*.safetensors' \ + -exec mv -- '{}' '{}.canonical-only' \; + - name: Zero-SSD-read end-to-end assertion + env: + COLI_RAMMAP_E2E_CANONICAL: ${{ runner.temp }}/glm_i4 + COLI_RAMMAP_E2E_STAGED: /dev/shm/glm_i4 + run: cd c && python3 -m unittest discover -s tests -p 'test_rammap_e2e.py' -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 11d166f73..38b45ca9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -108,6 +108,12 @@ jobs: cp c/resource_plan.py dist/ cp c/doctor.py dist/ cp c/autotune.py dist/ + cp c/ramdisk.py dist/ + cp c/ramdisk_ui.py dist/ + cp c/ramdisk_textual.py dist/ + mkdir -p dist/ramdisk_support + cp c/ramdisk_support/*.py dist/ramdisk_support/ + cp c/requirements-tui.txt dist/ cp LICENSE dist/ # web/dist sits NEXT TO coli in the archive; both coli and openai_server.py # probe that layout as well as the source checkout's one-level-up form. @@ -158,7 +164,7 @@ jobs: sys.exit("FAIL: coli would not resolve these next to itself: " + ", ".join(missing)) print("OK: every engine is where coli's engine_for() looks for it") PYCHK - out=$(python3 coli info 2>&1 || true) + out=$(python3 coli info --model . 2>&1) echo "$out" case "$out" in *"engine is not built"*) echo "FAIL: coli cannot find the packaged engine"; exit 1 ;; @@ -173,6 +179,22 @@ jobs: test -f web/dist/index.html || { echo "FAIL: dashboard missing from archive"; exit 1; } test -f web/dist/experts.json || { echo "FAIL: expert atlas missing from archive"; exit 1; } python3 -c "import sys; sys.path.insert(0, '.'); from openai_server import APIHandler as A; d = A.WEB_DIST; assert (d / 'index.html').is_file(), 'server resolved WEB_DIST to %s, which has no index.html' % d; assert (d / 'experts.json').is_file(), 'no expert atlas under %s' % d; print('OK: the packaged server resolves the dashboard at', d)" + echo "$out" | grep -Fq "ready ✓" || { echo "FAIL: coli cannot find the packaged engine"; exit 1; } + python3 coli ramdisk --help > ramdisk-help.txt + grep -Fq "interleaved = one shared model copy" ramdisk-help.txt || { + echo "FAIL: packaged RAM-disk UI modules did not load"; exit 1; + } + test -z "$(find ramdisk_support -type f ! -name '*.py' -print -quit)" || { + echo "FAIL: packaged RAM-disk support contains generated artifacts"; exit 1; + } + python3 -m compileall -q ramdisk.py ramdisk_ui.py ramdisk_textual.py ramdisk_support + PYTHONPATH=. python3 -c "import pkgutil, ramdisk_support; [__import__(item.name) for item in pkgutil.walk_packages(ramdisk_support.__path__, ramdisk_support.__name__ + '.')]" + test -f requirements-tui.txt || { + echo "FAIL: packaged Textual dependency contract is missing"; exit 1; + } + python3 -m pip install --disable-pip-version-check -r requirements-tui.txt + PYTHONPATH=. python3 -c "import ramdisk_textual; assert ramdisk_textual.RamdiskTextualApp" + echo "OK: packaged engine and guided Textual UI load from one archive" # Only the archives -- never the loose files. `dist/colibri-*.*` used to work by accident # (the engine was versioned, so it did not match); now that the engine is plainly named, diff --git a/.gitignore b/.gitignore index 8ab399166..8c71bf049 100644 --- a/.gitignore +++ b/.gitignore @@ -66,15 +66,38 @@ c/tests/test_tier c/mio_env/ c/bench/ c/tests/test_decode_batch +c/tests/test_st_mirror c/tests/test_i4_acc512 c/tests/test_idot +c/tests/test_i4_grouped +c/tests/test_stops +c/tests/test_kv_alloc +c/tests/test_int3 +c/tests/test_int3_load +c/tests/test_logit_nan +c/tests/test_pipe_block +c/tests/test_sample_nan +c/tests/test_tok_o200k c/tests/test_uring +c/tests/*.exe olmoe_merged/ olmoe_i4/ c/olmoe_merged/ c/olmoe_i4/ .idea c/tiny_inkling/ +c/tests/test_rammap +c/tests/test_rammap.exe +c/tests/test_st_pread +c/tests/test_st_pread.exe +c/tests/test_topp +c/tests/test_topp.exe +c/tests/test_dsa_select +c/tests/test_dsa_select.exe +c/tests/bench_topp +c/tests/bench_topp.exe +c/tests/bench_dsa_select +c/tests/bench_dsa_select.exe # Claude Code working directory (agent worktrees, session scratch) — never a repo artifact .claude/ diff --git a/README.md b/README.md index 127bcaa23..41f748199 100644 --- a/README.md +++ b/README.md @@ -328,8 +328,11 @@ git clone https://github.com/JustVugg/colibri && cd colibri/c ./setup.sh # checks gcc/OpenMP, builds, self-tests ``` -Want `coli` on your PATH? From a checkout, `pip install -e .` registers it (the -engine still lives in `c/` — an editable install from the clone, not a wheel). +Want `coli` on your PATH? From a checkout, `pip install -e .` registers it; the +editable install continues to use the engine in `c/`. A wheel bundles the +launcher, server control plane, and guided RAM-disk UI, but not a +platform-specific native engine binary. Build the engine from source or point +`COLI_ENGINE` at a compatible release binary before starting inference. ### 2. Get the model @@ -400,13 +403,23 @@ COLI_MODEL=/nvme/glm52_i4 ./coli plan # inspect the planned VRAM/RAM/disk pl COLI_MODEL=/nvme/glm52_i4 ./coli doctor # read-only readiness check COLI_MODEL=/nvme/glm52_i4 ./coli doctor --deep # strict tensors/shards/index/mirror preflight COLI_MODEL=/nvme/glm52_i4 ./coli tune # measure and save this machine's fastest safe execution profile +./coli ramdisk --model /nvme/glm52_i4 # Linux NUMA/tmpfs staging and managed engines ./coli web --model /nvme/glm52_i4 # API + dashboard, and opens a browser ./coli serve --model /nvme/glm52_i4 # API + dashboard, no browser (headless) ``` -On Windows the same commands work with `python coli chat --model D:\glm52_i4`. -The engine at runtime is pure C — python is only used by the one-time converter -and the optional API gateway. +On Windows the portable commands work with syntax such as +`python coli chat --model D:\glm52_i4`. +The engine runtime remains pure C. Python is used by the standard-library CLI, +one-time converter, optional API gateway, and RAM-disk control plane. + +`coli ramdisk` opens a guided Linux RAM-workspace console. A new workspace asks +for **Fastest GPU staging** (default), **Single RAM copy**, **Minimal RAM**, or +**Multiple NUMA replicas**. Each choice produces a draft that shows the exact +copy count, NUMA nodes, CPU mask, RAM cost, engines, and endpoints before +anything is mounted. GPU-aware placement falls back visibly to a shared +single-copy plan when it cannot prove a safe CUDA/NUMA layout; it never selects +replicas automatically. #### The same commands run any of the models @@ -456,6 +469,49 @@ Two things that differ per model, both documented in the per-model page: | OpenAI-compatible API, KV slots, web dashboard | [docs/api.md](docs/api.md) | | Grammar-forced drafts (structured output) | [docs/grammar-draft.md](docs/grammar-draft.md) | | Environment variable inventory | [docs/ENVIRONMENT.md](docs/ENVIRONMENT.md) | +| CLI flags and RAM-disk lifecycle commands | [docs/SETTINGS.md](docs/SETTINGS.md) | +| Shared full-model RAM-workspace TUI how-to | [docs/ramdisk-tui-howto.md](docs/ramdisk-tui-howto.md) | +| RAM-workspace TUI operator and developer guide | [docs/ramdisk-tui.md](docs/ramdisk-tui.md) | + +## Development checks + +The normal cross-platform gate is model-free: + +```bash +make -C c check +``` + +It builds the portable CPU engine and runs the C and Python unit, CLI, and +packaging suites. It does not require `COLI_MODEL`, download model weights, or +run full-model inference. The committed Nix flake provides the equivalent +reproducible Linux/macOS package validation: + +```bash +nix flake check --no-update-lock-file +nix build --no-link .#colibri +``` + +The repository uses Nix directly and does not carry a Devbox configuration; +`flake.lock` is the single committed dependency lock for this environment. + +Real Linux RAM-disk validation is explicitly opt-in. Run these only in a +suitable privileged/private mount environment: + +```bash +cd c +COLI_RAMDISK_INTEGRATION=1 python3 -m unittest discover -s tests -p 'test_ramdisk_integration.py' -v +COLI_RAMMAP_E2E_CANONICAL=/path/on/disk/glm_i4 \ +COLI_RAMMAP_E2E_STAGED=/dev/shm/glm_i4 \ + python3 -m unittest discover -s tests -p 'test_rammap_e2e.py' -v +``` + +The first gate exercises the real tmpfs lifecycle with a generated tiny +fixture. The second consumes two existing compatible GLM int4 namespaces +supplied by the caller; it never downloads one. The canonical namespace must be +block-backed (its safetensor shards are hidden after staging) and the staged +namespace must be tmpfs-backed and complete, proving the engine bound +`COLI_WEIGHTS_DIR` rather than falling back to SNAP. Both tests skip when their +environment gates are absent. ## What's next diff --git a/c/__init__.py b/c/__init__.py new file mode 100644 index 000000000..aabcc8c6d --- /dev/null +++ b/c/__init__.py @@ -0,0 +1,5 @@ +"""Bundled Python control-plane support for the Colibri engine. + +The native engine is built separately; this package keeps the ``coli`` launcher +and its Python support modules together in source, editable, and wheel installs. +""" diff --git a/c/coli b/c/coli index 44a09179a..2b35367f1 100755 --- a/c/coli +++ b/c/coli @@ -10,6 +10,8 @@ Run GLM-5.2 (744B) locally on CPU with roughly 15-26 GB of RAM. coli plan Disk / RAM / VRAM resource plan coli mirror Plan, stage, or verify a learned partial mirror coli doctor installation and execution-plan diagnostics + coli ramdisk NUMA-aware RAM-disk manager and interactive TUI + coli ramdisk plan --json scriptable RAM-disk capacity and staging plan coli bench [task...] quality benchmarks (MMLU/HellaSwag/...) coli convert convert GLM-5.2-FP8 to int4, one shard at a time coli build build the engine @@ -44,22 +46,140 @@ if sys.platform == "win32": except (AttributeError, OSError): pass HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, HERE) -# version.py sits next to this script in a source checkout, but an installed -# layout puts the launcher in $(PREFIX)/bin while the support modules live in -# $(PREFIX)/libexec/colibri — and a packager may strip it entirely (#575, -# FreeBSD port). The launcher must never crash over a version string. +_CONVENTIONAL_LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") +_LIBEXEC = _CONVENTIONAL_LIBEXEC +_LAYOUT_FILE = os.path.join(HERE, "coli.libexec") +_layout_present = os.path.isfile(_LAYOUT_FILE) +try: + with open(_LAYOUT_FILE, "r", encoding="utf-8") as _layout: + _configured_libexec = _layout.readline(4097).strip() +except FileNotFoundError: + _configured_libexec = None +except OSError as _layout_error: + raise SystemExit("coli: cannot read install layout %s: %s" % (_LAYOUT_FILE, _layout_error)) +if _layout_present: + if not _configured_libexec or not os.path.isabs(_configured_libexec): + raise SystemExit("coli: invalid install layout in %s" % _LAYOUT_FILE) + _LIBEXEC = os.path.normpath(_configured_libexec) + +_SUPPORT_MODULES = ( + "resource_plan.py", + "doctor.py", + "autotune.py", + "openai_server.py", + "ramdisk.py", + "ramdisk_ui.py", + "ramdisk_textual.py", +) +_SUPPORT_PACKAGE = "ramdisk_support" +_SUPPORT_PACKAGE_MODULES = ( + "__init__.py", + "accelerator.py", + "benchmark.py", + "cli.py", + "common.py", + "curses_ui.py", + "discovery.py", + "linux_ops.py", + "lifecycle.py", + "model.py", + "mounts.py", + "planning.py", + "platform_ops.py", + "presentation.py", + "presets.py", + "processes.py", + "runtime_monitor.py", + "state.py", +) +_engine_module_dir = ( + os.path.dirname(os.path.abspath(os.environ["COLI_ENGINE"])) + if os.environ.get("COLI_ENGINE") + else None +) + +def _support_presence(directory): + if not directory or not os.path.isdir(directory): + return () + present = [ + name for name in _SUPPORT_MODULES if os.path.isfile(os.path.join(directory, name)) + ] + package_path = os.path.join(directory, _SUPPORT_PACKAGE) + if os.path.lexists(package_path): + present.append(_SUPPORT_PACKAGE + "/") + return tuple(present) + +def _complete_support(directory, label): + present = _support_presence(directory) + if not present: + return False + missing = [name for name in _SUPPORT_MODULES if name not in present] + unsafe = [ + name + for name in _SUPPORT_MODULES + if os.path.islink(os.path.join(directory, name)) + ] + package_path = os.path.join(directory, _SUPPORT_PACKAGE) + if os.path.islink(package_path): + unsafe.append(_SUPPORT_PACKAGE + "/") + if not os.path.isdir(package_path): + missing.append(_SUPPORT_PACKAGE + "/") + else: + missing.extend( + _SUPPORT_PACKAGE + "/" + name + for name in _SUPPORT_PACKAGE_MODULES + if not os.path.isfile(os.path.join(package_path, name)) + ) + unsafe.extend( + _SUPPORT_PACKAGE + "/" + name + for name in _SUPPORT_PACKAGE_MODULES + if os.path.islink(os.path.join(package_path, name)) + ) + if unsafe: + raise SystemExit( + "coli: unsafe %s support bundle at %s; symlinked %s" + % (label, directory, ", ".join(unsafe)) + ) + if missing: + raise SystemExit( + "coli: incomplete %s support bundle at %s; missing %s" + % (label, directory, ", ".join(missing)) + ) + return True + +if _layout_present: + if not _complete_support(_LIBEXEC, "configured"): + raise SystemExit("coli: configured support bundle is missing: %s" % _LIBEXEC) + _SUPPORT_DIR = _LIBEXEC +elif _complete_support(HERE, "colocated"): + _SUPPORT_DIR = HERE +elif _complete_support(_CONVENTIONAL_LIBEXEC, "installed"): + _SUPPORT_DIR = _CONVENTIONAL_LIBEXEC +elif _complete_support(_engine_module_dir, "engine-directory"): + _SUPPORT_DIR = _engine_module_dir +else: + raise SystemExit( + "coli: no complete Python support bundle found beside the launcher or in %s" + % _CONVENTIONAL_LIBEXEC + ) + +# Select one complete support directory tree. Remove every competing candidate +# path, including the launcher's automatic sys.path[0], then expose exactly one. +for _candidate in (HERE, _LIBEXEC, _CONVENTIONAL_LIBEXEC, _engine_module_dir): + while _candidate and _candidate in sys.path: + sys.path.remove(_candidate) +sys.path.insert(0, _SUPPORT_DIR) +# A packager may strip version.py entirely (#575, FreeBSD port). The launcher +# must never crash over a version string once the required support bundle is +# otherwise complete. try: from version import __version__ as _version except ModuleNotFoundError: - sys.path.insert(0, os.path.join(os.path.dirname(HERE), "libexec", "colibri")) - try: - from version import __version__ as _version - except ModuleNotFoundError: - _version = "unknown" + _version = "unknown" # Run-in-place (source checkout, "cd c && ./coli ..."): the engine, the -# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py) and +# support modules (resource_plan.py, doctor.py, autotune.py, openai_server.py, +# ramdisk.py, ramdisk_ui.py, ramdisk_textual.py, ramdisk_support/) and # tools/ all live next to this script — unchanged from before. # # Installed layout ("make install"): this script is $(PREFIX)/bin/coli, @@ -68,7 +188,6 @@ except ModuleNotFoundError: # by users. COLI_ENGINE overrides the engine path explicitly if neither # guess is right (e.g. a custom packaging layout). _EXE = ".exe" if sys.platform == "win32" else "" -_LIBEXEC = os.path.join(os.path.dirname(HERE), "libexec", "colibri") _here_colibri = os.path.join(HERE, "colibri" + _EXE) _here_glm = os.path.join(HERE, "glm" + _EXE) @@ -84,7 +203,8 @@ elif os.path.exists(_here_glm): else: GLM = os.path.join(_LIBEXEC, "colibri" + _EXE) TOOLS = os.path.join(_LIBEXEC, "tools") - sys.path.insert(0, _LIBEXEC) # so `import resource_plan`, `doctor`, `openai_server` still resolve + if _LIBEXEC not in sys.path: + sys.path.insert(1, _LIBEXEC) # support custom layouts even before their directory exists # No invented default: a path from whoever's machine this was written on is worse than # no path at all, because every downstream error then names a directory the user never @@ -757,7 +877,7 @@ def chat_attached(a, base, model_id): print(f" {C.dim}goodbye — the engine keeps running for the next chat 🐦{C.r}") def kv_resume_notice(model_dir): - """SERVE mode silently resumes .coli_kv from disk (glm.c kv_disk_load): a chat + """SERVE mode silently resumes .coli_kv from durable state (glm.c kv_disk_load): a chat started today continues a conversation from days ago, with `first=0` so the turn is appended WITHOUT the [gMASK] prefix. The engine does announce it on stderr — but nothing here ever shows that: the drain thread's @@ -773,7 +893,8 @@ def kv_resume_notice(model_dir): So say it here, in Python, from the file itself: no pipe, no thread, no Windows deadlock risk (see the stderr comment below).""" - p=os.path.join(model_dir, ".coli_kv") + state_dir=os.environ.get("COLI_STATE_DIR") or model_dir + p=os.path.join(state_dir, ".coli_kv") try: with open(p,"rb") as f: if f.read(8)!=b"COLIKV1\0": return @@ -971,6 +1092,89 @@ def cmd_chat(a): def serve_pidfile(port): return os.path.join(tempfile.gettempdir(), f"coli-serve-{port}.pid") +def _stop_process_facts(pid): + """Return a signal-safe identity only for a demonstrable Colibri server.""" + try: + proc=f"/proc/{int(pid)}" + if os.stat(proc).st_uid != os.geteuid(): return None + raw=open(f"{proc}/stat").read() + close=raw.rfind(")") + fields=raw[close+2:].split() + if close<0 or len(fields)<20: return None + starttime=int(fields[19]) + argv=[ + item.decode("utf-8","replace") + for item in open(f"{proc}/cmdline","rb").read().split(b"\0") + if item + ] + comm=open(f"{proc}/comm").read().strip() + environ=set(open(f"{proc}/environ","rb").read().split(b"\0")) + except (OSError,ValueError,IndexError,PermissionError): + return None + try: serve_index=argv.index("serve") + except ValueError: serve_index=-1 + wrapper=serve_index>0 and any( + os.path.basename(value)=="coli" for value in argv[:serve_index] + ) + engine=comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3") and b"SERVE=1" in environ + if not wrapper and not engine: return None + return { + "pid":int(pid), + "starttime":starttime, + "kind":"wrapper" if wrapper else "engine", + } + +def _open_stop_target(pid, description): + """Pin a verified target with pidfd where available, closing PID-reuse races.""" + facts=_stop_process_facts(pid) + if not facts: return None + pidfd=None + if hasattr(os,"pidfd_open") and hasattr(signal,"pidfd_send_signal"): + try: + pidfd=os.pidfd_open(int(pid),0) + after=_stop_process_facts(pid) + if not after or ( + after["starttime"],after["kind"] + ) != ( + facts["starttime"],facts["kind"] + ): + os.close(pidfd); return None + facts=after + except OSError: + if pidfd is not None: + try: os.close(pidfd) + except OSError: pass + pidfd=None + facts["description"]=description + facts["pidfd"]=pidfd + return facts + +def _stop_target_matches(target): + current=_stop_process_facts(target["pid"]) + return bool( + current + and current["starttime"]==target["starttime"] + and current["kind"]==target["kind"] + ) + +def _signal_stop_target(target, signum): + """Revalidate immediately, then signal the pinned identity when possible.""" + if not _stop_target_matches(target): + raise ProcessLookupError("managed process identity changed") + if target.get("pidfd") is not None: + signal.pidfd_send_signal(target["pidfd"],signum,None,0) + else: + # The fallback retains a tiny read→kill race on older Python/kernels, + # but still removes the former two-second blind PID-reuse window. + os.kill(target["pid"],signum) + +def _close_stop_targets(targets): + for target in targets: + pidfd=target.get("pidfd") + if pidfd is not None: + try: os.close(pidfd) + except OSError: pass + def cmd_serve(a): arch=model_arch(a.model) engine=engine_for(a.model) @@ -994,7 +1198,7 @@ def cmd_serve(a): try: os.unlink(serve_pidfile(a.port)) except OSError: pass -def cmd_stop(a): +def cmd_stop(a, platform_name=None): """Shut down a running `coli serve` AND its engine — one command, no pkill. The engine re-execs itself for OMP tuning, so its process is named `exe`, not `glm`: every `pkill -x glm` in history silently killed nothing (that is @@ -1002,39 +1206,57 @@ def cmd_stop(a): real processes: the pidfile first, then /proc by cmdline/environ — only processes that are demonstrably ours (SERVE=1 + our SNAP, or `coli serve` in the command line).""" + selected_platform = sys.platform if platform_name is None else platform_name + if not str(selected_platform).startswith("linux"): + raise SystemExit( + "coli stop is supported only on Linux; use your platform process " + "manager to stop coli serve" + ) banner("stop") - targets=[] # (pid, descrizione) + targets={} # pid -> immutable start-time/signature identity (+ optional pidfd) pf=serve_pidfile(a.port) try: pid=int(open(pf).read().split()[0]) - os.kill(pid,0); targets.append((pid,f"coli serve (pidfile, port {a.port})")) + target=_open_stop_target(pid,f"coli serve (pidfile, port {a.port})") + if target: targets[pid]=target except (OSError,ValueError,IndexError): pass for pd in os.listdir("/proc"): if not pd.isdigit(): continue pid=int(pd) - try: - cmd=open(f"/proc/{pd}/cmdline","rb").read().replace(b"\0",b" ").decode("utf-8","replace") - if "coli" in cmd and " serve" in cmd and pid!=os.getpid(): - if not any(p==pid for p,_ in targets): targets.append((pid,"coli serve (cmdline)")) - comm=open(f"/proc/{pd}/comm").read().strip() - if comm in ("colibri","glm","exe","olmoe","inkling","kimi_k3"): - env=open(f"/proc/{pd}/environ","rb").read().replace(b"\0",b"\n").decode("utf-8","replace") - if "SERVE=1" in env: targets.append((pid,f"engine `{comm}` (SERVE=1)")) - except (OSError,PermissionError): continue + if pid==os.getpid() or pid in targets: continue + facts=_stop_process_facts(pid) + if not facts: continue + description=( + "coli serve (verified cmdline)" + if facts["kind"]=="wrapper" + else "Colibri engine (verified SERVE=1)" + ) + target=_open_stop_target(pid,description) + if target: targets[pid]=target if not targets: print(f" nothing running — no serve on port {a.port}, no SERVE engines"); return - for pid,desc in targets: print(f" {'would stop' if a.dry_run else 'stopping'} {pid}: {desc}") - if a.dry_run: return - for pid,_ in targets: - try: os.kill(pid, signal.SIGTERM) + selected=list(targets.values()) + try: + for target in selected: + print( + f" {'would stop' if a.dry_run else 'stopping'} " + f"{target['pid']}: {target['description']}" + ) + if a.dry_run: return + for target in selected: + try: _signal_stop_target(target,signal.SIGTERM) + except (OSError,ProcessLookupError): pass + time.sleep(2.0) + for target in selected: + try: + _signal_stop_target(target,signal.SIGKILL) + print(f" {target['pid']}: forced (SIGKILL)") + except (OSError,ProcessLookupError): pass # exited or identity changed: never signal a replacement + try: os.unlink(pf) except OSError: pass - time.sleep(2.0) - for pid,_ in targets: - try: os.kill(pid, signal.SIGKILL); print(f" {pid}: forced (SIGKILL)") - except OSError: pass # gia' morto: bene - try: os.unlink(pf) - except OSError: pass - print(f" {C.grn}✓ stopped{C.r} — RAM released") + print(f" {C.grn}✓ stopped{C.r} — RAM released") + finally: + _close_stop_targets(selected) def cmd_web(a): """serve + open the dashboard in the browser once the API answers.""" @@ -1117,7 +1339,6 @@ def cmd_convert(a): print(f" {C.dim}[2/2] int8 MTP head (speculative drafts){C.r}") sys.exit(subprocess.call(mtp_cmd+["--mtp"])) - def cmd_mirror(a): if not a.mirror: sys.exit("mirror path required: pass --mirror or set COLI_MODEL_MIRROR") @@ -1138,7 +1359,18 @@ def cmd_mirror(a): return subprocess.call(command) -def main(): +def cmd_ramdisk(a): + import ramdisk + if getattr(a,"ramdisk_action",None) is not None: + return ramdisk.dispatch(a,cli_path=os.path.abspath(__file__),engine_path=GLM) + if not (sys.stdin.isatty() and sys.stdout.isatty()): + print("coli ramdisk: the interactive TUI requires a terminal; " + "use `coli ramdisk plan --json` or another scriptable action", + file=sys.stderr) + return 2 + return ramdisk.launch_tui(a,cli_path=os.path.abspath(__file__),engine_path=GLM) + +def main(argv=None): common=argparse.ArgumentParser(add_help=False) common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile) common.add_argument("--auto-tier",action="store_true",help="automatically apply the RAM/VRAM plan") @@ -1174,6 +1406,9 @@ def main(): pm.add_argument("--usage") pm.add_argument("--budget-gib", type=float, default=0) pm.add_argument("--reserve-gib", type=float, default=10) + import ramdisk + prd=sub.add_parser("ramdisk",parents=[common],help="manage NUMA-aware tmpfs weight staging") + ramdisk.configure_parser(prd, common_parent=common) pd=sub.add_parser("doctor",parents=[common]) pd.add_argument("--json",action="store_true",help="emit a versioned JSON report") pd.add_argument("--deep",action="store_true", @@ -1236,11 +1471,11 @@ def main(): pc.add_argument("--group-size",type=int,default=64, help="int4 scale group size: 64 (default, group-scaled quality) or 0 (legacy per-row)") pc.add_argument("--no-mtp",action="store_true",help="skip the MTP head (no speculative drafts)") - a=ap.parse_args() + a=ap.parse_args(argv) handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"mirror":cmd_mirror, "doctor":cmd_doctor,"tune":cmd_tune, "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"stop":cmd_stop,"bench":cmd_bench, - "convert":cmd_convert,"web":cmd_web}.get(a.cmd) + "convert":cmd_convert,"web":cmd_web,"ramdisk":cmd_ramdisk}.get(a.cmd) if handler: sys.exit(handler(a) or 0) banner(); print(__doc__) diff --git a/c/ramdisk.py b/c/ramdisk.py new file mode 100644 index 000000000..5419072c1 --- /dev/null +++ b/c/ramdisk.py @@ -0,0 +1,1446 @@ +"""NUMA-aware tmpfs staging and managed-engine lifecycle for ``coli ramdisk``. + +The module intentionally uses only the Python standard library. Planning and +status are unprivileged; the only privileged subprocesses are the exact mount +and unmount commands issued by :func:`prepare` and :func:`destroy`. +""" + +from __future__ import print_function + +import argparse +import contextlib +import copy +import functools +import importlib +import json +import math +import os +import re +import secrets +import shutil +import signal +import socket +import subprocess +import sys +import threading +from ramdisk_support.common import ( + BENCHMARK_SCHEMA, + DEFAULT_MOUNT_ROOT, + GIB, + MANIFEST_VERSION, + MIB, + PLAN_SCHEMA, + PROFILE_LINE_RE, + STATUS_SCHEMA, + TMPFS_MAGIC, + USAGE_MERGE_RE, + RamdiskError, + _EngineCleanupError, + _OperationCancelled, + _format_range_list, + _parse_range_list, + _path_is_below, + _path_without_symlinks, + _percentile, + _positive_int, + _raise_if_cancelled, + _utc_now, +) +from ramdisk_support.cli import ( + _add_lifecycle_options as _cli_add_lifecycle_options, + _cli_exit_after_signal, + _cli_termination_guard, + _confirm as _cli_confirm, + _json_print as _cli_json_print, + _load_textual_frontend as _cli_load_textual_frontend, + _textual_dependency_missing as _cli_textual_dependency_missing, + configure_parser as _cli_configure_parser, + dispatch as _cli_dispatch, + launch_tui as _cli_launch_tui, +) +from ramdisk_support.accelerator import ( + ACCELERATOR_ENVIRONMENT_KEYS, + GPU_LAYOUT_CHOICES, + GPU_LAYOUT_DENSE_ATTENTION, + GPU_LAYOUT_DENSE_ATTENTION_SHARDED, + GPU_LAYOUT_EXPERTS_ONLY, + GPU_VRAM_RESERVE_BYTES, + _apply_managed_accelerator_environment, + _managed_accelerator_contract, + _managed_accelerator_environment, + apply_gpu_selection, + eligible_gpu_devices, + gpu_device_eligibility, +) +from ramdisk_support.discovery import ( + _cgroup_ancestors, + _cgroup_memberships, + _cgroup_mounts, + _discover_cgroup_memory, + _discover_gpus, + _mountinfo_unescape, + _parse_cgroup_bytes, + _resolve_cgroup_directory, + discover_hardware as _discover_hardware, +) +from ramdisk_support.linux_ops import ( + _ensure_busy_mount_scan_available as _linux_ensure_busy_mount_scan_available, + _filesystem_for_path as _linux_filesystem_for_path, + _fresh_user_binary, + _kernel_at_least, + _meminfo, + _mount_at as _linux_mount_at, + _mount_table, + _node_meminfo, + _noninteractive_privilege as _linux_noninteractive_privilege, + _physical_cores, + _privileged as _linux_privileged, + _process_status as _linux_process_status, + _read_cgroup_contract, + _read_cgroup_value, + _read_text, + _run, + _split_mount_options, + _status_allowed_list, + _sudo_ticket_keepalive, + _thread_sibling_groups, + _trusted_system_binary, + _unescape_mount, + _validate_noninteractive_sudo, +) +from ramdisk_support.lifecycle import ( + _assert_ready_mounts as _lifecycle_assert_ready_mounts, + _managed_path as _lifecycle_managed_path, + _managed_ports_for_plan as _lifecycle_managed_ports_for_plan, + _persisted_base_port as _lifecycle_persisted_base_port, + destroy as _lifecycle_destroy, + prepare as _lifecycle_prepare, + start as _lifecycle_start, + status as _lifecycle_status, + stop as _lifecycle_stop, +) +from ramdisk_support.mounts import ( + _available_for_mount as _mounts_available_for_mount, + _available_memory as _mounts_available_memory, + _busy_mount_references as _mounts_busy_mount_references, + _copy_one as _mounts_copy_one, + _copy_one_affined as _mounts_copy_one_affined, + _copy_stream, + _copy_worker_main, + _default_cgroup_available_memory as _mounts_default_cgroup_available_memory, + _host_available_for_mount as _mounts_host_available_for_mount, + _mount_option_list, + _mount_tmpfs as _mounts_mount_tmpfs, + _option_present, + _populate_mount as _mounts_populate_mount, + _reusable_empty_mountpoint, + _rollback_interrupted_mount as _mounts_rollback_interrupted_mount, + _sample_numa_allocation, + _sample_page_indices, + _source_still_matches as _mounts_source_still_matches, + _umount_path as _mounts_umount_path, + _validate_mount as _mounts_validate_mount, + _validate_namespace as _mounts_validate_namespace, +) +from ramdisk_support.model import ( + EXPERT_RE, + MAX_ST_HEADER, + _direct_tensor_set_eligible, + _sha256_file, + _shape_numel, + scan_model, +) +from ramdisk_support.planning import ( + _build_placement, + _engine_cpu_list as _planning_engine_cpu_list, + _load_profile, + _managed_numa_enabled as _planning_managed_numa_enabled, + _memory_node_list as _planning_memory_node_list, + _node_core_count as _planning_node_core_count, + _requested_ids, + _runtime_reserve, + _select_partial, + build_plan as _planning_build_plan, +) +from ramdisk_support.presets import ( + PRESET_CHOICES, + PRESET_GPU_FASTEST, + PRESET_MINIMAL, + PRESET_REPLICAS, + PRESET_SINGLE, + _engine_cuda_capable, + mark_preset_custom, + resolve_preset as _resolve_preset, +) +from ramdisk_support.platform_ops import ( + UNSUPPORTED_PLATFORM_REASON, + current_euid, + current_uid, + get_platform_ops, +) +from ramdisk_support.state import ( + _assert_canonical_usage_target as _state_assert_canonical_usage_target, + _assert_durable_state_dir as _state_assert_durable_state_dir, + _atomic_json, + _benchmarks_path, + _bind_usage_transaction as _state_bind_usage_transaction, + _canonical_usage_read as _state_canonical_usage_read, + _durable_unlink, + _ensure_atomic_parent, + _ensure_private_dir, + _fsync_directory, + _lifecycle_lock, + _load_manifest as _state_load_manifest, + _manifest_mount_layout, + _manifest_path, + _managed_usage_write as _state_managed_usage_write, + _merge_usage as _state_merge_usage, + _read_json, + _recover_delta as _state_recover_delta, + _save_manifest as _state_save_manifest, + _state_root, + _usage_merge_ids, + _usage_journal_transaction_id as _state_usage_journal_transaction_id, + _usage_read as _state_usage_read, + _usage_write as _state_usage_write, + _validate_usage_for_plan, +) + + +def _benchmark_module(): + return importlib.import_module("ramdisk_support.benchmark") + + +def _urllib_module(): + # Importing urllib.request initializes ssl and the HTTP stack. Keep that + # work behind the historical ``ramdisk.urllib.request`` compatibility seam + # instead of imposing it on every control-plane import. + importlib.import_module("urllib.request") + return importlib.import_module("urllib") + + +def _curses_ui_module(): + return importlib.import_module("ramdisk_support.curses_ui") + + +def _presentation_module(): + return importlib.import_module("ramdisk_support.presentation") + + +def _processes_module(): + return importlib.import_module("ramdisk_support.processes") + + +_LAZY_ATTRIBUTES = { + "BENCHMARK_PROMPT": (_benchmark_module, "BENCHMARK_PROMPT"), + "urllib": (_urllib_module, None), + "_TUI_SCREENS": (_curses_ui_module, "_TUI_SCREENS"), + "_TuiTerminationSignal": ( + _curses_ui_module, + "_TuiTerminationSignal", + ), + "_managed_children": (_processes_module, "_managed_children"), + "_managed_children_lock": ( + _processes_module, + "_managed_children_lock", + ), + "_runtime_admission_requirement": ( + _processes_module, + "_runtime_admission_requirement", + ), +} + + +def __getattr__(name): + try: + loader, attribute = _LAZY_ATTRIBUTES[name] + except KeyError: + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + loaded = loader() + value = loaded if attribute is None else getattr(loaded, attribute) + globals()[name] = value + return value + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY_ATTRIBUTES)) + + +def _proc_identity(pid, *, ops=None): + if ops is not None: + return ops.process_identity(pid) + return _processes_module()._proc_identity(pid) + + +def _group_alive(pgid, *, ops=None): + if ops is not None: + return ops.process_group_alive(pgid) + return _processes_module()._group_alive(pgid) + + +def _managed_launch_processes( + nonce, + uid, + *, + state_dir=None, + weights_dir=None, + not_before_starttime=None, + launcher_pid=None, + launcher_starttime=None, + launcher_cmdline=None, + expected_command=None, +): + return get_platform_ops().managed_launch_processes( + nonce, + uid, + state_dir=state_dir, + weights_dir=weights_dir, + not_before_starttime=not_before_starttime, + launcher_pid=launcher_pid, + launcher_starttime=launcher_starttime, + launcher_cmdline=launcher_cmdline, + expected_command=expected_command, + ) + + +def _process_start_boundary(): + return get_platform_ops().process_start_boundary() + + +def _current_process_identity(): + return get_platform_ops().process_identity(os.getpid()) + + +def _poll_managed_child(pid): + return _processes_module()._poll_managed_child(pid) + + +def _managed_child_liveness(pid): + return _processes_module()._managed_child_liveness(pid) + + +def _track_managed_child(process): + return _processes_module()._track_managed_child(process) + + +def _forget_managed_child(pid): + return _processes_module()._forget_managed_child(pid) + + +def _terminate_direct_child(*args, **kwargs): + return _processes_module()._terminate_direct_child(*args, **kwargs) + + +def _resolve_engine_path(*args, **kwargs): + return _processes_module()._resolve_engine_path(*args, **kwargs) + + +def _exclusive_lifecycle(function=None, *, require_process_control=False): + """Keep decorated facade calls patchable while state owns the lock.""" + if function is None: + return functools.partial( + _exclusive_lifecycle, + require_process_control=require_process_control, + ) + + @functools.wraps(function) + def wrapped(*args, **kwargs): + ops = get_platform_ops() + if not getattr(ops, "is_linux", False): + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + if require_process_control and not getattr( + ops, + "process_control_supported", + False, + ): + reason = getattr( + ops, + "process_control_reason", + UNSUPPORTED_PLATFORM_REASON, + ) + if not isinstance(reason, str) or not reason: + reason = UNSUPPORTED_PLATFORM_REASON + raise RamdiskError(reason) + with _lifecycle_lock(): + return function(*args, **kwargs) + + return wrapped + + +def _assert_durable_state_dir(path, plan=None): + return _state_assert_durable_state_dir( + path, + plan=plan, + filesystem_for_path=_filesystem_for_path, + ) + + +def _load_manifest(required=False): + return _state_load_manifest( + required=required, + filesystem_for_path=_filesystem_for_path, + read_json=_read_json, + manifest_path=_manifest_path, + state_root=_state_root, + benchmarks_path=_benchmarks_path, + assert_durable_state_dir=_assert_durable_state_dir, + ) + + +def _save_manifest(manifest): + return _state_save_manifest( + manifest, + atomic_json=_atomic_json, + manifest_path=_manifest_path, + ) + + +def _cgroup_available_memory(): + return _mounts_default_cgroup_available_memory( + discover_cgroup_memory=_discover_cgroup_memory, + ) + + +def discover_hardware(): + return _discover_hardware() + + + + + + + + + + + + +def build_plan(args, hardware=None, model=None): + return _planning_build_plan( + args, + hardware=hardware, + model=model, + discover_hardware=discover_hardware, + scan_model=scan_model, + load_profile=_load_profile, + select_partial=_select_partial, + runtime_reserve=_runtime_reserve, + build_placement=_build_placement, + reusable_empty_mountpoint=_reusable_empty_mountpoint, + filesystem_for_path=_filesystem_for_path, + state_root=_state_root, + manifest_path=_manifest_path, + benchmarks_path=_benchmarks_path, + current_euid=current_euid, + get_platform_ops=get_platform_ops, + ) + + +def resolve_preset( + preset_id, + args, + hardware=None, + model=None, + cli_path=None, + engine_path=None, +): + hardware = hardware or discover_hardware() + model = model or scan_model(args.model) + cuda_capable = None + if preset_id == PRESET_GPU_FASTEST: + try: + resolved_engine = _resolve_engine_path( + cli_path or os.path.join(os.path.dirname(__file__), "coli"), + engine_path=engine_path, + ) + except RamdiskError: + resolved_engine = engine_path + cuda_capable = _engine_cuda_capable(resolved_engine) + return _resolve_preset( + preset_id, + args, + hardware=hardware, + model=model, + build_plan=build_plan, + load_profile=_load_profile, + cuda_capable=cuda_capable, + ) + + +def _add_lifecycle_options(parser, suppress=False): + return _cli_add_lifecycle_options( + parser, + suppress=suppress, + ) + + +def configure_parser(parser, common_parent=None): + return _cli_configure_parser( + parser, + common_parent=common_parent, + ) + + +def _json_print(value): + return _cli_json_print(value) + + +def _human_plan(plan): + return _presentation_module()._human_plan(plan) + + +def _mount_at(path): + return _linux_mount_at(path, mount_table=_mount_table) + + +def _filesystem_for_path(path): + return _linux_filesystem_for_path( + path, + mount_table=_mount_table, + ) + + +@contextlib.contextmanager +def _noninteractive_privilege(keepalive=False, cancel_event=None): + with _linux_noninteractive_privilege( + keepalive=keepalive, + cancel_event=cancel_event, + trusted_system_binary=_trusted_system_binary, + sudo_ticket_keepalive=_sudo_ticket_keepalive, + ): + yield + + +def _privileged(command, hardware): + return _linux_privileged( + command, + hardware, + trusted_system_binary=_trusted_system_binary, + ) + + +def _mount_tmpfs(plan, mount): + return _mounts_mount_tmpfs( + plan, + mount, + trusted_system_binary=_trusted_system_binary, + run=_run, + privileged=_privileged, + rollback_interrupted_mount=_rollback_interrupted_mount, + ) + + +def _umount_path(path, hardware): + return _mounts_umount_path( + path, + hardware, + trusted_system_binary=_trusted_system_binary, + run=_run, + privileged=_privileged, + ) + + +def _rollback_interrupted_mount(plan, mount, effective_thp, effective_noswap, cause): + return _mounts_rollback_interrupted_mount( + plan, + mount, + effective_thp, + effective_noswap, + cause, + mount_at=_mount_at, + validate_mount=_validate_mount, + umount_path=_umount_path, + ) + + +def _validate_mount(mount, plan): + return _mounts_validate_mount( + mount, + plan, + mount_at=_mount_at, + ) + + +def _available_memory(): + return _mounts_available_memory( + meminfo=_meminfo, + cgroup_available_memory=_cgroup_available_memory, + ) + + +def _host_available_for_mount(mount, plan=None): + return _mounts_host_available_for_mount( + mount, + plan=plan, + meminfo=_meminfo, + node_meminfo=_node_meminfo, + ) + + +def _available_for_mount(mount, plan=None): + return _mounts_available_for_mount( + mount, + plan=plan, + host_available_for_mount=_host_available_for_mount, + cgroup_available_memory=_cgroup_available_memory, + ) + + +def _copy_one( + src, + destination, + expected_size, + reserve_floor, + progress=None, + available=None, + cancel_event=None, +): + return _mounts_copy_one( + src, + destination, + expected_size, + reserve_floor, + progress, + _available_memory if available is None else available, + cancel_event, + ) + + +def _copy_one_affined( + src, + destination, + expected_size, + node, + numactl, + cpu_list, + reserve_floor, + progress=None, + available=None, + cancel_event=None, +): + available = _available_memory if available is None else available + return _mounts_copy_one_affined( + src, + destination, + expected_size, + node, + numactl, + cpu_list, + reserve_floor, + progress, + available, + cancel_event, + run=_run, + worker_entrypoint=os.path.abspath(__file__), + ) + + +def _populate_mount(plan, mount, source_root=None, progress=None, cancel_event=None): + return _mounts_populate_mount( + plan, + mount, + source_root=source_root, + progress=progress, + cancel_event=cancel_event, + available_for_mount=_available_for_mount, + copy_one=_copy_one, + copy_one_affined=_copy_one_affined, + engine_cpu_list=_engine_cpu_list, + ) + + +def _validate_namespace(plan, mount, sample_numa=True): + return _mounts_validate_namespace( + plan, + mount, + sample_numa=sample_numa, + sample_numa_allocation=_sample_numa_allocation, + ) + + +def _source_still_matches(plan): + return _mounts_source_still_matches( + plan, + scan_model_fn=scan_model, + ) + + +def _confirm(message, accepted=False): + return _cli_confirm( + message, + accepted=accepted, + ) + + +@_exclusive_lifecycle +def prepare( + args, + progress=None, + display_plan=True, + expected_plan_token=None, + cancel_event=None, +): + return _lifecycle_prepare( + args, + progress=progress, + display_plan=display_plan, + expected_plan_token=expected_plan_token, + cancel_event=cancel_event, + load_manifest=_load_manifest, + build_plan=build_plan, + managed_ports_for_plan=_managed_ports_for_plan, + plan_confirmation_token=_plan_confirmation_token, + render_plan=_human_plan, + confirm=_confirm, + save_manifest=_save_manifest, + mount_at=_mount_at, + mount_tmpfs=_mount_tmpfs, + umount_path=_umount_path, + validate_mount=_validate_mount, + populate_mount=_populate_mount, + validate_namespace=_validate_namespace, + source_still_matches=_source_still_matches, + ensure_busy_mount_scan_available=( + _ensure_busy_mount_scan_available + ), + durable_unlink=_durable_unlink, + manifest_path=_manifest_path, + mount_table=_mount_table, + path_is_below=_path_is_below, + busy_mount_references=_busy_mount_references, + ) + + +def _process_group_members(pgid): + return _processes_module()._process_group_members( + pgid, + proc_identity=_proc_identity, + ) + + +def _process_matches(record): + return _processes_module()._process_matches( + record, + proc_identity=_proc_identity, + process_group_members=_process_group_members, + ) + + +def _process_tree_alive(record, actual): + return _processes_module()._process_tree_alive( + record, + actual, + group_alive=_group_alive, + proc_identity=_proc_identity, + ) + + +def _assert_canonical_usage_target(canonical_path, plan=None): + return _state_assert_canonical_usage_target( + canonical_path, + plan=plan, + source_still_matches=_source_still_matches, + ) + + +def _merge_usage(record, canonical_path, plan=None, keep_journal=False): + return _state_merge_usage( + record, + canonical_path, + plan=plan, + keep_journal=keep_journal, + filesystem_for_path=_filesystem_for_path, + source_still_matches=_source_still_matches, + ) + + +def _usage_read(path, plan=None): + if plan is None: + return _state_usage_read(path) + return _state_canonical_usage_read( + path, + plan=plan, + source_still_matches=_source_still_matches, + ) + + +def _usage_write( + path, + counts, + merge_id=None, + merge_ids=None, + *, + plan=None, + expected_snapshot=None, + validator=None, + require_native=False, +): + if plan is not None: + if ( + merge_id is not None + or merge_ids is not None + or expected_snapshot is not None + or validator is not None + or require_native + ): + raise RamdiskError( + "managed usage seed does not accept generic write authority" + ) + return _state_managed_usage_write( + path, + counts, + plan=plan, + filesystem_for_path=_filesystem_for_path, + ) + return _state_usage_write( + path, + counts, + merge_id=merge_id, + merge_ids=merge_ids, + expected_snapshot=expected_snapshot, + validator=validator, + require_native=require_native, + ) + + +def _bind_usage_transaction(record, plan=None, *, reserved_ids=None): + return _state_bind_usage_transaction( + record, + plan=plan, + filesystem_for_path=_filesystem_for_path, + reserved_ids=reserved_ids, + ) + + +def _usage_journal_transaction_id(state_dir, plan=None): + return _state_usage_journal_transaction_id( + state_dir, + plan=plan, + filesystem_for_path=_filesystem_for_path, + ) + + +def _recover_delta( + state_dir, + canonical_path, + plan=None, + expected_merge_id=None, +): + return _state_recover_delta( + state_dir, + canonical_path, + plan=plan, + expected_merge_id=expected_merge_id, + filesystem_for_path=_filesystem_for_path, + source_still_matches=_source_still_matches, + ) + + +def _assert_ready_mounts(manifest): + return _lifecycle_assert_ready_mounts( + manifest, + source_still_matches=_source_still_matches, + validate_mount=_validate_mount, + validate_namespace=_validate_namespace, + ) + + +def _admit_runtime(plan, mount, benchmark=False): + return _processes_module()._admit_runtime( + plan, + mount, + benchmark=benchmark, + available_for_mount=_available_for_mount, + ) + + +def _admit_concurrent_runtimes(plan, mounts, benchmark=False): + return _processes_module()._admit_concurrent_runtimes( + plan, + mounts, + benchmark=benchmark, + host_available_for_mount=_host_available_for_mount, + cgroup_available_memory=_cgroup_available_memory, + ) + + +def _assert_effective_masks_unchanged(plan): + return _processes_module()._assert_effective_masks_unchanged( + plan, + discover_hardware=discover_hardware, + ) + + +def _terminate_verified_group(record, term_seconds=10.0, kill_seconds=3.0): + return _processes_module()._terminate_verified_group( + record, + term_seconds=term_seconds, + kill_seconds=kill_seconds, + managed_child_liveness=_managed_child_liveness, + process_matches=_process_matches, + ) + + +def _wait_managed_ready( + record, + timeout, + api_key=None, + cancel_event=None, + *, + urlopen=None, +): + if urlopen is None: + urlopen = _urllib_module().request.urlopen + return _processes_module()._wait_managed_ready( + record, + timeout, + api_key=api_key, + cancel_event=cancel_event, + process_matches=_process_matches, + urlopen=urlopen, + ) + + +@_exclusive_lifecycle(require_process_control=True) +def start(args, cli_path=None, engine_path=None, cancel_event=None): + return _lifecycle_start( + args, + cli_path=cli_path, + engine_path=engine_path, + cancel_event=cancel_event, + default_cli_path=os.path.join(os.path.dirname(__file__), "coli"), + load_manifest=_load_manifest, + assert_effective_masks_unchanged=_assert_effective_masks_unchanged, + assert_ready_mounts=_assert_ready_mounts, + process_matches=_process_matches, + group_alive=_group_alive, + managed_child_liveness=_managed_child_liveness, + save_manifest=_save_manifest, + merge_usage=_merge_usage, + bind_usage_transaction=_bind_usage_transaction, + persisted_base_port=_persisted_base_port, + fresh_user_binary=_fresh_user_binary, + admit_concurrent_runtimes=_admit_concurrent_runtimes, + state_root=_state_root, + ensure_private_dir=_ensure_private_dir, + assert_durable_state_dir=_assert_durable_state_dir, + usage_journal_transaction_id=_usage_journal_transaction_id, + recover_delta=_recover_delta, + usage_read=_usage_read, + usage_write=_usage_write, + validate_usage_for_plan=_validate_usage_for_plan, + managed_numa_enabled=_managed_numa_enabled, + memory_node_list=_memory_node_list, + engine_cpu_list=_engine_cpu_list, + node_core_count=_node_core_count, + normalized_runtime_knobs=_normalized_runtime_knobs, + apply_managed_accelerator_environment=( + _apply_managed_accelerator_environment + ), + invoking_uid=current_uid, + process_start_boundary=_process_start_boundary, + current_process_identity=_current_process_identity, + proc_identity=_proc_identity, + wait_managed_ready=_wait_managed_ready, + track_managed_child=_track_managed_child, + terminate_verified_group=_terminate_verified_group, + terminate_direct_child=_terminate_direct_child, + forget_managed_child=_forget_managed_child, + ) + + +@_exclusive_lifecycle(require_process_control=True) +def stop(args=None): + return _lifecycle_stop( + args, + load_manifest=_load_manifest, + discover_managed_launches=_managed_launch_processes, + process_matches=_process_matches, + process_group_members=_process_group_members, + group_alive=_group_alive, + managed_child_liveness=_managed_child_liveness, + save_manifest=_save_manifest, + terminate_verified_group=_terminate_verified_group, + merge_usage=_merge_usage, + bind_usage_transaction=_bind_usage_transaction, + ) + + +def _busy_mount_references(path, *, ops=None, hardware=None): + return _mounts_busy_mount_references( + path, + ops=ops, + hardware=hardware, + ) + + +def _ensure_busy_mount_scan_available(path, hardware=None): + return _linux_ensure_busy_mount_scan_available( + path, + hardware, + trusted_system_binary=_trusted_system_binary, + ) + + +def _managed_path(path, mount_root): + return _lifecycle_managed_path(path, mount_root) + + +@_exclusive_lifecycle(require_process_control=True) +def destroy(args, expected_manifest_token=None): + return _lifecycle_destroy( + args, + expected_manifest_token=expected_manifest_token, + load_manifest=_load_manifest, + save_manifest=_save_manifest, + manifest_confirmation_token=_manifest_confirmation_token, + confirm=_confirm, + stop_action=stop, + mount_table=_mount_table, + path_is_below=_path_is_below, + managed_path=_managed_path, + mount_at=_mount_at, + validate_mount=_validate_mount, + validate_namespace=_validate_namespace, + busy_mount_references=_busy_mount_references, + umount_path=_umount_path, + durable_unlink=_durable_unlink, + manifest_path=_manifest_path, + ) + + +def status(deep=True): + """Return lifecycle status, optionally skipping shard/header revalidation. + + Scriptable ``status`` always uses the deep default. The curses dashboard + polls the cheap form and exposes an explicit refresh for a new deep model + scan, avoiding repeated reads of every safetensors header on large models. + """ + return _lifecycle_status( + deep=deep, + load_manifest=_load_manifest, + manifest_path=_manifest_path, + source_still_matches=_source_still_matches, + mount_at=_mount_at, + validate_mount=_validate_mount, + validate_namespace=_validate_namespace, + process_matches=_process_matches, + managed_child_liveness=_managed_child_liveness, + ) + + +def _source_build_identity(): + return _benchmark_module()._source_build_identity( + __file__, + environ=os.environ, + which=shutil.which, + run=_run, + ) + + +def _parse_profiler(text, elapsed): + return _benchmark_module()._parse_profiler(text, elapsed) + + +def _node_core_count(plan, node=None): + return _planning_node_core_count( + plan, + node=node, + ) + + +def _engine_cpu_list(plan, node=None): + return _planning_engine_cpu_list( + plan, + node=node, + ) + + +def _memory_node_list(plan, node=None): + return _planning_memory_node_list( + plan, + node=node, + ) + + +def _managed_numa_enabled(plan, node=None): + return _planning_managed_numa_enabled( + plan, + node=node, + ) + + +def _normalized_runtime_knobs(plan, knobs, node=None): + return _benchmark_module()._normalized_runtime_knobs( + plan, + knobs, + node=node, + node_core_count=_node_core_count, + ) + + +def _benchmark_environment(manifest, weights_dir, state_dir, rammap, node=None, knobs=None): + return _benchmark_module()._benchmark_environment( + manifest, + weights_dir, + state_dir, + rammap, + node=node, + knobs=knobs, + environ=os.environ, + node_core_count=_node_core_count, + engine_cpu_list=_engine_cpu_list, + memory_node_list=_memory_node_list, + managed_numa_enabled=_managed_numa_enabled, + normalized_runtime_knobs=_normalized_runtime_knobs, + apply_managed_accelerator_environment=( + _apply_managed_accelerator_environment + ), + ) + + +def _cancellable_engine_type( + engine_type, + read_engine_turn, + ready_marker, + cancel_event, +): + return _benchmark_module()._cancellable_engine_type( + engine_type, + read_engine_turn, + ready_marker, + cancel_event, + ) + + +def _benchmark_generate( + engine, + prompt, + on_text, + cancel_event, + client_cancelled_type, +): + return _benchmark_module()._benchmark_generate( + engine, + prompt, + on_text, + cancel_event, + client_cancelled_type, + ) + + +def _score_variant( + engine_path, + manifest, + name, + weights_dir, + rammap, + knobs, + cancel_event=None, +): + return _benchmark_module()._score_variant( + engine_path, + manifest, + name, + weights_dir, + rammap, + knobs, + cancel_event=cancel_event, + state_root=_state_root, + ensure_private_dir=_ensure_private_dir, + assert_durable_state_dir=_assert_durable_state_dir, + admit_runtime=_admit_runtime, + fresh_user_binary=_fresh_user_binary, + engine_cpu_list=_engine_cpu_list, + benchmark_environment=_benchmark_environment, + cancellable_engine_type=_cancellable_engine_type, + benchmark_generate=_benchmark_generate, + ) + + +def _aggregate_score(manifest, engine_path=None, knobs=None, cancel_event=None): + return _benchmark_module()._aggregate_score( + manifest, + engine_path=engine_path, + knobs=knobs, + cancel_event=cancel_event, + state_root=_state_root, + ensure_private_dir=_ensure_private_dir, + assert_durable_state_dir=_assert_durable_state_dir, + admit_concurrent_runtimes=_admit_concurrent_runtimes, + fresh_user_binary=_fresh_user_binary, + engine_cpu_list=_engine_cpu_list, + normalized_runtime_knobs=_normalized_runtime_knobs, + benchmark_environment=_benchmark_environment, + cancellable_engine_type=_cancellable_engine_type, + benchmark_generate=_benchmark_generate, + ) + + +def _system_score(manifest, variants, swap_before, swap_after, aggregate=None): + return _benchmark_module()._system_score( + manifest, + variants, + swap_before, + swap_after, + aggregate=aggregate, + meminfo=_meminfo, + statvfs=getattr(os, "statvfs", None), + ) + + +@_exclusive_lifecycle(require_process_control=True) +def benchmark(args, cli_path=None, engine_path=None, cancel_event=None): + cli_path = cli_path or os.path.join(os.path.dirname(__file__), "coli") + return _benchmark_module().run_benchmark( + args, + cli_path, + engine_path=engine_path, + cancel_event=cancel_event, + load_manifest=_load_manifest, + assert_effective_masks_unchanged=_assert_effective_masks_unchanged, + assert_ready_mounts=_assert_ready_mounts, + resolve_engine_path=_resolve_engine_path, + node_core_count=_node_core_count, + score_variant=_score_variant, + discover_hardware=discover_hardware, + aggregate_score=_aggregate_score, + system_score=_system_score, + filesystem_for_path=_filesystem_for_path, + source_build_identity=_source_build_identity, + read_json=_read_json, + benchmarks_path=_benchmarks_path, + atomic_json=_atomic_json, + save_manifest=_save_manifest, + argv=sys.argv, + ) + + +def _human_status(report): + return _presentation_module()._human_status(report) + + +def _human_benchmark(result): + return _presentation_module()._human_benchmark(result) + + +def _managed_process_metrics(record): + return _processes_module()._managed_process_metrics( + record, + process_matches=_process_matches, + process_group_members=_process_group_members, + process_status=lambda pid: _linux_process_status( + pid, + read_text=_read_text, + ), + ) + + +def _managed_ports_for_plan(plan, base_port=8000): + return _lifecycle_managed_ports_for_plan( + plan, + base_port=base_port, + ) + + +def _persisted_base_port(manifest): + """Recover the last base port, including manifests predating that field.""" + return _lifecycle_persisted_base_port(manifest) + + +def _placement_summary(plan, base_port=8000): + return _presentation_module()._placement_summary( + plan, + base_port=base_port, + ) + + +def _plan_confirmation_token(plan): + return _presentation_module()._plan_confirmation_token(plan) + + +def _manifest_confirmation_token(manifest): + return _presentation_module()._manifest_confirmation_token( + manifest, + persisted_base_port=_persisted_base_port, + ) + + +def _prepare_confirmation(plan, base_port=8000): + return _presentation_module()._prepare_confirmation( + plan, + base_port=base_port, + ) + + +def _prepare_confirmation_rows(plan, base_port=8000): + return _presentation_module()._prepare_confirmation_rows( + plan, + base_port=base_port, + ) + + +def dispatch(args, cli_path=None, engine_path=None, system=None): + return _cli_dispatch( + args, + cli_path=cli_path, + engine_path=engine_path, + system=system, + build_plan=build_plan, + prepare=prepare, + status=status, + benchmark=benchmark, + start=start, + stop=stop, + destroy=destroy, + human_plan=_human_plan, + human_status=_human_status, + human_benchmark=_human_benchmark, + json_print=_json_print, + termination_guard=_cli_termination_guard, + ) + + +_tui_worker_guard = threading.Lock() +_tui_worker = None + + +def _tui_plan_rows(plan, report, active=False, base_port=8000, confirmation=None): + return _presentation_module()._tui_plan_rows( + plan, + report, + active=active, + base_port=base_port, + confirmation=confirmation, + ) + + +def _tui_preset_rows(): + return _presentation_module()._tui_preset_rows() + + +def _tui_hardware_rows(hardware): + return _presentation_module()._tui_hardware_rows(hardware) + + +def _tui_activity_rows(report, hardware, process_metrics=None): + return _presentation_module()._tui_activity_rows( + report, + hardware, + process_metrics=process_metrics, + meminfo=_meminfo, + ) + + +def _tui_benchmark_rows(history): + return _presentation_module()._tui_benchmark_rows(history) + + +def _tui_settings_rows(args, plan, report, base_port=8000): + return _presentation_module()._tui_settings_rows( + args, + plan, + report, + base_port=base_port, + ) + + +def _tui_help_rows(): + return _presentation_module()._tui_help_rows() + + +def _tui_idle_action_hint(screen, plan, report): + return _presentation_module()._tui_idle_action_hint( + screen, + plan, + report, + ) + + +def _tui(stdscr, initial, cli_path, engine_path): + return _curses_ui_module()._tui( + stdscr, + initial, + cli_path, + engine_path, + bindings=sys.modules[__name__], + ) + + +def _load_textual_frontend(): + return _cli_load_textual_frontend() + + +def _textual_dependency_missing(error): + return _cli_textual_dependency_missing(error) + + +def _run_tui_frontend(callback): + return _curses_ui_module()._run_tui_frontend( + callback, + bindings=sys.modules[__name__], + ) + + +def _tui_review_scroll(pending_action, requested_scroll): + return _curses_ui_module()._tui_review_scroll( + pending_action, + requested_scroll, + ) + + +def _tui_wrap_rows(rows, width): + return _curses_ui_module()._tui_wrap_rows(rows, width) + + +@contextlib.contextmanager +def _curses_termination_guard(): + with _curses_ui_module()._curses_termination_guard(): + yield + + +def _join_tui_worker(active): + return _curses_ui_module()._join_tui_worker(active) + + +def launch_tui(args, cli_path=None, engine_path=None, system=None): + global _tui_worker + + def finish_frontend(): + global _tui_worker + with _tui_worker_guard: + if ( + _tui_worker is not None + and not _tui_worker["thread"].is_alive() + ): + _tui_worker = None + + return _cli_launch_tui( + args, + cli_path=cli_path, + engine_path=engine_path, + target_platform=system, + lifecycle=sys.modules[__name__], + run_tui_frontend=_run_tui_frontend, + legacy_tui=_tui, + curses_termination_guard=_curses_termination_guard, + finish_frontend=finish_frontend, + load_textual_frontend=_load_textual_frontend, + ) + + +__all__ = sorted( + set(name for name in globals() if not name.startswith("_")) + | set(name for name in _LAZY_ATTRIBUTES if not name.startswith("_")) +) + + +if __name__ == "__main__": + if len(sys.argv) == 5 and sys.argv[1] == "--copy-worker": + try: + sys.exit(_copy_worker_main(sys.argv[2], sys.argv[3], sys.argv[4])) + except Exception as error: + print(error, file=sys.stderr) + sys.exit(1) + print("ramdisk.py is a support module; run `coli ramdisk`", file=sys.stderr) + sys.exit(2) diff --git a/c/ramdisk_support/__init__.py b/c/ramdisk_support/__init__.py new file mode 100644 index 000000000..96645337a --- /dev/null +++ b/c/ramdisk_support/__init__.py @@ -0,0 +1 @@ +"""Internal implementation modules for the ``ramdisk`` compatibility facade.""" diff --git a/c/ramdisk_support/accelerator.py b/c/ramdisk_support/accelerator.py new file mode 100644 index 000000000..99da4c9b5 --- /dev/null +++ b/c/ramdisk_support/accelerator.py @@ -0,0 +1,572 @@ +"""Managed accelerator environment contracts for RAM-workspace processes.""" + +from __future__ import print_function + +import re + +from .common import GIB, RamdiskError, _format_range_list + + +GPU_LAYOUT_EXPERTS_ONLY = "experts-only" +GPU_LAYOUT_DENSE_ATTENTION = "dense-attention" +GPU_LAYOUT_DENSE_ATTENTION_SHARDED = "dense-attention-sharded" +GPU_LAYOUT_CHOICES = ( + GPU_LAYOUT_EXPERTS_ONLY, + GPU_LAYOUT_DENSE_ATTENTION, + GPU_LAYOUT_DENSE_ATTENTION_SHARDED, +) +GPU_VRAM_RESERVE_BYTES = 2 * GIB + +_GPU_LAYOUT_ENVIRONMENT = { + GPU_LAYOUT_EXPERTS_ONLY: { + "CUDA_DENSE": "0", + "COLI_CUDA_ATTN": "0", + "COLI_CUDA_ATTN_SHARD": "0", + }, + GPU_LAYOUT_DENSE_ATTENTION: { + "CUDA_DENSE": "1", + "COLI_CUDA_ATTN": "1", + "COLI_CUDA_ATTN_SHARD": "0", + }, + GPU_LAYOUT_DENSE_ATTENTION_SHARDED: { + "CUDA_DENSE": "1", + "COLI_CUDA_ATTN": "1", + "COLI_CUDA_ATTN_SHARD": "1", + }, +} + + +ACCELERATOR_ENVIRONMENT_KEYS = ( + "CUDA_DEVICE_ORDER", + "CUDA_VISIBLE_DEVICES", + "COLI_CUDA", + "COLI_METAL", + "COLI_VULKAN", + "COLI_CUDA_DUAL_PROJ", + "COLI_CUDA_MTP", + "COLI_CUDA_PIPE", + "COLI_CUDA_PIPE_SHARD", + "COLI_CUDA_PIPE_S_MIN", + "COLI_CUDA_PROFILE", + "COLI_CUDA_RESID", + "COLI_CUDA_ROUTER", + "COLI_CUDA_SHARED_W4A16", + "COLI_CUDA_SHARED_W4A16_MIN_ROWS", + "COLI_CUDA_TC_INT4", + "COLI_CUDA_TC_MIN_ROWS", + "COLI_CUDA_TC_W4A16", + "COLI_CUDA_TC_W4A16_MIN", + "COLI_CUDA_W4_PACKED", + "COLI_GPU", + "COLI_GPUS", + "COLI_GPU_FAIL_AFTER", + "CUDA_EXPERT_GB", + "CUDA_DENSE", + "CUDA_RAW_EXPERTS", + "CUDA_RESERVE_GB", + "COLI_CUDA_ATTN", + "COLI_CUDA_ATTN_SHARD", + "CUDA_RELEASE_HOST", + "COLI_CUDA_ASYNC", + "COLI_GROUP_ASYNC", + "COLI_DSA_GATHER", + "DRAFT", + "COLI_MMAP", + "COLI_RAMMAP", + "PIN", + "PIN_GB", + "PIN_FILL", + "REPIN", + "REPIN_VERBOSE", + "SPEC_PIN", +) + +ACCELERATOR_ENVIRONMENT_PREFIXES = ( + "COLI_ANS_", + "COLI_CUDA_", + "COLI_METAL_", + "COLI_VK_", +) + + +def _normalize_gpu_layout(value): + layout = str(value or GPU_LAYOUT_EXPERTS_ONLY) + if layout not in GPU_LAYOUT_CHOICES: + raise RamdiskError( + "GPU layout must be one of: %s" + % ", ".join(GPU_LAYOUT_CHOICES) + ) + return layout + + +def gpu_device_eligibility(device, hardware): + """Return ``(eligible, reason)`` for one discovered NVIDIA GPU.""" + if not isinstance(device, dict): + return False, "GPU discovery returned a malformed device record" + index = device.get("index") + if ( + isinstance(index, bool) + or not isinstance(index, int) + or index < 0 + ): + return False, "GPU discovery returned an invalid device index" + discovery = hardware.get("gpu_discovery") or {} + if discovery.get("cuda_visible_devices_present"): + return False, ( + discovery.get("selection_error") + or "ambient CUDA_VISIBLE_DEVICES prevents safe GPU selection" + ) + node = device.get("numa_node") + if device.get("locality") not in ("resolved", "single-node"): + if device.get("locality") == "outside-effective-mask": + return False, "its NUMA node is outside the effective host mask" + return False, "its NUMA-local node could not be resolved" + if ( + isinstance(node, bool) + or not isinstance(node, int) + or node < 0 + ): + return False, "its NUMA-local node is invalid" + effective_nodes = set(hardware.get("effective_nodes") or []) + if node not in effective_nodes: + return False, "its NUMA node is outside the effective host mask" + return True, None + + +def eligible_gpu_devices(hardware): + """Return eligible physical NVIDIA devices in stable index order.""" + devices = [] + for device in hardware.get("gpus") or []: + eligible, _reason = gpu_device_eligibility(device, hardware) + if eligible: + devices.append(device) + return sorted(devices, key=lambda item: int(item["index"])) + + +def _parse_gpu_selector(selector): + if isinstance(selector, str): + value = selector.strip().lower() + if value in ("auto", "none"): + return value + if not value or len(value) > 4096: + raise RamdiskError( + "--gpu must be auto, none, or a device list such as 0,1" + ) + selected = [] + for token in value.split(","): + token = token.strip() + match = re.fullmatch(r"(\d+)(?:-(\d+))?", token) + if not match: + raise RamdiskError( + "--gpu must be auto, none, or a device list such as 0,1" + ) + start = int(match.group(1)) + end = int(match.group(2) or start) + if end < start: + raise RamdiskError("--gpu contains a descending range") + if end > 65535: + raise RamdiskError("--gpu contains an unreasonable device index") + selected.extend(range(start, end + 1)) + elif isinstance(selector, (list, tuple, set)): + if any( + isinstance(index, bool) + or not isinstance(index, int) + or index < 0 + for index in selector + ): + raise RamdiskError( + "--gpu device lists must contain non-negative integers" + ) + selected = list(selector) + else: + raise RamdiskError( + "--gpu must be auto, none, or a device list such as 0,1" + ) + selected = sorted(set(selected)) + if not selected: + raise RamdiskError( + "--gpu device list is empty; use --gpu none for CPU mode" + ) + return selected + + +def _selected_gpu_devices(selector, hardware): + parsed = _parse_gpu_selector(selector) + if parsed == "none": + return [] + eligible = eligible_gpu_devices(hardware) + if parsed == "auto": + if not eligible: + discovery = hardware.get("gpu_discovery") or {} + detail = ( + discovery.get("selection_error") + or discovery.get("error") + ) + raise RamdiskError( + detail or "no usable NVIDIA GPU was detected" + ) + return eligible + discovered = { + int(device["index"]): device + for device in hardware.get("gpus") or [] + if isinstance(device, dict) + and isinstance(device.get("index"), int) + and not isinstance(device.get("index"), bool) + } + devices = [] + for index in parsed: + device = discovered.get(index) + if device is None: + raise RamdiskError( + "--gpu selects NVIDIA device %d, which was not discovered" + % index + ) + eligible_device, reason = gpu_device_eligibility(device, hardware) + if not eligible_device: + raise RamdiskError( + "--gpu selects unusable NVIDIA device %d: %s" + % (index, reason) + ) + devices.append(device) + return devices + + +def _gpu_local_placement(hardware, devices): + nodes = sorted(set(int(device["numa_node"]) for device in devices)) + rows = { + int(row["id"]): row + for row in hardware.get("nodes") or [] + if isinstance(row, dict) + and isinstance(row.get("id"), int) + and not isinstance(row.get("id"), bool) + } + effective_cpus = set(hardware.get("effective_cpus") or []) + cpus = set() + for node in nodes: + row = rows.get(node, {}) + node_cpus = ( + row.get("effective_cpus", []) + if "effective_cpus" in row + else row.get("cpus", []) + ) + cpus.update( + int(cpu) + for cpu in node_cpus + if isinstance(cpu, int) + and not isinstance(cpu, bool) + and (not effective_cpus or cpu in effective_cpus) + ) + if not cpus: + raise RamdiskError( + "GPU-local NUMA nodes expose no complete effective CPU cores" + ) + return _format_range_list(nodes), _format_range_list(sorted(cpus)) + + +def apply_gpu_selection( + args, + hardware, + selector=None, + layout=None, + *, + cuda_capable=None, + reset_placement=True, +): + """Apply one exact GPU selection and high-level layout to draft arguments.""" + if selector is None: + selector = getattr(args, "gpu", None) + if selector is None: + selector = "auto" + previous_accelerator = getattr(args, "managed_accelerator", None) or {} + parsed = _parse_gpu_selector(selector) + layout = _normalize_gpu_layout( + layout + if layout is not None + else getattr(args, "gpu_layout", None) + ) + if parsed == "none": + if layout != GPU_LAYOUT_EXPERTS_ONLY: + raise RamdiskError( + "%s requires one or more selected GPUs" % layout + ) + # Commit only after every validation succeeds. Frontends keep one + # mutable draft Namespace, so a rejected edit must leave it untouched. + args.gpu_layout = layout + args.gpu = "none" + args.managed_accelerator = None + if reset_placement: + args.memory_nodes = None + args.cpu_list = None + return args + + devices = _selected_gpu_devices(parsed, hardware) + if ( + layout == GPU_LAYOUT_DENSE_ATTENTION_SHARDED + and len(devices) < 2 + ): + raise RamdiskError( + "dense-attention-sharded requires at least two selected GPUs" + ) + memory_nodes = getattr(args, "memory_nodes", None) + cpu_list = getattr(args, "cpu_list", None) + if reset_placement: + memory_nodes, cpu_list = _gpu_local_placement( + hardware, + devices, + ) + gpu = ( + "auto" + if parsed == "auto" + else ",".join(str(device["index"]) for device in devices) + ) + managed_accelerator = { + "mode": "cuda", + "layout": layout, + "devices": [ + { + "index": int(device["index"]), + "cuda_ordinal": ordinal, + "name": str(device.get("name") or ""), + "uuid": str(device.get("uuid") or ""), + "pci_bus_id": str(device.get("pci_bus_id") or ""), + "numa_node": int(device["numa_node"]), + } + for ordinal, device in enumerate(devices) + ], + "mmap": True, + "rammap": False, + "async_copy": True, + "vram_budget": "auto", + "capability": ( + "available" + if ( + cuda_capable is True + or ( + cuda_capable is None + and previous_accelerator.get("capability") == "available" + ) + ) + else "unverified" + ), + } + args.gpu_layout = layout + if reset_placement: + args.memory_nodes = memory_nodes + args.cpu_list = cpu_list + args.topology = "interleaved" + args.gpu = gpu + args.managed_accelerator = managed_accelerator + return args + + +def _same_gpu_identity(expected, observed): + """Compare UUIDs when both are known, otherwise compare PCI identities.""" + expected_uuid = str(expected.get("uuid") or "") + observed_uuid = str(observed.get("uuid") or "") + if expected_uuid and observed_uuid: + return expected_uuid == observed_uuid + return ( + bool(expected.get("pci_bus_id")) + and expected.get("pci_bus_id") == observed.get("pci_bus_id") + ) + + +def _managed_accelerator_contract(plan): + contract = plan.get("managed_accelerator") + if contract is None: + return { + "mode": "cpu", + "layout": GPU_LAYOUT_EXPERTS_ONLY, + "devices": [], + "mmap": False, + "rammap": True, + "async_copy": False, + "vram_budget": None, + "capability": "legacy", + } + if not isinstance(contract, dict): + raise RamdiskError("managed accelerator plan is malformed") + mode = contract.get("mode") + if mode == "cpu": + if contract.get("devices") not in (None, []): + raise RamdiskError("CPU accelerator plan cannot select GPUs") + if _normalize_gpu_layout(contract.get("layout")) != ( + GPU_LAYOUT_EXPERTS_ONLY + ): + raise RamdiskError("CPU accelerator plan cannot use a GPU layout") + return { + "mode": "cpu", + "layout": GPU_LAYOUT_EXPERTS_ONLY, + "devices": [], + "mmap": False, + "rammap": True, + "async_copy": False, + "vram_budget": None, + "capability": str( + contract.get("capability") or "not-requested" + ), + } + if mode != "cuda": + raise RamdiskError("managed accelerator mode is invalid") + layout = _normalize_gpu_layout(contract.get("layout")) + devices = contract.get("devices") + if not isinstance(devices, list) or not devices: + raise RamdiskError("managed CUDA plan has no devices") + normalized = [] + seen = set() + ordinals = [] + for device in devices: + if not isinstance(device, dict): + raise RamdiskError("managed CUDA device record is malformed") + index = device.get("index") + node = device.get("numa_node") + uuid = device.get("uuid") + if ( + isinstance(index, bool) + or not isinstance(index, int) + or index < 0 + or index in seen + or isinstance(node, bool) + or not isinstance(node, int) + or node < 0 + or not isinstance(device.get("pci_bus_id"), str) + or not device["pci_bus_id"] + or (uuid is not None and not isinstance(uuid, str)) + ): + raise RamdiskError("managed CUDA device identity is invalid") + seen.add(index) + cuda_ordinal = device.get("cuda_ordinal") + if ( + cuda_ordinal is not None + and ( + isinstance(cuda_ordinal, bool) + or not isinstance(cuda_ordinal, int) + or cuda_ordinal < 0 + ) + ): + raise RamdiskError("managed CUDA ordinal mapping is invalid") + ordinals.append(cuda_ordinal) + normalized.append( + { + "index": index, + **( + {"cuda_ordinal": cuda_ordinal} + if cuda_ordinal is not None + else {} + ), + "name": str(device.get("name") or ""), + "uuid": str(uuid or ""), + "pci_bus_id": device["pci_bus_id"], + "numa_node": node, + } + ) + if any(ordinal is not None for ordinal in ordinals): + if ( + any(ordinal is None for ordinal in ordinals) + or ordinals != list(range(len(normalized))) + or any(not device["uuid"] for device in normalized) + ): + raise RamdiskError("managed CUDA ordinal mapping is invalid") + if ( + contract.get("mmap") is not True + or contract.get("rammap") is not False + or contract.get("vram_budget") != "auto" + ): + raise RamdiskError("managed CUDA memory contract is invalid") + if ( + layout == GPU_LAYOUT_DENSE_ATTENTION_SHARDED + and len(normalized) < 2 + ): + raise RamdiskError( + "dense-attention-sharded requires at least two selected GPUs" + ) + return { + "mode": "cuda", + "layout": layout, + "devices": normalized, + "mmap": True, + "rammap": False, + "async_copy": bool(contract.get("async_copy", True)), + "vram_budget": "auto", + "capability": str(contract.get("capability") or "unverified"), + } + + +def _managed_accelerator_environment(plan): + """Return the exact sanitized accelerator variables for one plan.""" + contract = _managed_accelerator_contract(plan) + if contract["mode"] == "cpu": + return { + "COLI_CUDA": "0", + "CUDA_DENSE": "0", + "COLI_CUDA_ATTN": "0", + "COLI_CUDA_ATTN_SHARD": "0", + "DRAFT": "0", + "COLI_MMAP": "0", + "COLI_RAMMAP": "1", + } + # Pin the process to reviewed physical identities. CUDA renumbers this + # visible list, and the persisted cuda_ordinal field gives telemetry a + # lossless way to map logical ordinals back to physical cards. A legacy + # manifest cannot safely assume NVML physical indices are CUDA ordinals. + logical_mapping = all( + "cuda_ordinal" in device for device in contract["devices"] + ) + if not logical_mapping: + raise RamdiskError( + "legacy managed CUDA plan has no safe ordinal mapping; " + "stop, destroy, and prepare the workspace again" + ) + indices = [ + str(device["cuda_ordinal"]) + for device in contract["devices"] + ] + environment = { + "COLI_CUDA": "1", + "CUDA_EXPERT_GB": "auto", + "CUDA_RESERVE_GB": "%.9f" % ( + GPU_VRAM_RESERVE_BYTES / 1e9 + ), + "COLI_CUDA_ASYNC": "1" if contract["async_copy"] else "0", + "DRAFT": "0", + "COLI_MMAP": "1", + "COLI_RAMMAP": "0", + # The VRAM tier is populated by pin_load. PIN_FILL lets a compatible + # history fill otherwise-unused VRAM after its measured hot prefix. + "PIN": "auto", + "PIN_GB": "all", + "PIN_FILL": "1", + "REPIN": "16", + } + environment["CUDA_VISIBLE_DEVICES"] = ",".join( + device["uuid"] for device in contract["devices"] + ) + environment.update(_GPU_LAYOUT_ENVIRONMENT[contract["layout"]]) + if len(indices) == 1: + environment["COLI_GPU"] = indices[0] + else: + environment["COLI_GPUS"] = ",".join(indices) + return environment + + +def _apply_managed_accelerator_environment(environment, plan): + contract = _managed_accelerator_contract(plan) + if ( + contract["mode"] == "cuda" + and "CUDA_VISIBLE_DEVICES" in environment + ): + raise RamdiskError( + "ambient CUDA_VISIBLE_DEVICES prevents safe physical GPU " + "selection; relaunch with it unset" + ) + for key in tuple(environment): + if ( + key in ACCELERATOR_ENVIRONMENT_KEYS + or key.startswith(ACCELERATOR_ENVIRONMENT_PREFIXES) + ): + environment.pop(key, None) + applied = _managed_accelerator_environment( + {"managed_accelerator": contract} + ) + environment.update(applied) + return applied diff --git a/c/ramdisk_support/cli.py b/c/ramdisk_support/cli.py new file mode 100644 index 000000000..00b470b63 --- /dev/null +++ b/c/ramdisk_support/cli.py @@ -0,0 +1,517 @@ +"""Scriptable CLI orchestration and lazy terminal-frontend selection.""" + +from __future__ import print_function + +import argparse +import contextlib +import json +import os +import signal +import subprocess +import sys +import threading + +from .accelerator import GPU_LAYOUT_CHOICES, GPU_LAYOUT_EXPERTS_ONLY +from .common import DEFAULT_MOUNT_ROOT, RamdiskError + + +def _add_lifecycle_options(parser, suppress=False): + default = argparse.SUPPRESS if suppress else None + # ``coli`` already supplies --model/--ctx on the outer ramdisk parser via + # its common parent. Add them only when this module is used standalone; + # the action-local parser always receives suppressing copies so the same + # options can also appear after ``plan``/``prepare`` without overwriting a + # value parsed before the action. + if "--model" not in parser._option_string_actions: + parser.add_argument( + "--model", + default=default, + help="canonical model directory on durable storage", + ) + parser.add_argument( + "--mode", + choices=("full", "partial"), + default=argparse.SUPPRESS if suppress else "full", + help="stage the full model or profile-selected shard closures", + ) + parser.add_argument( + "--topology", + choices=("interleaved", "per-node"), + default=argparse.SUPPRESS if suppress else "interleaved", + help=( + "interleaved = one shared model copy and one engine; " + "per-node = one complete copy and independent engine per " + "NUMA node (replication, not sharding)" + ), + ) + parser.add_argument( + "--memory-nodes", + default=default, + metavar="NODELIST", + help=( + "effective NUMA memory nodes (for example 0-3,8); " + "defaults to allowed CPU-bearing nodes" + ), + ) + parser.add_argument( + "--cpu-list", + default=default, + metavar="CPULIST", + help=( + "whole-core managed-engine CPUs (for example 0-15,32-47); " + "defaults to allowed CPUs on the selected memory nodes" + ), + ) + parser.add_argument( + "--capacity-gb", + type=float, + default=default, + help="per-copy staging budget; required for partial mode", + ) + parser.add_argument( + "--profile", + default=default, + help="compatible .coli_usage text or JSON profile", + ) + parser.add_argument( + "--mount-root", + default=argparse.SUPPRESS if suppress else DEFAULT_MOUNT_ROOT, + help="managed tmpfs root below /mnt", + ) + parser.add_argument( + "--thp", + choices=("auto", "within_size", "advise"), + default=argparse.SUPPRESS if suppress else "auto", + help="transparent huge-page policy for tmpfs", + ) + parser.add_argument( + "--allow-swappable", + action="store_true", + default=argparse.SUPPRESS if suppress else False, + help="allow tmpfs without noswap support", + ) + parser.add_argument( + "--prefault", + type=int, + choices=(0, 1), + default=default, + help="touch direct mappings at engine startup", + ) + parser.add_argument( + "--parallel", + type=int, + default=argparse.SUPPRESS if suppress else 2, + help="concurrent shard-copy workers (does not create replicas)", + ) + if "--ctx" not in parser._option_string_actions: + parser.add_argument( + "--ctx", + type=int, + default=argparse.SUPPRESS if suppress else 0, + help="managed engine context length (0 = 4096)", + ) + if "--gpu" not in parser._option_string_actions: + parser.add_argument( + "--gpu", + default=default, + help="auto, none, or an exact device list such as 0,1", + ) + if "--gpu-layout" not in parser._option_string_actions: + parser.add_argument( + "--gpu-layout", + choices=GPU_LAYOUT_CHOICES, + default=( + argparse.SUPPRESS + if suppress + else GPU_LAYOUT_EXPERTS_ONLY + ), + help=( + "experts-only (stable), dense-attention, or " + "dense-attention-sharded (experimental)" + ), + ) + + +def configure_parser(parser, common_parent=None): + """Attach scriptable subcommands; options work before or after the action.""" + _add_lifecycle_options(parser, suppress=False) + after = argparse.ArgumentParser( + add_help=False, + argument_default=argparse.SUPPRESS, + ) + _add_lifecycle_options(after, suppress=True) + actions = parser.add_subparsers( + dest="ramdisk_action", + metavar="ACTION", + ) + plan = actions.add_parser( + "plan", + parents=[after], + help="show an exact staging and reserve plan", + ) + plan.add_argument("--json", action="store_true") + prepare_parser = actions.add_parser( + "prepare", + parents=[after], + help="mount, stage, and validate weights", + ) + prepare_parser.add_argument( + "--yes", + action="store_true", + help="accept the reviewed plan non-interactively", + ) + status_parser = actions.add_parser( + "status", + parents=[after], + help="show mounts and managed processes", + ) + status_parser.add_argument("--json", action="store_true") + benchmark_parser = actions.add_parser( + "benchmark", + parents=[after], + help="run equal RAM/SSD scorecards", + ) + benchmark_parser.add_argument("--json", action="store_true") + start_parser = actions.add_parser( + "start", + parents=[after], + help="start managed engine process(es)", + ) + start_parser.add_argument( + "--base-port", + type=int, + default=None, + help=( + "managed base port " + "(defaults to the prepared deployment's last value)" + ), + ) + actions.add_parser( + "stop", + parents=[after], + help="stop only verified managed processes", + ) + destroy_parser = actions.add_parser( + "destroy", + parents=[after], + help="unmount volatile weights safely", + ) + destroy_parser.add_argument("--yes", action="store_true") + + +def _json_print(value): + print(json.dumps(value, indent=2, sort_keys=True)) + + +@contextlib.contextmanager +def _interruptible_confirmation(): + """Let terminal Ctrl-C interrupt a blocking confirmation read.""" + sigint = getattr(signal, "SIGINT", None) + previous = None + installed = False + if ( + sigint is not None + and threading.current_thread() is threading.main_thread() + ): + try: + previous = signal.getsignal(sigint) + signal.signal(sigint, signal.default_int_handler) + installed = True + except (OSError, ValueError): + pass + try: + yield + finally: + if installed: + try: + signal.signal(sigint, previous) + except (OSError, ValueError): + pass + + +def _confirm(message, accepted=False): + if accepted: + return + if not (sys.stdin.isatty() and sys.stdout.isatty()): + raise RamdiskError( + message + + "; rerun with --yes after reviewing `ramdisk plan`" + ) + with _interruptible_confirmation(): + answer = input(message + " [y/N] ").strip().lower() + if answer not in ("y", "yes"): + raise RamdiskError("cancelled") + + +@contextlib.contextmanager +def _cli_termination_guard(cancelable): + """Translate service/SSH termination into lifecycle-safe checkpoints. + + Prepare, Start, and Benchmark receive a cooperative cancellation event. + Stop and Destroy deliberately finish their verified cleanup transaction + before the CLI reports the deferred signal exit code. + """ + state = { + "cancel_event": threading.Event(), + "signum": None, + } + previous = {} + if threading.current_thread() is threading.main_thread(): + for name in ("SIGINT", "SIGHUP", "SIGTERM"): + signum = getattr(signal, name, None) + if signum is None: + continue + try: + previous[signum] = signal.getsignal(signum) + except (OSError, ValueError): + continue + + def request_termination(signum, _frame): + if state["signum"] is None: + state["signum"] = int(signum) + if cancelable: + state["cancel_event"].set() + + for signum in tuple(previous): + try: + signal.signal(signum, request_termination) + except (OSError, ValueError): + previous.pop(signum, None) + try: + yield state + finally: + for signum, handler in previous.items(): + try: + signal.signal(signum, handler) + except (OSError, ValueError): + pass + + +def _cli_exit_after_signal(termination, normal_code): + if termination is not None and termination.get("signum") is not None: + return 128 + int(termination["signum"]) + return normal_code + + +def dispatch( + args, + cli_path=None, + engine_path=None, + system=None, + *, + build_plan, + prepare, + status, + benchmark, + start, + stop, + destroy, + human_plan, + human_status, + human_benchmark, + json_print=None, + termination_guard=None, +): + """Route one parsed action through explicitly supplied application seams.""" + emit_json = _json_print if json_print is None else json_print + guard = ( + _cli_termination_guard + if termination_guard is None + else termination_guard + ) + action = getattr(args, "ramdisk_action", None) + termination = None + try: + if action == "plan": + value = build_plan(args) + if getattr(args, "json", False): + emit_json(value) + else: + human_plan(value) + return 2 if value["blockers"] else 0 + if action == "prepare": + with guard(True) as termination: + value = prepare( + args, + cancel_event=termination["cancel_event"], + ) + print( + "RAM-disk ready: %s" + % ", ".join(record["path"] for record in value["mounts"]) + ) + return _cli_exit_after_signal(termination, 0) + if action == "status": + value = status() + if getattr(args, "json", False): + emit_json(value) + else: + human_status(value) + return 0 + if action == "benchmark": + with guard(True) as termination: + value = benchmark( + args, + cli_path=cli_path, + engine_path=engine_path, + cancel_event=termination["cancel_event"], + ) + if getattr(args, "json", False): + emit_json(value) + else: + human_benchmark(value) + return _cli_exit_after_signal(termination, 0) + if action == "start": + with guard(True) as termination: + value = start( + args, + cli_path=cli_path, + engine_path=engine_path, + cancel_event=termination["cancel_event"], + ) + print( + "managed engine ports: %s" + % ", ".join(str(port) for port in value["ports"]) + ) + return _cli_exit_after_signal(termination, 0) + if action == "stop": + with guard(False) as termination: + value = stop(args) + if value.get("state") == "error": + print( + "managed engine cleanup completed, but the RAM " + "workspace is incomplete; review `coli ramdisk status`, " + "then run destroy", + file=sys.stderr, + ) + return _cli_exit_after_signal(termination, 2) + print("managed engines stopped; usage deltas merged") + return _cli_exit_after_signal(termination, 0) + if action == "destroy": + with guard(False) as termination: + value = destroy(args) + print( + "RAM-disk destroyed; durable state and benchmark history " + "preserved" + ) + return _cli_exit_after_signal(termination, 0) + raise RamdiskError( + "choose a ramdisk action or run the interactive TUI" + ) + except (RamdiskError, OSError, subprocess.SubprocessError) as exc: + if getattr(args, "json", False): + emit_json( + { + "schema": "colibri.ramdisk.error.v1", + "version": 1, + "error": str(exc), + } + ) + else: + print("coli ramdisk: %s" % exc, file=sys.stderr) + return _cli_exit_after_signal(termination, 2) + + +def _load_textual_frontend(): + """Import the optional frontend only when terminal routing selects it.""" + import ramdisk_textual + + return ramdisk_textual + + +def _textual_dependency_missing(error): + missing = getattr(error, "name", "") or "" + return missing == "textual" or missing.startswith("textual.") + + +def launch_tui( + args, + cli_path=None, + engine_path=None, + system=None, + *, + lifecycle, + run_tui_frontend, + legacy_tui=None, + curses_termination_guard=None, + finish_frontend=None, + load_textual_frontend=None, + curses_wrapper=None, + target_platform=None, + environment=None, +): + """Select Textual or curses without importing either frontend eagerly.""" + platform_name = ( + sys.platform if target_platform is None else target_platform + ) + if not platform_name.startswith("linux"): + print( + "coli ramdisk: the TUI is supported only on Linux", + file=sys.stderr, + ) + return 2 + + environment = os.environ if environment is None else environment + requested_ui = environment.get( + "COLI_RAMDISK_UI", + "auto", + ).strip().lower() + if requested_ui not in ("auto", "textual", "curses"): + print( + "coli ramdisk: COLI_RAMDISK_UI must be auto, textual, or curses", + file=sys.stderr, + ) + return 2 + + loader = ( + _load_textual_frontend + if load_textual_frontend is None + else load_textual_frontend + ) + textual_frontend = None + if requested_ui in ("auto", "textual"): + try: + textual_frontend = loader() + except ModuleNotFoundError as exc: + if not _textual_dependency_missing(exc): + raise + if requested_ui == "textual": + print( + "coli ramdisk: Textual UI requested but Textual is not " + "installed; install the TUI dependency or set " + "COLI_RAMDISK_UI=curses", + file=sys.stderr, + ) + return 2 + + try: + if textual_frontend is not None: + return run_tui_frontend( + lambda: textual_frontend.launch_tui( + args, + cli_path=cli_path, + engine_path=engine_path, + lifecycle=lifecycle, + ) + ) + + if curses_wrapper is None: + import curses + + curses_wrapper = curses.wrapper + if legacy_tui is None or curses_termination_guard is None: + raise TypeError( + "curses routing requires legacy_tui and " + "curses_termination_guard callbacks" + ) + with curses_termination_guard(): + return run_tui_frontend( + lambda: curses_wrapper( + legacy_tui, + args, + cli_path, + engine_path, + ) + ) + finally: + if finish_frontend is not None: + finish_frontend() diff --git a/c/ramdisk_support/common.py b/c/ramdisk_support/common.py new file mode 100644 index 000000000..b8e63f93e --- /dev/null +++ b/c/ramdisk_support/common.py @@ -0,0 +1,175 @@ +"""Dependency-free constants, errors, and pure helpers for RAM-disk control.""" + +from __future__ import print_function + +import datetime +import math +import os +import re + + +MANIFEST_VERSION = 1 + +PLAN_SCHEMA = "colibri.ramdisk.plan.v1" + +STATUS_SCHEMA = "colibri.ramdisk.status.v1" + +BENCHMARK_SCHEMA = "colibri.ramdisk.benchmark.v1" + +DEFAULT_MOUNT_ROOT = "/mnt/colibri-ram" + +GIB = 1 << 30 + +MIB = 1 << 20 + +TMPFS_MAGIC = 0x01021994 + +PROFILE_LINE_RE = re.compile(r"^\s*(-?\d+)\s+(\d+)\s+(\d+)\s*$") + +USAGE_FORMAT_VERSION = 1 + +USAGE_MERGE_RE = re.compile(r"^# coli-ramdisk-merge ([0-9a-f]{32})$") + +class RamdiskError(RuntimeError): + """An expected, user-actionable lifecycle failure.""" + +class _OperationCancelled(RamdiskError): + """A cooperative cancellation that reached a clean lifecycle checkpoint.""" + +class _EngineCleanupError(RamdiskError): + """A benchmark engine may still be live, so no later variant may launch.""" + +class _MountHelperCompletedError(RamdiskError): + """The privileged mount helper completed and reported a failure.""" + + +def _usage_engine_id(name): + """Return route_trace.h's stable 32-bit FNV-1a engine identity.""" + value = 2166136261 + for byte in str(name).encode("utf-8"): + value = ((value ^ byte) * 16777619) & 0xFFFFFFFF + return value + + +def _usage_engine_name(model_type): + """Mirror the launcher's model-type to route_trace engine mapping.""" + normalized = str(model_type or "").lower() + if "inkling" in normalized: + return "inkling" + if "kimi" in normalized: + return "kimi_k3" + return "glm_moe_dsa" + + +def _validated_usage_header( + records, + source="usage history", + expected_dimensions=None, + expected_engine_id=None, +): + """Validate the two route_trace.h identified-history records.""" + dimensions = [] + formats = [] + for layer, second, third in records: + if layer == -1: + dimensions.append((int(second), int(third))) + elif layer == -2: + formats.append((int(second), int(third))) + if not dimensions and not formats: + return None + if len(dimensions) != 1 or len(formats) != 1: + raise RamdiskError( + "%s must contain both usage headers exactly once" % source + ) + n_layers, n_experts = dimensions[0] + version, engine_id = formats[0] + if n_layers < 1 or n_experts < 1: + raise RamdiskError("%s has invalid history dimensions" % source) + if not 1 <= version <= USAGE_FORMAT_VERSION: + raise RamdiskError( + "%s uses unsupported usage format version %d" % (source, version) + ) + if not 1 <= engine_id <= 0xFFFFFFFF: + raise RamdiskError("%s has an invalid engine identity" % source) + if expected_dimensions is not None and ( + n_layers, + n_experts, + ) != tuple(expected_dimensions): + raise RamdiskError( + "%s dimensions %d x %d do not match the selected model's %d x %d" + % ( + source, + n_layers, + n_experts, + expected_dimensions[0], + expected_dimensions[1], + ) + ) + if expected_engine_id is not None and engine_id != expected_engine_id: + raise RamdiskError( + "%s engine identity does not match the selected model" % source + ) + return { + "n_layers": n_layers, + "n_experts": n_experts, + "format_version": version, + "engine_id": engine_id, + } + + +def _utc_now(): + return datetime.datetime.now(datetime.timezone.utc).isoformat() + +def _path_without_symlinks(path): + """True when no existing component redirects the reviewed absolute path.""" + return os.path.isabs(path) and os.path.realpath(path) == os.path.normpath(path) + +def _path_is_below(path, parent, allow_equal=False): + try: + normalized = os.path.normpath(os.path.abspath(path)) + root = os.path.normpath(os.path.abspath(parent)) + return os.path.commonpath([normalized, root]) == root and (allow_equal or normalized != root) + except ValueError: + return False + +def _positive_int(value): + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + +def _parse_range_list(value): + result = [] + for item in value.strip().split(","): + if not item: + continue + if "-" in item: + left, right = item.split("-", 1) + start, end = int(left), int(right) + if end < start: + raise ValueError("descending range") + result.extend(range(start, end + 1)) + else: + result.append(int(item)) + return sorted(set(result)) + +def _format_range_list(values): + values = sorted(set(int(value) for value in values)) + groups = [] + for value in values: + if not groups or value != groups[-1][1] + 1: + groups.append([value, value]) + else: + groups[-1][1] = value + return ",".join( + str(start) if start == end else "%d-%d" % (start, end) + for start, end in groups + ) + +def _raise_if_cancelled(cancel_event): + if cancel_event is not None and cancel_event.is_set(): + raise _OperationCancelled("operation cancelled by user at a safe checkpoint") + +def _percentile(values, percentile): + if not values: + return None + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, int(math.ceil(percentile * len(ordered))) - 1)) + return ordered[index] diff --git a/c/ramdisk_support/discovery.py b/c/ramdisk_support/discovery.py new file mode 100644 index 000000000..68d92db33 --- /dev/null +++ b/c/ramdisk_support/discovery.py @@ -0,0 +1,705 @@ +"""Normalized Linux hardware and cgroup discovery.""" + +from __future__ import print_function + +import csv +import io +import os +import posixpath +import re +import subprocess + +from .common import RamdiskError, _format_range_list, _parse_range_list +from .platform_ops import get_platform_ops + + +def _normalize_pci_bus_id(value): + """Return Linux's canonical PCI domain:bus:device.function spelling.""" + match = re.fullmatch( + r"\s*([0-9A-Fa-f]{1,8}):([0-9A-Fa-f]{1,2}):" + r"([0-9A-Fa-f]{1,2})\.([0-7])\s*", + str(value), + ) + if not match: + return None + domain, bus, device, function = ( + int(item, 16) for item in match.groups() + ) + if domain > 0xFFFF or bus > 0xFF or device > 0x1F: + return None + return "%04x:%02x:%02x.%x" % ( + domain, + bus, + device, + function, + ) + + +def _discover_gpus( + effective_nodes, + *, + ops=None, + run=None, + environ=None, +): + """Discover NVIDIA devices and resolve their PCI-local Linux NUMA nodes.""" + ops = get_platform_ops() if ops is None else ops + environ = os.environ if environ is None else environ + visibility_present = "CUDA_VISIBLE_DEVICES" in environ + visibility_value = ( + str(environ.get("CUDA_VISIBLE_DEVICES") or "") + if visibility_present + else None + ) + visibility = { + "cuda_visible_devices_present": visibility_present, + "cuda_visible_devices": visibility_value, + "selection_error": ( + "ambient CUDA_VISIBLE_DEVICES prevents safe physical GPU " + "selection; relaunch the RAM TUI with it unset" + if visibility_present + else None + ), + } + if not ops.is_linux: + return { + "status": "unsupported", + "error": "GPU NUMA discovery is supported only on Linux", + "devices": [], + **visibility, + } + run = subprocess.run if run is None else run + executable = ops.executable_path("nvidia-smi") + if not executable: + return { + "status": "unavailable", + "error": "nvidia-smi is not available", + "devices": [], + **visibility, + } + command = [ + executable, + "--query-gpu=index,name,uuid,pci.bus_id,memory.total,memory.free", + "--format=csv,noheader,nounits", + ] + try: + result = run( + command, + text=True, + capture_output=True, + check=False, + timeout=5, + ) + except (OSError, subprocess.SubprocessError) as exc: + return { + "status": "unavailable", + "error": "cannot query NVIDIA GPUs: %s" % exc, + "devices": [], + **visibility, + } + if result.returncode: + detail = (result.stderr or result.stdout or "").strip() + return { + "status": "unavailable", + "error": ( + "nvidia-smi GPU query failed" + + (": %s" % detail if detail else "") + ), + "devices": [], + **visibility, + } + + allowed = sorted(set(int(node) for node in effective_nodes)) + devices = [] + malformed = 0 + seen_indices = set() + seen_uuids = set() + for fields in csv.reader(io.StringIO(result.stdout or "")): + fields = [field.strip() for field in fields] + if len(fields) != 6: + malformed += 1 + continue + try: + index = int(fields[0]) + total_mib = int(fields[4]) + free_mib = int(fields[5]) + except ValueError: + malformed += 1 + continue + uuid = fields[2] + pci_bus_id = _normalize_pci_bus_id(fields[3]) + if ( + index < 0 + or index in seen_indices + or total_mib < 0 + or free_mib < 0 + or free_mib > total_mib + or not uuid + or uuid.lower() == "n/a" + or uuid in seen_uuids + or pci_bus_id is None + ): + malformed += 1 + continue + seen_indices.add(index) + seen_uuids.add(uuid) + raw_node = ops.read_text( + "/sys/bus/pci/devices/%s/numa_node" % pci_bus_id, + "", + ).strip() + try: + numa_node = int(raw_node) + except ValueError: + numa_node = None + if numa_node == -1 and len(allowed) == 1: + numa_node = allowed[0] + locality = "single-node" + elif numa_node is None or numa_node < 0: + numa_node = None + locality = "unknown" + elif numa_node not in allowed: + locality = "outside-effective-mask" + else: + locality = "resolved" + devices.append( + { + "index": index, + "name": fields[1], + "uuid": uuid, + "pci_bus_id": pci_bus_id, + "numa_node": numa_node, + "locality": locality, + "total_bytes": total_mib * 1024 * 1024, + "free_bytes": free_mib * 1024 * 1024, + } + ) + devices.sort(key=lambda item: item["index"]) + if devices: + status = "available" if not malformed else "partial" + error = ( + None + if not malformed + else "%d malformed nvidia-smi row(s) were ignored" % malformed + ) + elif malformed: + status = "unavailable" + error = "nvidia-smi returned no valid GPU rows" + else: + status = "none" + error = None + return { + "status": status, + "error": error, + "devices": devices, + **visibility, + } + + +def _mountinfo_unescape(value): + """Decode the octal escapes used for paths in ``/proc/*/mountinfo``.""" + return re.sub( + r"\\([0-7]{3})", + lambda match: chr(int(match.group(1), 8)), + value, + ) + + +def _cgroup_mounts(mountinfo): + """Return normalized cgroup mount records from one mountinfo snapshot.""" + records = [] + for line in mountinfo.splitlines(): + fields = line.split() + try: + separator = fields.index("-") + root = _mountinfo_unescape(fields[3]) + mountpoint = _mountinfo_unescape(fields[4]) + mount_options = fields[5].split(",") + filesystem = fields[separator + 1] + source = fields[separator + 2] + super_options = fields[separator + 3].split(",") + except (IndexError, ValueError): + continue + if filesystem not in ("cgroup", "cgroup2"): + continue + records.append( + { + "filesystem": filesystem, + "root": posixpath.normpath(root), + "mountpoint": posixpath.normpath(mountpoint), + "source": source, + "mount_options": mount_options, + "optional_fields": fields[6:separator], + "super_options": super_options, + } + ) + return records + + +def _cgroup_memberships(cgroup_text): + """Parse v1 controller paths and the v2 unified path.""" + memberships = {"v1": {}, "v2": None} + for line in cgroup_text.splitlines(): + fields = line.split(":", 2) + if len(fields) != 3: + continue + _, controllers, path = fields + if not path.startswith("/"): + continue + normalized = posixpath.normpath(path) + if not controllers: + memberships["v2"] = normalized + else: + for controller in controllers.split(","): + if controller: + memberships["v1"][controller] = normalized + return memberships + + +def _resolve_cgroup_directory(membership, mounts, filesystem, controller=None): + """Map a membership path to the most-specific visible cgroup mount.""" + candidates = [] + for mount in mounts: + if mount["filesystem"] != filesystem: + continue + controller_options = set(mount["super_options"]) | set( + mount["mount_options"] + ) | set(mount["source"].split(",")) | set(mount["optional_fields"]) + if controller is not None and controller not in controller_options: + continue + root = mount["root"] + explicit_root = membership == root or membership.startswith( + root.rstrip("/") + "/" + ) + relative = ( + posixpath.relpath(membership, root) + if explicit_root + else membership.lstrip("/") or "." + ) + resolved = posixpath.normpath( + posixpath.join(mount["mountpoint"], relative) + ) + try: + contained = ( + posixpath.commonpath([resolved, mount["mountpoint"]]) + == mount["mountpoint"] + ) + except ValueError: + contained = False + if contained: + candidates.append((int(explicit_root), len(root), mount, resolved)) + if not candidates: + return None, None + _, _, mount, resolved = max(candidates, key=lambda item: item[:2]) + return mount, resolved + + +def _cgroup_ancestors(path, mountpoint): + """Yield a cgroup and every visible ancestor through its mount root.""" + current = posixpath.normpath(path) + root = posixpath.normpath(mountpoint) + while True: + try: + if posixpath.commonpath([current, root]) != root: + raise ValueError("cgroup path escaped its mount") + except ValueError: + raise RamdiskError("resolved cgroup path is outside its controller mount") + yield current + if current == root: + break + parent = posixpath.dirname(current) + if parent == current: + raise RamdiskError("cgroup ancestry did not reach its controller mount") + current = parent + + +def _parse_cgroup_bytes(value, path, unlimited_word=False, v1_unlimited=False): + if value is None: + return None + if unlimited_word and value == "max": + return None + try: + parsed = int(value) + except (TypeError, ValueError): + raise RamdiskError("invalid cgroup memory value in %s" % path) + if parsed < 0: + raise RamdiskError("negative cgroup memory value in %s" % path) + if v1_unlimited and parsed >= (1 << 60): + return None + return parsed + + +def _empty_cgroup_memory(): + return { + "version": None, + "status": "none", + "path": None, + "mountpoint": None, + "limit_bytes": None, + "current_bytes": None, + "available_bytes": None, + "limiting_path": None, + "high_bytes": None, + "high_available_bytes": None, + "high_limiting_path": None, + "error": None, + } + + +def _discover_cgroup_memory_with_ops( + cgroup_text=None, + mountinfo_text=None, + ops=None, +): + """Return hard/high headroom across every limiting cgroup ancestor.""" + ops = get_platform_ops() if ops is None else ops + result = _empty_cgroup_memory() + if not ops.is_linux: + if cgroup_text is None or mountinfo_text is None: + return result + # Synthetic contracts are portable parser inputs. Use only the + # import-safe file helpers needed to inspect their fixture hierarchy. + from .linux_ops import LinuxPlatformOps + + ops = LinuxPlatformOps(ops.platform_name) + try: + if cgroup_text is None: + cgroup_text = ops.read_cgroup_contract("/proc/self/cgroup") + if mountinfo_text is None: + mountinfo_text = ops.read_cgroup_contract("/proc/self/mountinfo") + except RamdiskError as exc: + result.update({"status": "unavailable", "error": str(exc)}) + return result + memberships = _cgroup_memberships(cgroup_text) + mounts = _cgroup_mounts(mountinfo_text) + version = None + membership = None + mount = resolved = None + if memberships["v2"] is not None: + version = 2 + membership = memberships["v2"] + mount, resolved = _resolve_cgroup_directory( + membership, mounts, "cgroup2" + ) + v2_memory_visible = ( + mount is not None + and resolved is not None + and any( + ops.path_exists(posixpath.join(ancestor, leaf)) + for ancestor in _cgroup_ancestors(resolved, mount["mountpoint"]) + for leaf in ("memory.current", "memory.max", "memory.high") + ) + ) + if "memory" in memberships["v1"] and not v2_memory_visible: + v1_membership = memberships["v1"]["memory"] + v1_mount, v1_resolved = _resolve_cgroup_directory( + v1_membership, mounts, "cgroup", controller="memory" + ) + if v1_mount is not None and v1_resolved is not None: + version = 1 + membership = v1_membership + mount, resolved = v1_mount, v1_resolved + elif "memory" in memberships["v1"]: + version = 1 + membership = memberships["v1"]["memory"] + mount, resolved = _resolve_cgroup_directory( + membership, mounts, "cgroup", controller="memory" + ) + if version is None: + return result + result.update({"version": version, "path": membership}) + if mount is None or resolved is None: + result.update( + { + "status": "unavailable", + "error": "memory cgroup membership has no visible controller mount", + } + ) + return result + result["mountpoint"] = mount["mountpoint"] + try: + for ancestor in _cgroup_ancestors(resolved, mount["mountpoint"]): + if version == 2: + limit_path = posixpath.join(ancestor, "memory.max") + current_path = posixpath.join(ancestor, "memory.current") + high_path = posixpath.join(ancestor, "memory.high") + limit = _parse_cgroup_bytes( + ops.read_cgroup_value(limit_path), + limit_path, + unlimited_word=True, + ) + high = _parse_cgroup_bytes( + ops.read_cgroup_value(high_path), + high_path, + unlimited_word=True, + ) + else: + limit_path = posixpath.join( + ancestor, + "memory.limit_in_bytes", + ) + current_path = posixpath.join( + ancestor, + "memory.usage_in_bytes", + ) + limit = _parse_cgroup_bytes( + ops.read_cgroup_value(limit_path), + limit_path, + v1_unlimited=True, + ) + high = None + if limit is None and high is None: + continue + current = _parse_cgroup_bytes( + ops.read_cgroup_value(current_path), + current_path, + ) + if current is None: + raise RamdiskError( + "cgroup memory limit is visible but usage is unavailable at %s" + % ancestor + ) + if limit is not None: + available = max(0, limit - current) + if ( + result["available_bytes"] is None + or available < result["available_bytes"] + ): + result.update( + { + "limit_bytes": limit, + "current_bytes": current, + "available_bytes": available, + "limiting_path": ancestor, + } + ) + if high is not None: + high_available = max(0, high - current) + if ( + result["high_available_bytes"] is None + or high_available < result["high_available_bytes"] + ): + result.update( + { + "high_bytes": high, + "high_available_bytes": high_available, + "high_limiting_path": ancestor, + } + ) + result["status"] = ( + "limited" + if result["available_bytes"] is not None + or result["high_available_bytes"] is not None + else "unlimited" + ) + except RamdiskError as exc: + result.update({"status": "unavailable", "error": str(exc)}) + return result + + +def _discover_cgroup_memory(cgroup_text=None, mountinfo_text=None): + return _discover_cgroup_memory_with_ops( + cgroup_text=cgroup_text, + mountinfo_text=mountinfo_text, + ) + + +def _unsupported_hardware(ops): + cpus = list(range(ops.cpu_count())) + cpu_list = _format_range_list(cpus) + return { + "linux": False, + "capabilities": ops.capabilities(), + "kernel_release": ops.kernel_release(), + "online_nodes": [0], + "effective_nodes": [0], + "effective_cpus": cpus, + "effective_cpu_list": cpu_list, + "effective_mask_source": "portable-fallback", + "core_groups": [[cpu] for cpu in cpus], + "nodes": [ + { + "id": 0, + "cpus": cpus, + "cpu_list": cpu_list, + "physical_cores": len(cpus), + "memory_total_bytes": 0, + "memory_available_bytes": 0, + "distance": [], + "effective_cpus": cpus, + "effective_cpu_list": cpu_list, + } + ], + "physical_cores": len(cpus), + "effective_physical_cores": len(cpus), + "memory": {"total_bytes": 0, "available_bytes": 0}, + "cgroup_memory": _empty_cgroup_memory(), + "swap": {"configured": [], "used_bytes": 0}, + "tmpfs": {"supported": False, "noswap_supported": False}, + "thp": { + "shmem_enabled": "", + "modes": [], + "within_size_supported": False, + "advise_supported": False, + }, + "numactl": None, + "gpus": [], + "gpu_discovery": { + "status": "unsupported", + "error": "GPU NUMA discovery is supported only on Linux", + }, + "mount": None, + "umount": None, + "sudo": None, + "hugetlb": { + "total_pages": 0, + "free_pages": 0, + "page_size_bytes": 0, + }, + } + + +def discover_hardware(ops=None, gpu_discovery=None): + """Return normalized Linux discovery or explicit unsupported capabilities.""" + ops = get_platform_ops() if ops is None else ops + if not ops.is_linux: + return _unsupported_hardware(ops) + online_text = ops.read_text("/sys/devices/system/node/online", "0") + try: + online = _parse_range_list(online_text) + except ValueError: + online = [0] + if not online: + online = [0] + nodes = [] + all_cpus = [] + for node in online: + cpus_text = ops.read_text( + "/sys/devices/system/node/node%d/cpulist" % node, + ops.read_text("/sys/devices/system/cpu/online", "0"), + ) + try: + cpus = _parse_range_list(cpus_text) + except ValueError: + cpus = [] + all_cpus.extend(cpus) + memory = ops.node_meminfo(node) + distance = [] + for word in ops.read_text( + "/sys/devices/system/node/node%d/distance" % node + ).split(): + try: + distance.append(int(word)) + except ValueError: + pass + nodes.append( + { + "id": node, + "cpus": cpus, + "cpu_list": cpus_text.strip(), + "physical_cores": ops.physical_cores(cpus), + "memory_total_bytes": memory.get("MemTotal", 0), + "memory_available_bytes": memory.get( + "MemFree", memory.get("MemAvailable", 0) + ), + "distance": distance, + } + ) + all_cpus = sorted(set(all_cpus)) + affinity = ops.cpu_affinity() + if affinity is None: + affinity = ops.status_allowed_list("Cpus_allowed_list", all_cpus) + effective_cpus = sorted(set(affinity) & set(all_cpus)) + effective_nodes = sorted( + set(ops.status_allowed_list("Mems_allowed_list", online)) & set(online) + ) + core_groups = ops.thread_sibling_groups(effective_cpus) + for node in nodes: + node["effective_cpus"] = sorted( + set(node["cpus"]) & set(effective_cpus) + ) + node["effective_cpu_list"] = _format_range_list(node["effective_cpus"]) + memory = ops.meminfo() + swaps = [] + swap_text = ops.read_text("/proc/swaps") + for line in swap_text.splitlines()[1:]: + fields = line.split() + if len(fields) >= 5: + swaps.append( + { + "path": fields[0], + "type": fields[1], + "size_bytes": int(fields[2]) * 1024, + "used_bytes": int(fields[3]) * 1024, + } + ) + shmem_enabled = ops.read_text( + "/sys/kernel/mm/transparent_hugepage/shmem_enabled" + ).strip() + thp_modes = re.findall(r"\[?([A-Za-z_]+)\]?", shmem_enabled) + filesystems = ops.read_text("/proc/filesystems") + cgroup_memory = _discover_cgroup_memory_with_ops(ops=ops) + gpu_report = ( + _discover_gpus(effective_nodes, ops=ops) + if gpu_discovery is None + else gpu_discovery(effective_nodes, ops=ops) + ) + return { + "linux": True, + "capabilities": ops.capabilities(), + "kernel_release": ops.kernel_release(), + "online_nodes": online, + "effective_nodes": effective_nodes, + "effective_cpus": effective_cpus, + "effective_cpu_list": _format_range_list(effective_cpus), + "effective_mask_source": "kernel-task-status", + "core_groups": core_groups, + "nodes": nodes, + "physical_cores": ops.physical_cores(all_cpus), + "effective_physical_cores": len(core_groups), + "memory": { + "total_bytes": memory.get("MemTotal", 0), + "available_bytes": memory.get( + "MemAvailable", + memory.get("MemFree", 0), + ), + }, + "cgroup_memory": cgroup_memory, + "swap": { + "configured": swaps, + "used_bytes": sum(item["used_bytes"] for item in swaps), + }, + "tmpfs": { + "supported": any( + line.strip().endswith("tmpfs") + for line in filesystems.splitlines() + ), + "noswap_supported": ops.kernel_at_least(6, 4), + }, + "thp": { + "shmem_enabled": shmem_enabled, + "modes": sorted(set(thp_modes)), + "within_size_supported": "within_size" in thp_modes, + "advise_supported": "advise" in thp_modes or bool(shmem_enabled), + }, + "numactl": ops.executable_path("numactl"), + "gpus": list(gpu_report.get("devices") or []), + "gpu_discovery": { + "status": gpu_report.get("status", "unavailable"), + "error": gpu_report.get("error"), + "cuda_visible_devices_present": bool( + gpu_report.get("cuda_visible_devices_present") + ), + "cuda_visible_devices": gpu_report.get( + "cuda_visible_devices" + ), + "selection_error": gpu_report.get("selection_error"), + }, + "mount": ops.executable_path("mount"), + "umount": ops.executable_path("umount"), + "sudo": ops.executable_path("sudo"), + "hugetlb": { + "total_pages": memory.get("HugePages_Total", 0) // 1024, + "free_pages": memory.get("HugePages_Free", 0) // 1024, + "page_size_bytes": memory.get("Hugepagesize", 0), + }, + } diff --git a/c/ramdisk_support/lifecycle.py b/c/ramdisk_support/lifecycle.py new file mode 100644 index 000000000..b83094b46 --- /dev/null +++ b/c/ramdisk_support/lifecycle.py @@ -0,0 +1,3691 @@ +"""RAM-disk preparation, engine, teardown, and status orchestration.""" + +from __future__ import print_function + +import copy +import errno +import math +import os +import secrets +import socket +import subprocess +import sys +import threading +import time + +from .common import ( + MANIFEST_VERSION, + MIB, + STATUS_SCHEMA, + RamdiskError, + _MountHelperCompletedError, + _OperationCancelled, + _positive_int, + _raise_if_cancelled, + _utc_now, +) + + +_POPEN_BASE_TYPE = subprocess.Popen +_PENDING_RECOVERY_ERROR_PREFIX = "pending managed-launch recovery: " + + +def _managed_ports_for_plan(plan, base_port=8000): + return [ + int(base_port) + + (0 if mount.get("node") is None else int(mount["node"])) + for mount in plan["mounts"] + ] + + +def _persisted_base_port(manifest): + """Recover the last base port, including manifests predating that field.""" + explicit = manifest.get("base_port") + if ( + isinstance(explicit, int) + and not isinstance(explicit, bool) + and 1 <= explicit <= 65535 + ): + return explicit + + candidates = [] + for process in manifest.get("processes", []): + port = process.get("port") + node = process.get("node") + if isinstance(port, int) and not isinstance(port, bool): + candidates.append( + port - (0 if node is None else int(node)) + ) + if not candidates: + for mount, port in zip( + manifest.get("mounts", []), + manifest.get("ports", []), + ): + if isinstance(port, int) and not isinstance(port, bool): + node = mount.get("node") + candidates.append( + port - (0 if node is None else int(node)) + ) + if ( + candidates + and len(set(candidates)) == 1 + and 1 <= candidates[0] <= 65535 + ): + return candidates[0] + return 8000 + + +def _managed_path(path, mount_root): + normalized = os.path.normpath(os.path.abspath(path)) + root = os.path.normpath(os.path.abspath(mount_root)) + if root in ("/", "", os.path.expanduser("~")): + return False + return ( + normalized == root + or os.path.commonpath([normalized, root]) == root + ) + + +def _retained_process_recovery(manifest): + recovery = manifest.get("recovery") + if not isinstance(recovery, dict): + return [] + retained = recovery.get("retained_processes") + return retained if isinstance(retained, list) else [] + + +def _pending_launch_recovery(manifest): + pending = manifest.get("pending_launches") + return pending if isinstance(pending, list) else [] + + +def _unresolved_process_recovery_error(action): + return RamdiskError( + "refusing %s while unpublished managed-child absence is unproven; " + "run `coli ramdisk stop` to reconcile it after the process group " + "exits, then inspect recovery.retained_processes in status" + % action + ) + + +def _pending_launch_recovery_error(action): + return RamdiskError( + "refusing %s while a pre-spawn managed launch has an unknown " + "outcome; run `coli ramdisk stop` to discover, stop, and reconcile " + "the pending launch" % action + ) + + +def _valid_usage_transaction_id(value): + return ( + isinstance(value, str) + and len(value) == 32 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _usage_authority_records(manifest): + records = [] + records.extend( + record + for record in manifest.get("processes", []) + if isinstance(record, dict) + ) + records.extend( + record + for record in manifest.get("pending_launches", []) + if isinstance(record, dict) + ) + records.extend( + record + for record in _retained_process_recovery(manifest) + if isinstance(record, dict) + ) + return records + + +_RETAINED_AUTHORITY_VERSION = 1 + + +def _retained_process_authority(pending_entry): + """Persist the discovery authority needed to positively prove absence. + + A retained unpublished child was created (its direct Popen handle existed) + but never identity-published, so its original process group can disappear + while a nonce-attributed descendant survives in another session (for + example after ``setsid()``). Absence cannot be proven from the original + PGID alone; recovery must re-run the same global nonce-attribution scan used + for pending launches. Carry exactly that authority on the retained record. + """ + return { + "authority_version": _RETAINED_AUTHORITY_VERSION, + "nonce": pending_entry["nonce"], + "uid": pending_entry["uid"], + "weights_dir": pending_entry["weights_dir"], + "launch_not_before": pending_entry["launch_not_before"], + "launcher_pid": pending_entry["launcher_pid"], + "launcher_starttime": pending_entry["launcher_starttime"], + "launcher_cmdline": list(pending_entry["launcher_cmdline"]), + "expected_command": list(pending_entry["expected_command"]), + } + + +def _retained_authority(entry): + """Return validated discovery authority for a retained entry, or None. + + Legacy records created before discovery authority was persisted cannot be + safely reconciled: their absence cannot be positively established. Return + None so callers fail closed instead of guessing or signalling by PID. + """ + if entry.get("authority_version") != _RETAINED_AUTHORITY_VERSION: + return None + nonce = entry.get("nonce") + uid = entry.get("uid") + weights_dir = entry.get("weights_dir") + launch_not_before = entry.get("launch_not_before") + launcher_pid = entry.get("launcher_pid") + launcher_starttime = entry.get("launcher_starttime") + launcher_cmdline = entry.get("launcher_cmdline") + expected_command = entry.get("expected_command") + if not (isinstance(nonce, str) and nonce and "\0" not in nonce): + return None + if not isinstance(uid, int) or isinstance(uid, bool) or uid < 0: + return None + if ( + not isinstance(weights_dir, str) + or not weights_dir + or "\0" in weights_dir + ): + return None + if ( + not isinstance(launch_not_before, int) + or isinstance(launch_not_before, bool) + or launch_not_before < 0 + ): + return None + if ( + not isinstance(launcher_pid, int) + or isinstance(launcher_pid, bool) + or launcher_pid <= 0 + ): + return None + if ( + not isinstance(launcher_starttime, int) + or isinstance(launcher_starttime, bool) + or launcher_starttime <= 0 + ): + return None + for label, value in ( + ("launcher command", launcher_cmdline), + ("expected command", expected_command), + ): + if ( + not isinstance(value, list) + or not value + or any( + not isinstance(item, str) or not item + for item in value + ) + ): + return None + return { + "nonce": nonce, + "uid": uid, + "weights_dir": weights_dir, + "launch_not_before": launch_not_before, + "launcher_pid": launcher_pid, + "launcher_starttime": launcher_starttime, + "launcher_cmdline": list(launcher_cmdline), + "expected_command": list(expected_command), + } + + +def _retained_absence_failure( + entry, + *, + group_alive, + discover_managed_launches, +): + """Return a failure string unless process absence is positively established. + + Positive absence requires BOTH the original process group to be gone AND + zero nonce-attributable live processes anywhere on the system. A + descendant that re-sessioned (``setsid()``) evades the original-PGID check + but is still found by the global nonce scan, so the scan is mandatory and + is re-run here, immediately before any accounting merge. + """ + pid = entry.get("pgid", entry.get("pid")) + authority = _retained_authority(entry) + if authority is None: + return ( + "PID/PGID %s unpublished managed-child recovery lacks durable " + "discovery authority; absence is unproven" % pid + ) + try: + alive = group_alive(int(pid)) + except BaseException as exc: + return ( + "PID/PGID %s absence check failed during unpublished " + "recovery: %s" % (pid, exc) + ) + if alive is not False: + return ( + "PID/PGID %s unpublished managed process group is still " + "live or its absence is unproven" % pid + ) + try: + candidates = discover_managed_launches( + nonce=authority["nonce"], + uid=authority["uid"], + state_dir=entry.get("state_dir"), + weights_dir=authority["weights_dir"], + not_before_starttime=authority["launch_not_before"], + launcher_pid=authority["launcher_pid"], + launcher_starttime=authority["launcher_starttime"], + launcher_cmdline=authority["launcher_cmdline"], + expected_command=authority["expected_command"], + ) + except BaseException as exc: + return ( + "PID/PGID %s global nonce attribution failed during " + "unpublished recovery: %s" % (pid, exc) + ) + if not isinstance(candidates, list): + return ( + "PID/PGID %s global nonce attribution returned an invalid " + "result during unpublished recovery" % pid + ) + if candidates: + return ( + "PID/PGID %s unpublished managed process still has %d " + "nonce-attributable live process(es); absence is unproven" + % (pid, len(candidates)) + ) + return None + + +def _reserved_usage_transaction_ids(manifest): + owners = {} + for record in _usage_authority_records(manifest): + merge_id = record.get("usage_merge_id") + if merge_id is None: + continue + if not _valid_usage_transaction_id(merge_id): + raise RamdiskError("managed recovery has an invalid usage transaction") + if merge_id in owners and owners[merge_id] is not record: + raise RamdiskError( + "duplicate usage transaction authority: %s" % merge_id + ) + owners[merge_id] = record + return set(owners) + + +def _mint_usage_transaction_id(reserved_ids): + for _ in range(128): + merge_id = secrets.token_hex(16) + if merge_id not in reserved_ids: + reserved_ids.add(merge_id) + return merge_id + raise RamdiskError("could not allocate a unique usage transaction id") + + +def _bind_recovery_usage_transactions( + manifest, + records, + *, + plan, + bind_usage_transaction, + reserved_ids=None, +): + """Resolve every candidate before mutating any durable manifest record.""" + reserved = _reserved_usage_transaction_ids(manifest) + reserved.update(reserved_ids or ()) + resolved = [] + for record in records: + persisted = record.get("usage_merge_id") + local_reserved = set(reserved) + if persisted is not None: + local_reserved.discard(persisted) + candidate_record = copy.deepcopy(record) + merge_id = bind_usage_transaction( + candidate_record, + plan=plan, + reserved_ids=local_reserved, + ) + if not _valid_usage_transaction_id(merge_id): + raise RamdiskError("managed recovery has an invalid usage transaction") + if merge_id in local_reserved: + raise RamdiskError( + "duplicate usage transaction authority: %s" % merge_id + ) + if persisted is not None: + reserved.discard(persisted) + reserved.add(merge_id) + resolved.append((record, merge_id)) + for record, merge_id in resolved: + record["usage_merge_id"] = merge_id + return reserved + + +def _state_dir_authority_key(path): + return os.path.normcase(os.path.normpath(os.path.abspath(path))) + + +def _preflight_seed_usage_journals( + manifest, + state_dirs, + *, + plan, + usage_journal_transaction_id, +): + """Resolve every current seed journal before the first canonical replay.""" + reserved = _reserved_usage_transaction_ids(manifest) + manifest_ids_by_state = {} + for record in _usage_authority_records(manifest): + state_dir = record.get("state_dir") + if not isinstance(state_dir, str) or not state_dir: + continue + merge_id = record.get("usage_merge_id") + if merge_id is None: + continue + manifest_ids_by_state.setdefault( + _state_dir_authority_key(state_dir), + set(), + ).add(merge_id) + + expected_by_state = {} + journal_owner_by_id = {} + for state_dir in state_dirs: + state_key = _state_dir_authority_key(state_dir) + if state_key in expected_by_state: + continue + merge_id = usage_journal_transaction_id(state_dir, plan=plan) + if merge_id is None: + expected_by_state[state_key] = None + continue + if not _valid_usage_transaction_id(merge_id): + raise RamdiskError("managed recovery has an invalid usage transaction") + prior_owner = journal_owner_by_id.get(merge_id) + if prior_owner is not None and prior_owner != state_key: + raise RamdiskError( + "duplicate usage transaction authority: %s" % merge_id + ) + same_state_manifest_ids = manifest_ids_by_state.get(state_key, set()) + if same_state_manifest_ids and merge_id not in same_state_manifest_ids: + raise RamdiskError( + "usage delta journal transaction does not match managed record" + ) + if merge_id in reserved and merge_id not in same_state_manifest_ids: + raise RamdiskError( + "duplicate usage transaction authority: %s" % merge_id + ) + journal_owner_by_id[merge_id] = state_key + expected_by_state[state_key] = merge_id + reserved.add(merge_id) + + # Canonical markers are intentionally not included. They are historical, + # not live authorities; the 128-bit transaction space is the explicit + # collision assumption that permits an old marked journal to replay as a + # harmless idempotent retry. + return expected_by_state, reserved + + +def _preflight_pending_launches( + manifest, + *, + discover_managed_launches, + process_matches, + process_group_members, + group_alive, +): + """Resolve pending nonces without authorizing a signal or usage merge.""" + preflights = [] + failures = [] + for entry in _pending_launch_recovery(manifest): + label = "pending launch on port %s (node %s)" % ( + entry.get("port", "unknown"), + entry.get("node") if entry.get("node") is not None else "shared", + ) + try: + candidates = discover_managed_launches( + nonce=entry["nonce"], + uid=entry["uid"], + state_dir=entry["state_dir"], + weights_dir=entry["weights_dir"], + not_before_starttime=entry["launch_not_before"], + launcher_pid=entry["launcher_pid"], + launcher_starttime=entry["launcher_starttime"], + launcher_cmdline=entry["launcher_cmdline"], + expected_command=entry["expected_command"], + ) + except Exception as exc: + failures.append("%s discovery failed: %s" % (label, exc)) + continue + if not isinstance(candidates, list): + failures.append("%s discovery returned an invalid result" % label) + continue + + observed = entry.get("observed_group") + if not candidates: + if observed is not None: + try: + still_alive = group_alive(observed["pgid"]) + except Exception as exc: + failures.append( + "%s observed group absence check failed: %s" + % (label, exc) + ) + continue + if still_alive is not False: + failures.append( + "%s observed process group %s remains live or its " + "absence is unproven" + % (label, observed["pgid"]) + ) + continue + preflights.append( + { + "entry": entry, + "record": None, + "live": False, + "observed_group": observed, + } + ) + continue + + malformed = [ + candidate + for candidate in candidates + if ( + not isinstance(candidate, dict) + or not _positive_int(candidate.get("pid")) + or candidate.get("uid") != entry["uid"] + or candidate.get("nonce") != entry["nonce"] + or candidate.get("state_dir") != entry["state_dir"] + or candidate.get("weights_dir") != entry["weights_dir"] + or not _positive_int(candidate.get("starttime")) + or not _positive_int(candidate.get("pgid")) + or candidate.get("sid") != candidate.get("pgid") + ) + ] + if malformed: + failures.append( + "%s discovery returned malformed or mismatched identities" + % label + ) + continue + pgids = {candidate["pgid"] for candidate in candidates} + if len(pgids) != 1: + failures.append( + "%s nonce is attributable to multiple process groups: %s" + % (label, ", ".join(str(value) for value in sorted(pgids))) + ) + continue + pgid = next(iter(pgids)) + try: + members, unreadable = process_group_members(pgid) + except Exception as exc: + failures.append( + "%s process-group discovery failed: %s" % (label, exc) + ) + continue + if unreadable or not members: + failures.append( + "%s process group %s has unreadable or absent members" + % (label, pgid) + ) + continue + + leader = next( + ( + candidate + for candidate in candidates + if candidate["pid"] == pgid + ), + None, + ) + inert_leader = next( + ( + member + for member in members + if isinstance(member, dict) + and member.get("pid") == pgid + and member.get("inert") is True + ), + None, + ) + leader_identity = leader or inert_leader + leader_starttime = ( + leader_identity.get("starttime") + if leader_identity is not None + else None + ) + persisted_leader_starttime = ( + observed.get("leader_starttime") + if observed is not None + else leader_starttime + ) + next_observed = { + "pgid": pgid, + "uid": entry["uid"], + "leader_starttime": persisted_leader_starttime, + } + if observed is not None and ( + observed.get("pgid") != pgid + or observed.get("uid") != entry["uid"] + or ( + leader_identity is not None + and ( + observed.get("leader_starttime") is None + or observed.get("leader_starttime") != leader_starttime + ) + ) + ): + failures.append( + "%s observed process-group identity changed" % label + ) + continue + + def member_matches(member): + if ( + not isinstance(member, dict) + or not _positive_int(member.get("pid")) + or not _positive_int(member.get("starttime")) + ): + return False + if member.get("inert") is True: + if ( + member.get("uid") != entry["uid"] + or member.get("pgid") != pgid + or member.get("sid") != pgid + ): + return False + if member["pid"] == pgid: + return ( + member is inert_leader + and member["starttime"] + == persisted_leader_starttime + ) + return True + return ( + member.get("uid") == entry["uid"] + and member.get("nonce") == entry["nonce"] + and member.get("pgid") == pgid + and member.get("sid") == pgid + and member.get("state_dir") == entry["state_dir"] + and member.get("weights_dir") == entry["weights_dir"] + ) + + if any( + not member_matches(member) + for member in members + ): + failures.append( + "%s process group %s contains a foreign or mismatched member" + % (label, pgid) + ) + continue + + record = dict(entry) + record.update( + { + "pid": pgid, + "pgid": pgid, + "starttime": leader_starttime, + } + ) + try: + matches, reason, _ = process_matches(record) + except Exception as exc: + matches, reason = False, "identity-check-failed: %s" % exc + if not matches: + failures.append( + "%s process group %s is not safely attributable (%s)" + % (label, pgid, reason) + ) + continue + preflights.append( + { + "entry": entry, + "record": record, + "live": True, + "observed_group": next_observed, + } + ) + return preflights, failures + + +def _preflight_unpublished_processes( + manifest, + *, + group_alive, + discover_managed_launches, +): + """Prove every retained direct-created process positively absent. + + Absence is positive only when the original process group is gone AND the + global nonce scan finds zero attributable live processes. A re-sessioned + descendant evades the original-PGID check and is caught only by the scan, + so the scan is mandatory here. Nothing is mutated. + """ + failures = [] + for entry in _retained_process_recovery(manifest): + pid = entry.get("pgid", entry.get("pid")) + failure = _retained_absence_failure( + entry, + group_alive=group_alive, + discover_managed_launches=discover_managed_launches, + ) + baseline = entry.get("usage_baseline") + merge_id = entry.get("usage_merge_id") + if not isinstance(baseline, dict) or not _valid_usage_transaction_id( + merge_id + ): + failure = failure or ( + "PID/PGID %s unpublished recovery is missing exact durable " + "usage accounting metadata" % pid + ) + if failure: + failures.append(failure) + return failures + + +def _reconcile_unpublished_processes( + manifest, + *, + group_alive, + discover_managed_launches, + merge_usage, + save_manifest, +): + """Merge an unpublished child only after its absence is positively proven. + + Absence is re-proven globally immediately before each accounting merge: the + original process group must be gone AND the global nonce scan must find zero + attributable live processes. A re-sessioned descendant that evades the + original-PGID check keeps the entry retained instead of being merged. + """ + retained = list(_retained_process_recovery(manifest)) + if not retained: + return manifest + plan = manifest["plan"] + canonical_usage = os.path.join( + plan["model"]["path"], + ".coli_usage", + ) + preflight_failures = _preflight_unpublished_processes( + manifest, + group_alive=group_alive, + discover_managed_launches=discover_managed_launches, + ) + if preflight_failures: + recovery = manifest.setdefault("recovery", {}) + for entry in retained: + entry["error"] = ( + "retained process recovery did not pass global preflight" + ) + recovery["retained_processes"] = retained + recovery["state"] = "attention-required" + manifest["state"] = "error" + save_manifest(manifest) + raise RamdiskError( + "unpublished managed-child recovery is incomplete: " + + "; ".join(preflight_failures) + ) + failures = [] + remaining = [] + released = [] + for entry in retained: + pid = entry.get("pgid", entry.get("pid")) + # Revalidate immediately before merging accounting. Time has passed + # since preflight and other entries may have merged in between, so + # absence (original group gone AND zero attributable processes) must be + # re-proven right here. No numeric PID is ever signalled on refusal. + revalidation = _retained_absence_failure( + entry, + group_alive=group_alive, + discover_managed_launches=discover_managed_launches, + ) + if revalidation is not None: + retained_entry = dict(entry) + retained_entry["error"] = revalidation + remaining.append(retained_entry) + failures.append(revalidation) + continue + try: + merge_usage( + entry, + canonical_usage, + plan=plan, + ) + entry["usage_merged_at"] = _utc_now() + save_manifest(manifest) + except BaseException as exc: + failure = ( + "PID/PGID %s unpublished usage delta was not merged: %s" + % (pid, exc) + ) + retained_entry = dict(entry) + retained_entry["error"] = failure + remaining.append(retained_entry) + failures.append(failure) + continue + released.append( + { + "pid": entry.get("pid"), + "pgid": pid, + "state_dir": entry.get("state_dir"), + "usage_merged_at": entry.get("usage_merged_at"), + } + ) + + recovery = manifest.setdefault("recovery", {}) + recovery["retained_processes"] = remaining + recovery["released_processes"] = released + recovery["state"] = ( + "attention-required" if remaining else "reconciled" + ) + manifest["state"] = "error" + if failures: + manifest.setdefault("cleanup_errors", []).extend(failures) + save_manifest(manifest) + if failures: + raise RamdiskError( + "unpublished managed-child recovery is incomplete: " + + "; ".join(failures) + ) + return manifest + + +def _assert_ready_mounts( + manifest, + *, + source_still_matches, + validate_mount, + validate_namespace, +): + plan = manifest["plan"] + source_still_matches(plan) + for record in manifest.get("mounts", []): + if record.get("ownership", "managed") != "managed": + raise RamdiskError( + "mount ownership is still pending at %s" % record["path"] + ) + actual = validate_mount(record, plan) + expected = record.get("identity", {}) + if expected and ( + actual["mount_id"] != expected.get("mount_id") + or actual["device"] != expected.get("device") + ): + raise RamdiskError( + "mount identity changed at %s" % record["path"] + ) + validate_namespace(plan, record) + + +def _same_managed_mount(actual, expected): + """Return whether an observed mount is the exact recorded tmpfs.""" + return bool( + actual + and isinstance(expected, dict) + and actual.get("filesystem") == "tmpfs" + and actual.get("source") == "tmpfs" + and actual.get("mount_id") == expected.get("mount_id") + and actual.get("device") == expected.get("device") + ) + + +def _rollback_preparation_mounts( + manifest, + *, + mount_at, + mount_table, + path_is_below, + busy_mount_references, + umount_path, + validate_mount, +): + """Roll back only mounts whose exact persisted ownership is still valid.""" + plan = manifest["plan"] + cleanup_errors = [] + retained = set() + released = set() + candidates = [] + records = list(manifest.get("mounts", [])) + if not records: + return cleanup_errors, retained, released + + def retain(record, message): + path = record["path"] + cleanup_errors.append(message) + retained.add(path) + record["cleanup"] = { + "state": "retained", + "error": message, + } + + def nested_paths(table, path): + return sorted( + item["path"] + for item in table + if isinstance(item, dict) + and isinstance(item.get("path"), str) + and path_is_below(item["path"], path) + ) + + def inspect_candidate(record, table, phase): + path = record["path"] + try: + actual = mount_at(path) + except BaseException as exc: + return None, ( + "could not inspect mount %s at %s: %s" + % (path, phase, exc) + ) + if actual is None: + return "absent", None + if not _same_managed_mount(actual, record.get("identity")): + return None, ( + "refusing changed mount during preparation rollback at %s: %s" + % (phase, path) + ) + try: + verified = validate_mount(record, plan) + except BaseException as exc: + return None, ( + "could not validate mount during rollback at %s (%s): %s" + % (path, phase, exc) + ) + if not _same_managed_mount(verified, record.get("identity")): + return None, ( + "refusing changed mount during preparation rollback at %s: %s" + % (phase, path) + ) + nested = nested_paths(table, path) + if nested: + return None, ( + "refusing preparation rollback for %s with nested mount(s) " + "at %s: %s" % (path, phase, ", ".join(nested)) + ) + try: + busy = busy_mount_references( + path, + hardware=plan["hardware"], + ) + except BaseException as exc: + return None, ( + "could not inspect busy references during rollback at %s " + "(%s): %s" % (path, phase, exc) + ) + if busy: + return None, ( + "refusing preparation rollback for busy mount %s at %s; " + "referenced by PID(s): %s" + % (path, phase, ",".join(str(pid) for pid in busy)) + ) + if phase == "immediately before unmount": + try: + final_actual = mount_at(path) + final_verified = validate_mount(record, plan) + except BaseException as exc: + return None, ( + "could not revalidate exact mount identity after busy " + "scan at %s: %s" % (path, exc) + ) + if ( + not _same_managed_mount( + final_actual, + record.get("identity"), + ) + or not _same_managed_mount( + final_verified, + record.get("identity"), + ) + ): + return None, ( + "refusing changed mount after busy scan immediately " + "before unmount: %s" % path + ) + return "verified", None + + try: + table = mount_table() + except BaseException as exc: + message = "could not inspect nested mounts during rollback: %s" % exc + for record in records: + retain(record, message) + return cleanup_errors, retained, released + + for record in reversed(records): + path = record["path"] + ownership = record.get("ownership", "managed") + if ownership == "pending": + retain(record, ( + "refusing pathname-only rollback for pending mount: %s" + % path + )) + continue + result, error = inspect_candidate(record, table, "initial preflight") + if result == "absent": + released.add(path) + record["cleanup"] = {"state": "absent"} + continue + if error: + retain(record, error) + continue + candidates.append(record) + + if cleanup_errors: + message = ( + "rollback withheld because another managed mount could not be " + "verified safely" + ) + for record in candidates: + path = record["path"] + retained.add(path) + record["cleanup"] = { + "state": "retained", + "error": message, + } + return cleanup_errors, retained, released + + # Repeat the all-mount safety preflight at the latest shared boundary. This + # preserves the no-known-partial-cleanup contract when a mount changes + # after the initial scan but before the first unmount. + try: + latest_table = mount_table() + except BaseException as exc: + message = "could not refresh nested mounts before rollback: %s" % exc + for record in candidates: + retain(record, message) + return cleanup_errors, retained, released + + latest_candidates = [] + latest_errors = False + for record in candidates: + path = record["path"] + result, error = inspect_candidate( + record, + latest_table, + "latest all-mount preflight", + ) + if result == "absent": + released.add(path) + record["cleanup"] = {"state": "absent"} + elif error: + retain(record, error) + latest_errors = True + else: + latest_candidates.append(record) + if latest_errors: + message = ( + "rollback withheld because another managed mount changed after " + "initial preflight" + ) + for record in latest_candidates: + retain(record, message) + return cleanup_errors, retained, released + + for index, record in enumerate(latest_candidates): + path = record["path"] + try: + immediate_table = mount_table() + result, error = inspect_candidate( + record, + immediate_table, + "immediately before unmount", + ) + except BaseException as exc: + result, error = None, ( + "could not perform final rollback preflight at %s: %s" + % (path, exc) + ) + if result == "absent": + released.add(path) + record["cleanup"] = {"state": "absent"} + continue + if error: + retain(record, error) + message = ( + "rollback withheld after a late mount change at %s" % path + ) + for pending in latest_candidates[index + 1:]: + retain(pending, message) + break + try: + umount_path(path, plan["hardware"]) + except BaseException as exc: + message = "could not unmount %s during preparation rollback: %s" % ( + path, + exc, + ) + retain(record, message) + for pending in latest_candidates[index + 1:]: + retain( + pending, + "rollback withheld after unmount failure at %s" % path, + ) + break + try: + after = mount_at(path) + except BaseException as exc: + retain( + record, + "could not verify mount absence after rollback at %s: %s" + % (path, exc), + ) + for pending in latest_candidates[index + 1:]: + retain( + pending, + "rollback withheld after post-unmount verification " + "failure at %s" % path, + ) + break + if after is not None: + retain( + record, + "mount remains or was replaced after rollback helper at %s" + % path, + ) + for pending in latest_candidates[index + 1:]: + retain( + pending, + "rollback withheld after mount remained at %s" % path, + ) + break + released.add(path) + record["cleanup"] = {"state": "unmounted"} + + return cleanup_errors, retained, released + + +def _strongest_prepare_recovery_manifest(current, durable): + """Union recovery records without downgrading exact mount authority.""" + recovery = copy.deepcopy(current) + ownership_rank = {"pending": 0, "identified": 1, "managed": 2} + selected = {} + order = [] + for source_index, source in enumerate((durable or {}, current or {})): + for record in source.get("mounts", []): + key = ( + record.get("operation_id"), + record.get("path"), + ) + if key not in selected: + if ( + source_index == 1 + and record.get("ownership") == "pending" + ): + # A pending record that never reached a successful save is + # current-only evidence from before helper invocation and + # therefore is not a recovery authority. + continue + order.append(key) + selected[key] = copy.deepcopy(record) + continue + existing = selected[key] + if ownership_rank.get(record.get("ownership"), -1) >= ( + ownership_rank.get(existing.get("ownership"), -1) + ): + selected[key] = copy.deepcopy(record) + recovery["mounts"] = [selected[key] for key in order] + return recovery + + +def prepare( + args, + progress=None, + display_plan=True, + expected_plan_token=None, + cancel_event=None, + *, + load_manifest, + build_plan, + managed_ports_for_plan, + plan_confirmation_token, + render_plan, + confirm, + save_manifest, + mount_at, + mount_tmpfs, + umount_path, + validate_mount, + populate_mount, + validate_namespace, + source_still_matches, + ensure_busy_mount_scan_available, + durable_unlink, + manifest_path, + mount_table, + path_is_below, + busy_mount_references, +): + if load_manifest(required=False) is not None: + raise RamdiskError( + "a RAM-disk manifest already exists; stop/destroy it before " + "preparing another" + ) + plan = build_plan(args) + try: + base_port = int(getattr(args, "base_port", 8000)) + except (TypeError, ValueError): + raise RamdiskError("managed base port must be an integer") + planned_ports = managed_ports_for_plan(plan, base_port) + if ( + not 1 <= base_port <= 65535 + or len(set(planned_ports)) != len(planned_ports) + or any( + port < 1 or port > 65535 + for port in planned_ports + ) + ): + raise RamdiskError( + "managed base port produces invalid or duplicate replica ports" + ) + _raise_if_cancelled(cancel_event) + if ( + expected_plan_token is not None + and plan_confirmation_token(plan) != expected_plan_token + ): + raise RamdiskError( + "RAM-disk plan changed since review; inspect the updated plan " + "and confirm again" + ) + if plan["blockers"]: + raise RamdiskError( + "preparation blocked: " + "; ".join(plan["blockers"]) + ) + ensure_busy_mount_scan_available( + plan["mount_root"], + hardware=plan["hardware"], + ) + if display_plan: + render_plan(plan) + confirm( + "Mount tmpfs and stage the reviewed bytes?", + bool(getattr(args, "yes", False)), + ) + if progress is None: + progress_lock = threading.Lock() + + def progress(name, size, elapsed): + with progress_lock: + rate = size / elapsed / MIB if elapsed > 0 else 0.0 + print( + " staged %-36s %8.1f MiB/s" % (name, rate), + flush=True, + ) + + manifest = { + "version": MANIFEST_VERSION, + "deployment_id": secrets.token_hex(16), + "base_port": base_port, + "state": "preparing", + "created_at": _utc_now(), + "plan": plan, + "model_fingerprint": plan["model"]["fingerprint"], + "mounts": [], + "processes": [], + "ports": [], + "benchmark_results": [], + "initial_swap_used_bytes": plan["hardware"]["swap"]["used_bytes"], + } + durable_manifest = None + + def persist_manifest(): + nonlocal durable_manifest + save_manifest(manifest) + durable_manifest = copy.deepcopy(manifest) + + persist_manifest() + try: + for mount_index, mount in enumerate(plan["mounts"]): + _raise_if_cancelled(cancel_event) + if mount_at(mount["path"]): + raise RamdiskError( + "refusing already-mounted path: %s" + % mount["path"] + ) + record = dict(mount) + record["ownership"] = "pending" + record["operation_id"] = "%s:mount:%d" % ( + manifest["deployment_id"], + mount_index, + ) + record["requested"] = { + "filesystem": "tmpfs", + "source": "tmpfs", + "size_bytes": mount.get("size_bytes"), + "policy": mount.get("policy"), + "thp": plan.get("mount_options", {}).get("thp"), + "noswap": plan.get("mount_options", {}).get("noswap"), + "safety_options": [ + "noatime", + "nodev", + "nosuid", + "noexec", + "mode=0700", + ], + } + manifest["mounts"].append(record) + # This atomic write is the recovery boundary: no helper is invoked + # until the intended pathname exists durably as unowned/pending. + persist_manifest() + try: + mount_tmpfs(plan, mount) + except _MountHelperCompletedError: + # Only this typed failure proves the helper process completed. + # A generic runner exception can happen while a privileged + # helper is still in flight and must leave ownership pending. + # Even after completed failure, observation proves absence but + # cannot prove that an observed mount belongs to this attempt. + try: + failed_actual = mount_at(mount["path"]) + except Exception: + pass + else: + if failed_actual is None: + manifest["mounts"] = [ + candidate + for candidate in manifest["mounts"] + if candidate.get("operation_id") + != record["operation_id"] + ] + persist_manifest() + raise + mounted_actual = mount_at(mount["path"]) + if not mounted_actual: + raise RamdiskError( + "mounted %s but could not read its mount identity; " + "retained its pending recovery record" + % mount["path"] + ) + for key in ("effective_thp", "effective_noswap"): + if key in mount: + record[key] = mount[key] + record["identity"] = mounted_actual + record["ownership"] = "identified" + record["validated"] = False + persist_manifest() + identity = validate_mount(mount, plan) + if not _same_managed_mount(identity, mounted_actual): + raise RamdiskError( + "mount identity changed while validating %s" + % mount["path"] + ) + record["identity"] = identity + record["ownership"] = "managed" + record["validated"] = True + persist_manifest() + + seed = None + for index, mount in enumerate(plan["mounts"]): + _raise_if_cancelled(cancel_event) + source = ( + seed + if index and plan["topology"] == "per-node" + else None + ) + populate_mount( + plan, + mount, + source_root=source, + progress=progress, + cancel_event=cancel_event, + ) + if seed is None: + seed = mount["path"] + manifest["mounts"][index]["numa_allocation"] = ( + validate_namespace(plan, mount) + ) + persist_manifest() + _raise_if_cancelled(cancel_event) + source_still_matches(plan) + manifest["state"] = "ready" + manifest["ready_at"] = _utc_now() + persist_manifest() + return manifest + except BaseException as exc: + # Keep the strongest exact observation even when its directory fsync + # failed, while retaining any durable-only pending operation that an + # ambiguous removal may still leave recoverable. + recovery_manifest = _strongest_prepare_recovery_manifest( + manifest, + durable_manifest, + ) + recovery_manifest["state"] = "error" + recovery_manifest["error"] = str(exc) + cleanup_errors = [] + rollback_authority_persisted = False + try: + save_manifest(recovery_manifest) + rollback_authority_persisted = True + except BaseException as save_exc: + cleanup_errors.append( + "could not persist preparation error: %s" % save_exc + ) + if rollback_authority_persisted: + ( + mount_cleanup_errors, + retained_mounts, + released_mounts, + ) = _rollback_preparation_mounts( + recovery_manifest, + mount_at=mount_at, + mount_table=mount_table, + path_is_below=path_is_below, + busy_mount_references=busy_mount_references, + umount_path=umount_path, + validate_mount=validate_mount, + ) + cleanup_errors.extend(mount_cleanup_errors) + else: + retained_mounts = { + record["path"] + for record in recovery_manifest.get("mounts", []) + if isinstance(record.get("path"), str) + } + released_mounts = set() + for record in recovery_manifest.get("mounts", []): + record["cleanup"] = { + "state": "retained", + "error": ( + "rollback withheld because exact recovery authority " + "was not proven durable" + ), + } + recovery_manifest["recovery"] = { + "operation": "prepare", + "state": ( + "attention-required" + if retained_mounts + else "clean" + ), + "retained_mounts": sorted(retained_mounts), + "released_mounts": sorted(released_mounts), + } + if cleanup_errors: + recovery_manifest["cleanup_errors"] = cleanup_errors + else: + recovery_manifest.pop("cleanup_errors", None) + try: + save_manifest(recovery_manifest) + except BaseException as save_exc: + cleanup_errors.append( + "could not persist cleanup result: %s" % save_exc + ) + if isinstance(exc, _OperationCancelled) and not cleanup_errors: + try: + durable_unlink(manifest_path()) + except Exception as cleanup_exc: + cleanup_errors.append( + "could not remove cancelled preparation manifest: %s" + % cleanup_exc + ) + if not cleanup_errors: + raise + if cleanup_errors: + recovery_manifest["cleanup_errors"] = cleanup_errors + raise RamdiskError( + "%s; preparation rollback/reporting errors: %s" + % (exc, "; ".join(cleanup_errors)) + ) from exc + raise + + +def _rollback_launched_children( + spawned, + records, + *, + process_matches, + group_alive, + track_managed_child, + terminate_verified_group, + terminate_direct_child, + forget_managed_child, + launch_contexts=None, +): + """Terminate launch children without mistaking uncertainty for absence.""" + cleanup_failures = [] + surviving_groups = set() + records_by_pid = { + int(record["pid"]): record + for record in records + } + contexts_by_pid = { + int(process.pid): context + for process, context in zip( + spawned, + launch_contexts or (), + ) + } + + # Published children get immediate identity+nonce revalidation. An + # unpublished child is signaled only through its retained Popen handle. + for process in reversed(spawned): + pid = int(process.pid) + record = records_by_pid.get(pid) + context = contexts_by_pid.get(pid) + termination_failure = None + try: + if record is not None: + track_managed_child(process) + termination_failure = terminate_verified_group(record) + # Identity drift can make group signaling unsafe while the + # retained Popen still proves this exact direct child is ours. + # Terminating that child by handle is safe and may also let a + # later group scan prove that no forked engine survived. + try: + direct_child_needs_fallback = process.poll() is None + except BaseException: + direct_child_needs_fallback = True + if direct_child_needs_fallback: + try: + direct_failure = terminate_direct_child(process) + except BaseException as exc: + direct_failure = ( + "direct-child termination attempt failed: %s" + % exc + ) + if direct_failure: + termination_failure = "; ".join( + item + for item in ( + str(termination_failure or ""), + str(direct_failure), + ) + if item + ) + else: + termination_failure = terminate_direct_child(process) + except BaseException as exc: + termination_failure = "termination attempt failed: %s" % exc + + wait_failure = None + try: + process.wait(timeout=1) + except ( + subprocess.TimeoutExpired, + ChildProcessError, + ): + pass + except BaseException as exc: + wait_failure = "could not wait for direct child: %s" % exc + + poll_failure = None + try: + direct_child_alive = process.poll() is None + except BaseException as exc: + direct_child_alive = True + poll_failure = "could not poll direct child: %s" % exc + + identity_matches = False + identity_reason = None + identity_failure = None + direct_created_group_alive = None + group_identity_failure = None + if record is not None: + try: + ( + identity_matches, + identity_reason, + _, + ) = process_matches(record) + except BaseException as exc: + identity_reason = "identity-check-failed" + identity_failure = ( + "could not establish managed process identity: %s" % exc + ) + else: + # Popen.poll() establishes only whether the direct wrapper exited. + # start_new_session=True made its PID the directly-created PGID, + # and a forked engine can remain there after the wrapper dies. + try: + direct_created_group_alive = group_alive(pid) + except BaseException as exc: + group_identity_failure = ( + "could not establish direct-created process group %s " + "absence: %s" % (pid, exc) + ) + + absence_proven = ( + not direct_child_alive + and ( + ( + record is None + and direct_created_group_alive is False + ) + or ( + not identity_matches + and identity_reason == "not-running" + ) + ) + ) + if absence_proven: + try: + forget_managed_child(pid) + except BaseException as exc: + cleanup_failures.append( + "could not forget reaped direct child %s: %s" + % (pid, exc) + ) + continue + + details = [] + if termination_failure: + details.append(str(termination_failure)) + if wait_failure: + details.append(wait_failure) + if poll_failure: + details.append(poll_failure) + if identity_failure: + details.append(identity_failure) + if group_identity_failure: + details.append(group_identity_failure) + if direct_child_alive: + details.append("direct child is still alive") + elif identity_matches: + details.append("persisted process identity is still running") + elif record is None and direct_created_group_alive: + details.append( + "direct-created process group %s is still alive" % pid + ) + elif record is None: + details.append( + "direct-created process group %s absence remains unproven" + % pid + ) + else: + details.append( + "managed process absence remains unproven (%s)" + % (identity_reason or "unknown") + ) + failure = "; ".join(details) + cleanup_failures.append(failure) + surviving_pgid = ( + int(record["pgid"]) + if record is not None + else pid + ) + surviving_groups.add(surviving_pgid) + if record is not None: + record["stop_error"] = failure + if context is not None: + context["rollback_process_alive"] = True + context["rollback_pid"] = pid + context["rollback_error"] = failure + + return cleanup_failures, surviving_groups + + +def _construct_retained_popen( + popen_factory, + attempt_context, + *args, + **kwargs, +): + """Construct Popen while retaining a real partially initialized attempt. + + Normal callable test doubles remain opaque: if they raise, no object exists + whose child-creation fields can be inspected, so their outcome is unknown. + A real attempt is registered before ``__init__`` can create a child, making + its exact handle available to rollback even if construction is interrupted. + """ + if not ( + isinstance(popen_factory, type) + and issubclass(popen_factory, _POPEN_BASE_TYPE) + ): + try: + return popen_factory(*args, **kwargs), None, False + except BaseException as exc: + return None, exc, False + + try: + attempt = popen_factory.__new__(popen_factory) + except BaseException as exc: + return None, exc, False + + attempt_context["popen_attempt"] = attempt + try: + popen_factory.__init__(attempt, *args, **kwargs) + except BaseException as exc: + return attempt, exc, True + return attempt, None, True + + +def _launch_identity_matches( + identity, + *, + pid, + uid, + nonce, + state_dir, + weights_dir, +): + """Require complete observed provenance before publishing a launch.""" + return ( + isinstance(identity, dict) + and identity.get("pid") == pid + and identity.get("uid") == uid + and identity.get("inert") is False + and _positive_int(identity.get("starttime")) + and identity.get("nonce") == nonce + and identity.get("pgid") == pid + and identity.get("sid") == pid + and identity.get("state_dir") == state_dir + and identity.get("weights_dir") == weights_dir + ) + + +def start( + args, + cli_path=None, + engine_path=None, + cancel_event=None, + *, + default_cli_path, + load_manifest, + assert_effective_masks_unchanged, + assert_ready_mounts, + process_matches, + group_alive, + managed_child_liveness, + save_manifest, + merge_usage, + bind_usage_transaction, + persisted_base_port, + fresh_user_binary, + admit_concurrent_runtimes, + state_root, + ensure_private_dir, + assert_durable_state_dir, + usage_journal_transaction_id, + recover_delta, + usage_read, + usage_write, + validate_usage_for_plan, + managed_numa_enabled, + memory_node_list, + engine_cpu_list, + node_core_count, + normalized_runtime_knobs, + apply_managed_accelerator_environment=None, + invoking_uid, + process_start_boundary, + current_process_identity, + proc_identity, + wait_managed_ready, + track_managed_child, + terminate_verified_group, + terminate_direct_child, + forget_managed_child, +): + if apply_managed_accelerator_environment is None: + from .accelerator import _apply_managed_accelerator_environment + + apply_managed_accelerator_environment = ( + _apply_managed_accelerator_environment + ) + launch_uid = invoking_uid() + if ( + not isinstance(launch_uid, int) + or isinstance(launch_uid, bool) + or launch_uid < 0 + ): + raise RamdiskError("managed launch has an invalid invoking UID") + manifest = load_manifest(required=True) + if _pending_launch_recovery(manifest): + raise _pending_launch_recovery_error("start") + if _retained_process_recovery(manifest): + raise _unresolved_process_recovery_error("start") + _raise_if_cancelled(cancel_event) + if manifest.get("state") not in ("ready", "stopped"): + raise RamdiskError( + "manifest state is %s, not ready" + % manifest.get("state") + ) + assert_effective_masks_unchanged(manifest["plan"]) + assert_ready_mounts(manifest) + plan = manifest["plan"] + cli_path = cli_path or default_cli_path + resolved_engine_path = None + if engine_path is not None: + resolved_engine_path = os.path.realpath( + os.path.abspath(str(engine_path)) + ) + if ( + not os.path.isfile(resolved_engine_path) + or not os.access(resolved_engine_path, os.X_OK) + ): + raise RamdiskError( + "reviewed engine is not an executable file: %s" + % resolved_engine_path + ) + model = plan["model"]["path"] + canonical_usage = os.path.join(model, ".coli_usage") + foreign = [] + recovered = False + recovery_records = [] + for record in manifest.get("processes", []): + _raise_if_cancelled(cancel_event) + try: + child_alive = managed_child_liveness(record["pid"]) + child_liveness_failure = None + except BaseException as child_exc: + child_alive = True + child_liveness_failure = ( + "retained-child-liveness-check-failed: %s" % child_exc + ) + try: + matches, reason, _ = process_matches(record) + except BaseException as identity_exc: + matches, reason = ( + False, + "identity-check-failed: %s" % identity_exc, + ) + if child_liveness_failure: + matches = False + reason = child_liveness_failure + if record.get("stopped_at"): + if matches or reason != "not-running" or child_alive is True: + foreign.append( + "PID %s (%s)" + % ( + record.get("pid"), + ( + "stopped-record-process-group-live" + if matches + else "retained-managed-child-live" + if reason == "not-running" + and child_alive is True + else reason + ), + ) + ) + continue + recovery_records.append((record, "stopped")) + continue + if matches: + raise RamdiskError( + "managed engine is already running on port %s" + % record.get("port") + ) + if reason == "not-running" and child_alive is not True: + recovery_records.append((record, "crashed")) + else: + foreign.append( + "PID %s (%s)" + % ( + record.get("pid"), + ( + "retained-managed-child-live" + if reason == "not-running" + and child_alive is True + else reason + ), + ) + ) + if foreign: + refusal = RamdiskError( + "refusing stale foreign process records: " + + ", ".join(foreign) + ) + manifest["state"] = "error" + manifest["launch_error"] = str(refusal) + manifest.setdefault("cleanup_errors", []).append(str(refusal)) + try: + save_manifest(manifest) + except Exception as save_exc: + raise RamdiskError( + "%s; could not persist managed-child recovery: %s" + % (refusal, save_exc) + ) from refusal + raise refusal + + fingerprint_dir = manifest["model_fingerprint"].split( + ":", + 1, + )[-1] + seed_state_dirs = [] + for mount in manifest["mounts"]: + _raise_if_cancelled(cancel_event) + node = mount.get("node") + label = "interleaved" if node is None else "node-%d" % node + state_dir = os.path.join( + state_root(), + "engines", + fingerprint_dir, + label, + ) + ensure_private_dir(state_dir) + assert_durable_state_dir(state_dir, plan=plan) + seed_state_dirs.append(state_dir) + expected_seed_journals, reserved_usage_ids = ( + _preflight_seed_usage_journals( + manifest, + seed_state_dirs, + plan=plan, + usage_journal_transaction_id=usage_journal_transaction_id, + ) + ) + recovery_state_keys = { + _state_dir_authority_key(record["state_dir"]) + for record, _ in recovery_records + } + orphan_reserved_ids = { + merge_id + for state_key, merge_id in expected_seed_journals.items() + if merge_id is not None and state_key not in recovery_state_keys + } + if recovery_records: + reserved_usage_ids = _bind_recovery_usage_transactions( + manifest, + [record for record, _ in recovery_records], + plan=plan, + bind_usage_transaction=bind_usage_transaction, + reserved_ids=orphan_reserved_ids, + ) + # Every adopted or minted authority becomes durable before the first + # canonical replay, so a crash cannot later mint a second transaction. + save_manifest(manifest) + for record, recovery_kind in recovery_records: + merge_usage( + record, + canonical_usage, + plan=plan, + ) + if not record.get("usage_merged_at"): + record["usage_merged_at"] = _utc_now() + if recovery_kind == "crashed": + record["stopped_at"] = _utc_now() + record["crash_recovered_at"] = _utc_now() + record.pop("usage_merge_error", None) + recovered = True + save_manifest(manifest) + if recovered: + save_manifest(manifest) + + requested_base_port = getattr(args, "base_port", None) + if requested_base_port is None: + base_port = persisted_base_port(manifest) + else: + if isinstance(requested_base_port, bool): + raise RamdiskError( + "managed base port must be an integer" + ) + try: + base_port = int(requested_base_port) + except (TypeError, ValueError): + raise RamdiskError( + "managed base port must be an integer" + ) + ports = [ + base_port + + ( + 0 + if record.get("node") is None + else int(record["node"]) + ) + for record in manifest["mounts"] + ] + if ( + len(set(ports)) != len(ports) + or any(port < 1 or port > 65535 for port in ports) + ): + raise RamdiskError( + "managed ports are invalid or duplicated" + ) + for port in ports: + probe = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM, + ) + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: + raise RamdiskError( + "port %d is unavailable: %s" % (port, exc) + ) + finally: + probe.close() + + previous_state = manifest["state"] + previous_processes = copy.deepcopy( + manifest.get("processes", []) + ) + previous_ports = list(manifest.get("ports", [])) + previous_base_port = persisted_base_port(manifest) + manifest["base_port"] = base_port + managed_numactl = ( + fresh_user_binary("numactl") + if plan["topology"] == "per-node" + else None + ) + records = [] + runtime = plan.get("managed_runtime", {}) + saved_runtime_knobs = dict( + manifest.get("best_runtime", {}) + .get(plan["topology"], {}) + .get("knobs") + or {} + ) + # Thread counts are node-relative and the managed topology contract always + # uses every physical core. Retain other measured knobs for this topology. + saved_runtime_knobs.pop("OMP_NUM_THREADS", None) + managed_ctx = int(runtime.get("ctx", 4096)) + managed_slots = int(runtime.get("kv_slots", 1)) + managed_cap = int(runtime.get("cache_cap", 8)) + try: + startup_timeout = float( + os.environ.get( + "COLI_RAMDISK_START_TIMEOUT", + "7200", + ) + ) + except ValueError: + raise RamdiskError( + "COLI_RAMDISK_START_TIMEOUT must be numeric" + ) + if ( + not math.isfinite(startup_timeout) + or not 1 <= startup_timeout <= 86400 + ): + raise RamdiskError( + "COLI_RAMDISK_START_TIMEOUT must be between 1 and 86400 " + "seconds" + ) + # Recover every stable replica state first, then validate one canonical + # seed before creating any per-engine copy or spawning the first child. + # A later replica's journal therefore cannot introduce an incompatible + # identified header after an earlier replica has already launched. + for state_dir in seed_state_dirs: + _raise_if_cancelled(cancel_event) + state_key = _state_dir_authority_key(state_dir) + if state_key in recovery_state_keys: + continue + recover_delta( + state_dir, + canonical_usage, + plan=plan, + expected_merge_id=expected_seed_journals[state_key], + ) + canonical_baseline = usage_read(canonical_usage, plan=plan) + validate_usage_for_plan( + canonical_baseline, + plan, + source="canonical usage history %s" % canonical_usage, + ) + launcher_identity = current_process_identity() + launcher_pid = ( + launcher_identity.get("pid") + if isinstance(launcher_identity, dict) + else None + ) + launcher_starttime = ( + launcher_identity.get("starttime") + if isinstance(launcher_identity, dict) + else None + ) + launcher_identity_uid = ( + launcher_identity.get("uid") + if isinstance(launcher_identity, dict) + else None + ) + launcher_cmdline = ( + launcher_identity.get("cmdline") + if isinstance(launcher_identity, dict) + else None + ) + if ( + not _positive_int(launcher_pid) + or launcher_identity_uid != launch_uid + or not _positive_int(launcher_starttime) + or not isinstance(launcher_cmdline, list) + or not launcher_cmdline + or any(not isinstance(item, str) or not item for item in launcher_cmdline) + ): + raise RamdiskError( + "managed launch cannot establish its exact launcher identity" + ) + # Admit the complete replica set from one shared cgroup snapshot before + # spawning the first child. + admit_concurrent_runtimes( + plan, + manifest["mounts"], + benchmark=False, + ) + spawned = [] + launch_contexts = [] + manifest["processes"] = [] + manifest["ports"] = [] + manifest["pending_launches"] = [] + manifest["state"] = "starting" + save_manifest(manifest) + + try: + for index, mount in enumerate(manifest["mounts"]): + _raise_if_cancelled(cancel_event) + node = mount.get("node") + state_dir = seed_state_dirs[index] + baseline = dict(canonical_baseline) + usage_write( + os.path.join(state_dir, ".coli_usage"), + baseline, + plan=plan, + ) + port = base_port + ( + 0 if node is None else int(node) + ) + command = [ + cli_path, + "serve", + "--model", + model, + "--port", + str(port), + "--cap", + str(managed_cap), + "--ctx", + str(managed_ctx), + "--kv-slots", + str(managed_slots), + ] + if not os.access(cli_path, os.X_OK): + command.insert(0, sys.executable) + if node is not None: + command = [ + managed_numactl, + "--physcpubind=%s" + % engine_cpu_list(plan, node=node), + "--membind=%d" % node, + ] + command + launch_nonce = secrets.token_hex(24) + usage_merge_id = _mint_usage_transaction_id(reserved_usage_ids) + launch_not_before = process_start_boundary() + if ( + not isinstance(launch_not_before, int) + or isinstance(launch_not_before, bool) + or launch_not_before < 0 + ): + raise RamdiskError( + "managed launch has an invalid process-start boundary" + ) + pending_entry = { + "operation_id": "start:%s" % usage_merge_id, + "nonce": launch_nonce, + "uid": launch_uid, + "port": port, + "node": node, + "state_dir": state_dir, + "weights_dir": mount["path"], + "launch_not_before": launch_not_before, + "launcher_pid": launcher_pid, + "launcher_starttime": launcher_starttime, + "launcher_cmdline": list(launcher_cmdline), + "expected_command": list(command), + "usage_baseline": baseline, + "usage_merge_id": usage_merge_id, + } + context = { + "node": node, + "state_dir": state_dir, + "usage_baseline": baseline, + "usage_merge_id": usage_merge_id, + "pending_entry": pending_entry, + "spawn_outcome": "not-attempted", + "record": None, + } + launch_contexts.append(context) + manifest["pending_launches"].append(pending_entry) + # A hard manager crash after this durable write but before exact + # process publication leaves an outcome-unknown launch. Stop is + # the sole supported reconciler; every other mutation must retain + # and surface it, never silently ignore it. + save_manifest(manifest) + environment = os.environ.copy() + environment.update( + { + "COLI_WEIGHTS_DIR": mount["path"], + "COLI_STATE_DIR": state_dir, + "COLI_MANAGED_NONCE": launch_nonce, + "COLI_NUMA": ( + "1" + if managed_numa_enabled(plan, node) + else "0" + ), + "COLI_NUMA_NODES": memory_node_list( + plan, + node=node, + ), + "COLI_CPU_AFFINITY": engine_cpu_list( + plan, + node=node, + ), + "OMP_NUM_THREADS": str( + node_core_count(plan, node) + ), + "OMP_PROC_BIND": "close", + "OMP_PLACES": "cores", + "CTX": str(managed_ctx), + "KV_SLOTS": str(managed_slots), + "COLI_KV_SLOTS": str(managed_slots), + # Managed engines deliberately keep durable node-specific + # KV state; shell overrides may not disable that promise. + "KVSAVE": "1", + "CAP_RAISE": "0", + "AUTOPIN": "0", + "PROF": "1", + } + ) + for inherited in ( + "COLI_ENGINE", + "COLI_MMAP", + "PIN", + "PIN_GB", + "PIN_FILL", + "RAM_GB", + "COLI_RAM_OVERCOMMIT", + "CUDA_EXPERT_GB", + "CUDA_DENSE", + "COLI_GPUS", + "COLI_GPU", + "COLI_CUDA", + "COLI_NO_OMP_TUNE", + "COLI_OMP_TUNED", + "COLI_USAGE_DECAY", + "DIRECT", + "PIPE", + "PIPE_WORKERS", + "URING", + ): + environment.pop(inherited, None) + if resolved_engine_path is not None: + environment["COLI_ENGINE"] = resolved_engine_path + applied_runtime_knobs = normalized_runtime_knobs( + plan, + saved_runtime_knobs, + node=node, + ) + for key, value in applied_runtime_knobs.items(): + environment[key] = str(value) + # Managed accounting is reconciled against an exact launch + # baseline, so ambient decay may not lower cumulative counters. + environment["COLI_USAGE_DECAY"] = "1.0" + applied_accelerator = apply_managed_accelerator_environment( + environment, + plan, + ) + environment["COLI_RAM_PREFAULT"] = str( + plan["prefault"] + if applied_accelerator.get("COLI_RAMMAP") == "1" + else 0 + ) + log_path = os.path.join( + state_dir, + "engine.log", + ) + _raise_if_cancelled(cancel_event) + log = open(log_path, "ab", buffering=0) + child_stdin = None + try: + child_stdin = open(os.devnull, "rb") + # Retain outcome-unknown until a successful handle or a real + # partially initialized Popen attempt proves what happened. + context["spawn_outcome"] = "outcome-unknown" + ( + process, + construction_error, + attempt_inspected, + ) = _construct_retained_popen( + subprocess.Popen, + context, + command, + env=environment, + stdin=child_stdin, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + close_fds=True, + ) + if construction_error is not None: + child_created = bool( + getattr(process, "_child_created", False) + ) if process is not None else False + child_pid = ( + getattr(process, "pid", None) + if process is not None + else None + ) + exact_child_handle = ( + isinstance(child_pid, int) + and not isinstance(child_pid, bool) + and child_pid > 0 + and attempt_inspected + ) + if exact_child_handle: + # Popen writes pid immediately before _child_created; + # normalize an interruption in that narrow window so + # poll/wait/destruction retain the exact child handle. + process._child_created = True + spawned.append(process) + context["spawn_outcome"] = "created" + elif ( + isinstance(construction_error, Exception) + and attempt_inspected + and not child_created + and child_pid is None + ): + # A normal exception plus the inspected attempt state + # proves construction never published a child PID. + context["spawn_outcome"] = "proven-absent" + raise construction_error + if process is None: + raise RamdiskError( + "Popen returned no process handle" + ) + spawned.append(process) + context["spawn_outcome"] = "created" + finally: + try: + if child_stdin is not None: + child_stdin.close() + finally: + log.close() + identity = None + identity_deadline = time.monotonic() + 1.0 + while ( + time.monotonic() < identity_deadline + and process.poll() is None + ): + _raise_if_cancelled(cancel_event) + identity = proc_identity(process.pid) + if ( + isinstance(identity, dict) + and identity.get("pid") == process.pid + and _positive_int(identity.get("starttime")) + ): + context["observed_leader_starttime"] = identity[ + "starttime" + ] + if _launch_identity_matches( + identity, + pid=process.pid, + uid=launch_uid, + nonce=launch_nonce, + state_dir=state_dir, + weights_dir=mount["path"], + ): + break + time.sleep(0.01) + if ( + process.poll() is not None + or not _launch_identity_matches( + identity, + pid=process.pid, + uid=launch_uid, + nonce=launch_nonce, + state_dir=state_dir, + weights_dir=mount["path"], + ) + ): + context["promotion_identity_failed"] = True + raise RamdiskError( + "managed engine exited or failed exact identity " + "attribution during launch; see %s" + % log_path + ) + record = { + "pid": identity["pid"], + "pgid": identity["pgid"], + "uid": identity["uid"], + "starttime": identity["starttime"], + "nonce": identity["nonce"], + "port": port, + "node": node, + "command": command, + "state_dir": identity["state_dir"], + "weights_dir": identity["weights_dir"], + "usage_baseline": baseline, + "usage_merge_id": usage_merge_id, + "started_at": _utc_now(), + "log": log_path, + "runtime_knobs": applied_runtime_knobs, + "accelerator_environment": applied_accelerator, + } + candidate_records = records + [record] + manifest["processes"] = candidate_records + manifest["ports"] = [ + item["port"] + for item in candidate_records + ] + manifest["pending_launches"] = [ + pending + for pending in manifest["pending_launches"] + if pending.get("operation_id") + != pending_entry["operation_id"] + ] + manifest["state"] = "starting" + # A raised save has an ambiguous commit outcome: os.replace may + # already have published this exact record. Keep the candidate in + # memory so rollback never downgrades it to PID-only authority. + save_manifest(manifest) + records.append(record) + context["record"] = record + + # Launch all nodes before waiting so replicated model loading proceeds + # concurrently. Publish `running` only after every health check passes. + for record in records: + wait_managed_ready( + record, + startup_timeout, + api_key=os.environ.get("COLI_API_KEY"), + cancel_event=cancel_event, + ) + save_manifest(manifest) + _raise_if_cancelled(cancel_event) + manifest["state"] = "running" + save_manifest(manifest) + for process in spawned: + track_managed_child(process) + return manifest + except BaseException as launch_error: + cleanup_failures = [] + surviving_groups = set() + + # Close the line-level publication windows before rollback. A real + # Popen attempt is registered in its context before __init__ can create + # a child, so recover its exact handle even if control never returned + # far enough to append it to ``spawned``. A handle already present in + # ``spawned`` proves Popen returned even if control landed before + # ``spawn_outcome`` was updated. Likewise, an exact record still in the + # candidate manifest is the published authority even if interruption + # landed before the local records list and context were updated. + spawned_ids = {id(process) for process in spawned} + for context in launch_contexts: + attempt = context.get("popen_attempt") + attempt_pid = getattr(attempt, "pid", None) + exact_attempt = ( + isinstance(attempt_pid, int) + and not isinstance(attempt_pid, bool) + and attempt_pid > 0 + ) + if exact_attempt and id(attempt) not in spawned_ids: + # Popen assigns pid immediately before _child_created. Restore + # that invariant when an asynchronous exception landed between + # the two assignments or anywhere before caller registration. + attempt._child_created = True + spawned.append(attempt) + spawned_ids.add(id(attempt)) + for context in launch_contexts[:len(spawned)]: + context["spawn_outcome"] = "created" + published_by_launch = { + ( + record.get("usage_merge_id"), + record.get("state_dir"), + ): record + for record in manifest.get("processes", []) + if isinstance(record, dict) + } + for context in launch_contexts: + published = published_by_launch.get( + ( + context.get("usage_merge_id"), + context.get("state_dir"), + ) + ) + if published is not None: + context["record"] = published + rollback_records = list(manifest.get("processes", [])) + + def rollback_save(label): + try: + save_manifest(manifest) + return True + except Exception as save_exc: + cleanup_failures.append( + "%s: %s" % (label, save_exc) + ) + return False + + ( + child_cleanup_failures, + surviving_groups, + ) = _rollback_launched_children( + spawned, + rollback_records, + process_matches=process_matches, + group_alive=group_alive, + track_managed_child=track_managed_child, + terminate_verified_group=terminate_verified_group, + terminate_direct_child=terminate_direct_child, + forget_managed_child=forget_managed_child, + launch_contexts=launch_contexts, + ) + cleanup_failures.extend(child_cleanup_failures) + outcome_unknown_contexts = [ + context + for context in launch_contexts + if context.get("spawn_outcome") == "outcome-unknown" + ] + for context in outcome_unknown_contexts: + cleanup_failures.append( + "Popen was interrupted after launch began for %s; process " + "creation outcome is unknown and pending recovery was retained" + % context["state_dir"] + ) + mismatched_live_contexts = [ + context + for context in launch_contexts + if ( + context.get("promotion_identity_failed") + and context.get("rollback_process_alive") + and context.get("record") is None + ) + ] + for context in mismatched_live_contexts: + pending = context["pending_entry"] + leader_starttime = context.get("observed_leader_starttime") + pending["observed_group"] = { + "pgid": context["rollback_pid"], + "uid": pending["uid"], + "leader_starttime": ( + leader_starttime + if _positive_int(leader_starttime) + else None + ), + } + retained_processes = [ + dict( + pid=context.get("rollback_pid"), + pgid=context.get("rollback_pid"), + node=context.get("node"), + state_dir=context["state_dir"], + usage_baseline=context["usage_baseline"], + usage_merge_id=context["usage_merge_id"], + error=context.get("rollback_error"), + **_retained_process_authority(context["pending_entry"]), + ) + for context in launch_contexts + if ( + context.get("rollback_process_alive") + and context.get("record") is None + and not context.get("promotion_identity_failed") + ) + ] + # Cooperative rollback has now classified every pending launch as + # never-created, proven absent, still discoverable under its pending + # nonce plus observed PGID, or retained with an exact direct PGID. + # Publish that transition before any usage transaction can be saved. + manifest["pending_launches"] = [ + context["pending_entry"] + for context in outcome_unknown_contexts + mismatched_live_contexts + ] + if retained_processes: + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": retained_processes, + } + + for context in launch_contexts: + record = context.get("record") or { + "state_dir": context["state_dir"], + "usage_baseline": context["usage_baseline"], + "usage_merge_id": context["usage_merge_id"], + } + if context.get("rollback_process_alive"): + # The child may still be writing this state directory. Never + # merge or publish its usage while direct-child absence is + # unproven, even when identity publication never completed. + continue + if context.get("spawn_outcome") == "outcome-unknown": + # An unknown child may still be writing. Its baseline and + # transaction id stay private and durable in pending_launches. + continue + if ( + context.get("record") + and record["pgid"] in surviving_groups + ): + continue + if record.get("usage_merge_id") is None: + record["usage_merge_id"] = _mint_usage_transaction_id( + reserved_usage_ids + ) + transaction_persisted = bool( + context.get("record") + ) and rollback_save( + "could not persist usage transaction for %s" + % context["state_dir"] + ) + try: + # If the transaction id could not be serialized, retain the + # journal after applying it so a future start can recognize it. + merge_usage( + record, + canonical_usage, + plan=plan, + keep_journal=not transaction_persisted, + ) + if context.get("record"): + record["usage_merged_at"] = _utc_now() + record["stopped_at"] = _utc_now() + rollback_save( + "could not persist usage recovery for %s" + % context["state_dir"] + ) + except Exception as exc: + if context.get("record"): + record["usage_merge_error"] = str(exc) + cleanup_failures.append( + "usage recovery for %s: %s" + % (context["state_dir"], exc) + ) + if ( + isinstance(launch_error, _OperationCancelled) + and not cleanup_failures + ): + manifest["state"] = previous_state + manifest["processes"] = previous_processes + manifest["ports"] = previous_ports + manifest["base_port"] = previous_base_port + manifest.pop("launch_error", None) + manifest.pop("cleanup_errors", None) + try: + save_manifest(manifest) + except Exception as save_exc: + cleanup_failures.append( + "could not persist clean launch cancellation: %s" + % save_exc + ) + if not cleanup_failures: + raise + + manifest["state"] = "error" + manifest["launch_error"] = str(launch_error) + manifest["cleanup_errors"] = cleanup_failures + rollback_save("could not persist launch rollback") + if cleanup_failures: + raise RamdiskError( + "%s; launch rollback/reporting errors: %s" + % ( + launch_error, + "; ".join(cleanup_failures), + ) + ) from launch_error + raise + + +def stop( + args=None, + *, + load_manifest, + discover_managed_launches=None, + process_matches, + process_group_members=None, + group_alive, + managed_child_liveness, + save_manifest, + terminate_verified_group, + merge_usage, + bind_usage_transaction, +): + manifest = load_manifest(required=True) + pending_preflights, pending_refusals = _preflight_pending_launches( + manifest, + discover_managed_launches=discover_managed_launches, + process_matches=process_matches, + process_group_members=process_group_members, + group_alive=group_alive, + ) + retained_refusals = _preflight_unpublished_processes( + manifest, + group_alive=group_alive, + discover_managed_launches=discover_managed_launches, + ) + recovery_refusals = pending_refusals + retained_refusals + if recovery_refusals: + refusal = RamdiskError( + "refusing unresolved managed-process recovery: " + + "; ".join(recovery_refusals) + ) + manifest["state"] = "error" + cleanup_errors = manifest.setdefault("cleanup_errors", []) + cleanup_errors[:] = [ + error + for error in cleanup_errors + if not str(error).startswith(_PENDING_RECOVERY_ERROR_PREFIX) + ] + cleanup_errors.append( + _PENDING_RECOVERY_ERROR_PREFIX + "; ".join(recovery_refusals) + ) + try: + save_manifest(manifest) + except BaseException as save_exc: + raise RamdiskError( + "%s; could not persist managed-process recovery refusal: %s" + % (refusal, save_exc) + ) from refusal + raise refusal + non_managed_mounts = [ + record + for record in manifest.get("mounts", []) + if record.get("ownership", "managed") != "managed" + ] + if non_managed_mounts: + refusal = RamdiskError( + "refusing stop while non-managed mount ownership requires " + "recovery: " + + ", ".join( + "%s (%s)" + % (record.get("path"), record.get("ownership")) + for record in non_managed_mounts + ) + ) + manifest["state"] = "error" + manifest.setdefault("cleanup_errors", []).append(str(refusal)) + try: + save_manifest(manifest) + except BaseException as save_exc: + raise RamdiskError( + "%s; could not persist mount recovery refusal: %s" + % (refusal, save_exc) + ) from refusal + raise refusal + plan = manifest["plan"] + canonical_usage = os.path.join( + manifest["plan"]["model"]["path"], + ".coli_usage", + ) + refusals = [] + identities = [] + for record in manifest.get("processes", []): + try: + child_alive = managed_child_liveness(record["pid"]) + child_liveness_failure = None + except BaseException as child_exc: + child_alive = True + child_liveness_failure = ( + "retained-child-liveness-check-failed: %s" % child_exc + ) + try: + matches, reason, actual = process_matches(record) + except BaseException as identity_exc: + matches, reason, actual = ( + False, + "identity-check-failed: %s" % identity_exc, + None, + ) + if child_liveness_failure: + matches = False + reason = child_liveness_failure + if record.get("stopped_at"): + if matches or reason != "not-running" or child_alive is True: + record["stop_error"] = ( + "stopped-record-process-group-live" + if matches + else child_liveness_failure + or "retained-managed-child-live" + if reason == "not-running" + and child_alive is True + else reason + ) + refusals.append( + "PID %s is %s" + % (record.get("pid"), record["stop_error"]) + ) + identities.append( + (record, False, "already-stopped", None) + ) + continue + if not matches and reason == "not-running" and child_alive is True: + reason = "retained-managed-child-live" + if not matches and reason != "not-running": + record["stop_error"] = reason + refusals.append( + "PID %s is %s" + % (record.get("pid"), reason) + ) + identities.append( + (record, matches, reason, actual) + ) + # Validate every identity before signaling any process. + if refusals: + refusal = RamdiskError( + "refusing to signal unverified processes: " + + "; ".join(refusals) + ) + manifest["state"] = "error" + manifest.setdefault("cleanup_errors", []).append(str(refusal)) + try: + save_manifest(manifest) + except Exception as save_exc: + raise RamdiskError( + "%s; could not persist stop recovery: %s" + % (refusal, save_exc) + ) from refusal + raise refusal + if identities: + _bind_recovery_usage_transactions( + manifest, + [record for record, _, _, _ in identities], + plan=plan, + bind_usage_transaction=bind_usage_transaction, + ) + # Resolve every ordinary process authority together and make the whole + # set durable before retained recovery, replay, or the first signal. + save_manifest(manifest) + if _retained_process_recovery(manifest): + manifest = _reconcile_unpublished_processes( + manifest, + group_alive=group_alive, + discover_managed_launches=discover_managed_launches, + merge_usage=merge_usage, + save_manifest=save_manifest, + ) + observed_changed = False + for preflight in pending_preflights: + observed_group = preflight.get("observed_group") + entry = preflight["entry"] + if preflight.get("live") and entry.get("observed_group") != observed_group: + entry["observed_group"] = observed_group + observed_changed = True + if observed_changed: + try: + # The only signalable group identity must be durable before the + # first signal. A crash after this point can therefore prove that + # exact group's absence before accounting is merged. + save_manifest(manifest) + except BaseException as save_exc: + raise RamdiskError( + "could not persist pending launch process-group identity: %s" + % save_exc + ) from save_exc + failures = [] + for preflight in pending_preflights: + entry = preflight["entry"] + label = "pending launch on port %s (node %s)" % ( + entry.get("port", "unknown"), + entry.get("node") if entry.get("node") is not None else "shared", + ) + observed_group = entry.get("observed_group") + termination_failure = None + if preflight.get("live"): + try: + termination_failure = terminate_verified_group( + preflight["record"] + ) + except Exception as termination_exc: + termination_failure = ( + "pending process-group termination revalidation failed: %s" + % termination_exc + ) + + try: + remaining_candidates = discover_managed_launches( + nonce=entry["nonce"], + uid=entry["uid"], + state_dir=entry["state_dir"], + weights_dir=entry["weights_dir"], + not_before_starttime=entry["launch_not_before"], + launcher_pid=entry["launcher_pid"], + launcher_starttime=entry["launcher_starttime"], + launcher_cmdline=entry["launcher_cmdline"], + expected_command=entry["expected_command"], + ) + except Exception as discovery_exc: + remaining_candidates = None + recovery_failure = ( + "%s post-termination discovery failed: %s" + % (label, discovery_exc) + ) + else: + recovery_failure = None + if remaining_candidates is not None and not isinstance( + remaining_candidates, + list, + ): + recovery_failure = ( + "%s post-termination discovery returned an invalid result" + % label + ) + elif remaining_candidates: + recovery_failure = ( + "%s still has nonce-attributable processes after recovery" + % label + ) + if observed_group is not None: + try: + observed_group_alive = group_alive(observed_group["pgid"]) + except Exception as group_exc: + observed_group_alive = None + recovery_failure = ( + "%s observed group absence check failed: %s" + % (label, group_exc) + ) + if observed_group_alive is not False: + recovery_failure = recovery_failure or ( + "%s observed process group %s remains live or its " + "absence is unproven" + % (label, observed_group["pgid"]) + ) + if recovery_failure: + if termination_failure: + recovery_failure = "%s; %s" % ( + termination_failure, + recovery_failure, + ) + entry["recovery_error"] = recovery_failure + failures.append(recovery_failure) + continue + + # Positive absence supersedes a benign termination race. The strict + # second process-table scan plus any persisted PGID absence is the + # authority for whether accounting may now be merged. + entry.pop("recovery_error", None) + try: + # Always replay the stable transaction. The canonical merge marker + # is authoritative; a timestamp alone must never suppress a + # missing accounting write after a corrupt or partial snapshot. + merge_usage( + entry, + canonical_usage, + plan=plan, + ) + if not entry.get("usage_merged_at"): + entry["usage_merged_at"] = _utc_now() + save_manifest(manifest) + except Exception as merge_exc: + recovery_failure = ( + "%s usage delta was not durably reconciled: %s" + % (label, merge_exc) + ) + entry["recovery_error"] = recovery_failure + failures.append(recovery_failure) + continue + + previous_pending = list(manifest.get("pending_launches", [])) + manifest["pending_launches"] = [ + pending + for pending in previous_pending + if pending.get("operation_id") != entry.get("operation_id") + ] + try: + save_manifest(manifest) + except Exception as remove_exc: + manifest["pending_launches"] = previous_pending + recovery_failure = ( + "%s was reconciled but pending authority could not be " + "removed: %s" % (label, remove_exc) + ) + entry["recovery_error"] = recovery_failure + failures.append(recovery_failure) + + for record, matches, reason, actual in identities: + if matches: + pgid = int(record.get("pgid", record["pid"])) + try: + failure = terminate_verified_group(record) + except BaseException as termination_exc: + failure = ( + "PID/PGID %s termination revalidation failed: %s" + % (pgid, termination_exc) + ) + try: + post_matches, post_reason, _ = process_matches(record) + except BaseException as identity_exc: + post_matches, post_reason = ( + False, + "identity-check-failed: %s" % identity_exc, + ) + try: + child_alive = managed_child_liveness(record["pid"]) + child_liveness_failure = None + except BaseException as child_exc: + child_alive = True + child_liveness_failure = ( + "retained-child liveness check failed: %s" % child_exc + ) + absence_proven = ( + not post_matches + and post_reason == "not-running" + and child_alive is not True + ) + if not absence_proven: + retained_failure = "; ".join( + item + for item in ( + str(failure or ""), + "PID/PGID %s absence is unproven after termination " + "(%s)" % (pgid, post_reason), + str(child_liveness_failure or ""), + ) + if item + ) + record["stop_error"] = retained_failure + failures.append(retained_failure) + continue + record.pop("stop_error", None) + try: + # Always replay the stable transaction. The canonical merge-id + # marker is authoritative and makes this idempotent; a timestamp + # alone must never suppress a missing accounting write. + merge_usage( + record, + canonical_usage, + plan=plan, + ) + if not record.get("usage_merged_at"): + record["usage_merged_at"] = _utc_now() + record.pop("usage_merge_error", None) + except Exception as exc: + record["usage_merge_error"] = str(exc) + failures.append( + "PID %s usage delta was not merged: %s" + % (record.get("pid"), exc) + ) + else: + try: + # Every node is its own committed transaction. If this + # intermediate write fails, the final recovery write (or a + # fresh retry using the same stable transaction id) can + # publish the already-applied marker without reapplying it. + save_manifest(manifest) + except Exception as exc: + failures.append( + "PID %s usage merge completed but its manifest marker " + "could not be persisted: %s" + % (record.get("pid"), exc) + ) + record.setdefault("stopped_at", _utc_now()) + record.pop("stop_error", None) + planned_paths = { + record["path"] + for record in plan["mounts"] + } + recorded_paths = { + record["path"] + for record in manifest.get("mounts", []) + } + incomplete_mount_layout = recorded_paths != planned_paths + manifest["state"] = ( + "error" + if ( + failures + or incomplete_mount_layout + or any( + record.get("stop_error") + or record.get("usage_merge_error") + for record in manifest.get("processes", []) + ) + ) + else "stopped" + ) + if manifest["state"] == "stopped": + # A verified recovery reconciles every retained process and leaves no + # stale launch-time error to advertise. Clear the advertised-recovery + # keys so a later status does not report a false attention-required + # state, and so a subsequent start does not inherit stale errors. + manifest.pop("launch_error", None) + manifest.pop("cleanup_errors", None) + manifest.pop("recovery", None) + elif isinstance(manifest.get("cleanup_errors"), list): + manifest["cleanup_errors"] = [ + error + for error in manifest["cleanup_errors"] + if not str(error).startswith(_PENDING_RECOVERY_ERROR_PREFIX) + ] + if not manifest["cleanup_errors"]: + manifest.pop("cleanup_errors", None) + save_manifest(manifest) + if failures: + raise RamdiskError( + "managed engine cleanup is incomplete: " + + "; ".join(failures) + ) + return manifest + + +def destroy( + args, + expected_manifest_token=None, + *, + load_manifest, + save_manifest, + manifest_confirmation_token, + confirm, + stop_action, + mount_table, + path_is_below, + managed_path, + mount_at, + validate_mount, + validate_namespace, + busy_mount_references, + umount_path, + durable_unlink, + manifest_path, +): + manifest = load_manifest(required=True) + if ( + expected_manifest_token is not None + and manifest_confirmation_token(manifest) + != expected_manifest_token + ): + raise RamdiskError( + "RAM workspace changed since review; inspect the active " + "deployment and confirm Destroy again" + ) + confirm( + "Stop engines and unmount all volatile RAM-disk weights?", + bool(getattr(args, "yes", False)), + ) + if ( + manifest.get("processes") + or _retained_process_recovery(manifest) + or _pending_launch_recovery(manifest) + ): + manifest = stop_action(args) + retained_processes = _retained_process_recovery(manifest) + if retained_processes: + raise RamdiskError( + "refusing destroy while unpublished managed-child absence is " + "unproven; inspect recovery.retained_processes" + ) + root = manifest["plan"]["mount_root"] + preserved_mountpoints = [] + all_mounts_verified_here = True + verified_mounts = [] + # Preflight every replica before changing any mount. A foreign or busy + # final node must not leave earlier nodes already unmounted. + planned_mounts = manifest["plan"]["mounts"] + managed_paths = [ + record["path"] + for record in planned_mounts + ] + released_mounts = set() + + def persist_destroy_failure(error): + message = str(error) + retained_mounts = sorted( + set(managed_paths) - released_mounts + ) + manifest["state"] = "error" + manifest["destroy_error"] = message + manifest["recovery"] = { + "operation": "destroy", + "state": "attention-required", + "retained_mounts": retained_mounts, + "released_mounts": sorted(released_mounts), + } + retained_set = set(retained_mounts) + for record in manifest.get("mounts", []): + if record.get("path") in retained_set: + record["cleanup"] = { + "state": "retained", + "error": message, + } + try: + save_manifest(manifest) + except Exception as save_exc: + raise RamdiskError( + "%s; could not persist destroy recovery: %s" + % (message, save_exc) + ) from error + if isinstance(error, RamdiskError): + raise error + raise RamdiskError(message) from error + + recorded_by_path = { + record["path"]: record + for record in manifest.get("mounts", []) + } + pending_mounts = sorted( + record["path"] + for record in manifest.get("mounts", []) + if record.get("ownership") == "pending" + ) + if pending_mounts: + # A hard manager crash can leave a privileged mount helper in flight. + # Path absence is only a momentary observation and cannot prove that + # helper will not publish an untracked tmpfs after this process exits. + persist_destroy_failure(RamdiskError( + "refusing destroy while managed mount helper outcome is unknown " + "for pending path(s): %s" % ", ".join(pending_mounts) + )) + try: + observed_mounts = mount_table() + except Exception as exc: + persist_destroy_failure(exc) + nested_mounts = sorted( + mount["path"] + for mount in observed_mounts + if any( + path_is_below(mount["path"], path) + for path in managed_paths + ) + ) + if nested_mounts: + persist_destroy_failure(RamdiskError( + "refusing managed mount(s) with nested child mounts: %s" + % ", ".join(nested_mounts) + )) + for planned in planned_mounts: + path = planned["path"] + if not managed_path(path, root): + persist_destroy_failure(RamdiskError( + "refusing unsafe managed path: %s" % path + )) + try: + actual = mount_at(path) + except Exception as exc: + persist_destroy_failure(exc) + record = recorded_by_path.get(path) + if actual and ( + record is None + or record.get("ownership") == "pending" + ): + # Without a recorded mount-id/device pair, a surviving mount could + # now be foreign. Retain the recovery manifest for an operator. + persist_destroy_failure(RamdiskError( + "refusing unverified surviving mount at planned path: %s" + % path + )) + if record is None: + all_mounts_verified_here = False + preserved_mountpoints.append(path) + released_mounts.add(path) + continue + expected = record.get("identity") or {} + if actual: + if not _same_managed_mount(actual, expected): + persist_destroy_failure(RamdiskError( + "refusing foreign or replaced mount: %s" + % path + )) + try: + validated = validate_mount( + record, + manifest["plan"], + ) + except RamdiskError as exc: + persist_destroy_failure(RamdiskError( + "refusing foreign or altered mount at %s: %s" + % (path, exc) + )) + except Exception as exc: + persist_destroy_failure(exc) + if not _same_managed_mount(validated, expected): + persist_destroy_failure(RamdiskError( + "refusing foreign or replaced mount: %s" + % path + )) + if manifest.get("state") in ("ready", "stopped"): + try: + validate_namespace( + manifest["plan"], + record, + sample_numa=False, + ) + except Exception as exc: + persist_destroy_failure(exc) + try: + busy = busy_mount_references( + path, + hardware=manifest["plan"]["hardware"], + ) + except Exception as exc: + persist_destroy_failure(exc) + if busy: + persist_destroy_failure(RamdiskError( + "mount %s is busy in PID(s): %s" + % ( + path, + ",".join(str(pid) for pid in busy), + ) + )) + verified_mounts.append(record) + else: + # An externally unmounted path no longer has an identity we can + # prove. Never remove it based only on serialized metadata. + all_mounts_verified_here = False + preserved_mountpoints.append(path) + released_mounts.add(path) + continue + + for record in reversed(verified_mounts): + path = record["path"] + expected = record.get("identity") or {} + # The initial sweep prevents known partial teardown. Re-read every + # safety predicate again at the latest possible boundary so a mount + # replacement, nested mount, or new busy reference cannot ride a stale + # preflight into a pathname-based unmount. + try: + latest = mount_at(path) + except BaseException as exc: + persist_destroy_failure(exc) + if not _same_managed_mount(latest, expected): + persist_destroy_failure(RamdiskError( + "refusing foreign or replaced mount immediately before " + "unmount: %s" % path + )) + try: + latest_validated = validate_mount(record, manifest["plan"]) + except BaseException as exc: + persist_destroy_failure(exc) + if not _same_managed_mount(latest_validated, expected): + persist_destroy_failure(RamdiskError( + "refusing foreign or replaced mount immediately before " + "unmount: %s" % path + )) + try: + latest_table = mount_table() + except BaseException as exc: + persist_destroy_failure(exc) + latest_nested = sorted( + item["path"] + for item in latest_table + if isinstance(item, dict) + and isinstance(item.get("path"), str) + and path_is_below(item["path"], path) + ) + if latest_nested: + persist_destroy_failure(RamdiskError( + "refusing managed mount with nested child mounts " + "immediately before unmount at %s: %s" + % (path, ", ".join(latest_nested)) + )) + try: + latest_busy = busy_mount_references( + path, + hardware=manifest["plan"]["hardware"], + ) + except BaseException as exc: + persist_destroy_failure(exc) + if latest_busy: + persist_destroy_failure(RamdiskError( + "mount %s became busy before unmount in PID(s): %s" + % ( + path, + ",".join(str(pid) for pid in latest_busy), + ) + )) + try: + post_busy_identity = mount_at(path) + post_busy_validated = validate_mount( + record, + manifest["plan"], + ) + except BaseException as exc: + persist_destroy_failure(exc) + if ( + not _same_managed_mount(post_busy_identity, expected) + or not _same_managed_mount(post_busy_validated, expected) + ): + persist_destroy_failure(RamdiskError( + "refusing foreign or replaced mount after busy scan " + "immediately before unmount: %s" % path + )) + try: + umount_path( + path, + manifest["plan"]["hardware"], + ) + except BaseException as exc: + persist_destroy_failure(exc) + try: + after_unmount = mount_at(path) + except BaseException as exc: + persist_destroy_failure(exc) + if after_unmount is not None: + persist_destroy_failure(RamdiskError( + "mount remains or was replaced after unmount helper at %s" + % path + )) + released_mounts.add(path) + if record.get("path_preexisting"): + preserved_mountpoints.append(path) + continue + try: + os.rmdir(path) + except FileNotFoundError: + pass + except OSError as exc: + if exc.errno in (errno.EACCES, errno.EPERM): + # X-mount.mkdir creates /mnt targets as root. Sudo stays + # deliberately limited to umount. + preserved_mountpoints.append(path) + continue + if exc.errno not in ( + errno.ENOTEMPTY, + errno.EBUSY, + ): + persist_destroy_failure(exc) + persist_destroy_failure(RamdiskError( + "refusing to remove non-empty mount path: %s" + % path + )) + if ( + manifest["plan"]["topology"] == "per-node" + and all_mounts_verified_here + and not manifest["plan"].get( + "mount_root_preexisting", + True, + ) + ): + try: + os.rmdir(root) + except FileNotFoundError: + pass + except OSError: + pass + try: + durable_unlink(manifest_path()) + except Exception as exc: + persist_destroy_failure(exc) + return { + "destroyed": True, + "durable_state_preserved": True, + "benchmark_history_preserved": True, + "empty_mountpoints_preserved": sorted( + set(preserved_mountpoints) + ), + } + + +def status( + deep=True, + *, + load_manifest, + manifest_path, + source_still_matches, + mount_at, + validate_mount, + validate_namespace, + process_matches, + managed_child_liveness, +): + """Return lifecycle status, optionally skipping shard/header revalidation. + + Scriptable ``status`` always uses the deep default. The curses dashboard + polls the cheap form and exposes an explicit refresh for a new deep model + scan, avoiding repeated reads of every safetensors header on large models. + """ + manifest = load_manifest(required=False) + result = { + "schema": STATUS_SCHEMA, + "version": MANIFEST_VERSION, + "manifest_path": manifest_path(), + "present": bool(manifest), + "state": ( + "absent" + if not manifest + else manifest.get("state", "unknown") + ), + "deep_validation": bool(deep), + "mounts": [], + "processes": [], + "recovery": None, + } + if not manifest: + return result + recovery = manifest.get("recovery") + retained_processes = _retained_process_recovery(manifest) + pending_launches = _pending_launch_recovery(manifest) + planned_mount_paths = { + record.get("path") + for record in manifest.get("plan", {}).get("mounts", []) + if isinstance(record, dict) and isinstance(record.get("path"), str) + } + serialized_retained_mounts = ( + recovery.get("retained_mounts", []) + if isinstance(recovery, dict) + and isinstance(recovery.get("retained_mounts", []), list) + else [] + ) + serialized_released_mounts = ( + recovery.get("released_mounts", []) + if isinstance(recovery, dict) + and isinstance(recovery.get("released_mounts", []), list) + else [] + ) + ownership_recovery_mounts = [ + record.get("path") + for record in manifest.get("mounts", []) + if ( + isinstance(record, dict) + and record.get("ownership", "managed") != "managed" + and isinstance(record.get("path"), str) + ) + ] + retained_mounts = sorted( + set( + path + for path in serialized_retained_mounts + if isinstance(path, str) and path in planned_mount_paths + ) + | set(ownership_recovery_mounts) + ) + released_mounts = sorted( + path + for path in serialized_released_mounts + if isinstance(path, str) and path in planned_mount_paths + ) + cleanup_errors = manifest.get("cleanup_errors", []) + if not isinstance(cleanup_errors, list): + cleanup_errors = [str(cleanup_errors)] + stop_errors = [ + { + "pid": record.get("pid"), + "error": str( + record.get("stop_error") + or record.get("usage_merge_error") + ), + } + for record in manifest.get("processes", []) + if record.get("stop_error") or record.get("usage_merge_error") + ] + error_summary = { + key: str(manifest[key]) + for key in ("error", "launch_error", "destroy_error") + if manifest.get(key) + } + if cleanup_errors: + error_summary["cleanup_errors"] = [ + str(item) for item in cleanup_errors + ] + if stop_errors: + error_summary["stop_errors"] = stop_errors + if ( + isinstance(recovery, dict) + or pending_launches + or retained_mounts + or error_summary + ): + result["recovery"] = { + "operation": ( + recovery.get("operation") + if isinstance(recovery, dict) + else "prepare" + if ownership_recovery_mounts + else "start" + if pending_launches + else None + ), + "state": ( + "attention-required" + if ( + pending_launches + or retained_processes + or ownership_recovery_mounts + ) + else recovery.get("state") + if isinstance(recovery, dict) + else "attention-required" + ), + "retained_mounts": retained_mounts, + "released_mounts": released_mounts, + "retained_processes": [ + { + "pid": entry.get("pid"), + "pgid": entry.get("pgid", entry.get("pid")), + "state_dir": entry.get("state_dir"), + "error": entry.get("error"), + } + for entry in retained_processes + if isinstance(entry, dict) + ], + "pending_launches": [ + { + "port": entry.get("port"), + "node": entry.get("node"), + "state_dir": entry.get("state_dir"), + "state": "outcome-unknown", + } + for entry in pending_launches + if isinstance(entry, dict) + ], + "errors": error_summary, + "action": ( + "Run `coli ramdisk stop` to discover, stop, and reconcile " + "the outcome-unknown pending launch." + if pending_launches + else "Run `coli ramdisk stop` after the retained process " + "group exits to reconcile its exact usage transaction." + if retained_processes + else "Inspect pending ownership, the retained mount identity, " + "nested mounts, and busy references; explicitly reconcile " + "any uncertainty, then retry `coli ramdisk destroy` only " + "after confirming it is safe." + if retained_mounts + else None + ), + } + source_verified = None if not deep else True + source_error = None + if deep: + try: + source_still_matches(manifest["plan"]) + except Exception as exc: + source_verified = False + source_error = str(exc) + for record in manifest.get("mounts", []): + try: + actual = mount_at(record["path"]) + mount_read_error = None + except Exception as mount_exc: + actual = None + mount_read_error = str(mount_exc) + expected = record.get("identity", {}) + identity_verified = bool( + actual + and actual["filesystem"] == "tmpfs" + and actual["source"] == "tmpfs" + and actual["mount_id"] + == expected.get("mount_id") + and actual["device"] + == expected.get("device") + ) + options_verified = False + namespace_verified = None if not deep else False + option_error = mount_read_error + namespace_error = None + if identity_verified: + try: + validate_mount(record, manifest["plan"]) + options_verified = True + except RamdiskError as exc: + option_error = str(exc) + if deep and options_verified and source_verified: + try: + validate_namespace( + manifest["plan"], + record, + sample_numa=False, + ) + namespace_verified = True + except (OSError, RamdiskError) as exc: + namespace_error = str(exc) + result["mounts"].append( + { + "path": record["path"], + "node": record.get("node"), + "ownership": record.get("ownership", "managed"), + "mounted": None if mount_read_error is not None else bool(actual), + "verified": ( + identity_verified + and options_verified + and ( + namespace_verified + if deep + else True + ) + ), + "identity_verified": identity_verified, + "options_verified": options_verified, + "namespace_verified": namespace_verified, + "option_error": option_error, + "namespace_error": namespace_error, + "filesystem": ( + actual.get("filesystem") + if actual + else None + ), + "numa_allocation": record.get( + "numa_allocation", + {}, + ), + } + ) + for record in manifest.get("processes", []): + try: + matches, reason, _ = process_matches(record) + except Exception as identity_exc: + matches, reason = ( + False, + "identity-check-failed: %s" % identity_exc, + ) + try: + child_alive = managed_child_liveness(record.get("pid")) + except Exception as child_exc: + child_alive = None + if reason == "not-running": + reason = "retained-child-liveness-check-failed: %s" % child_exc + attention_required = False + if record.get("stopped_at"): + if matches: + reason = "stopped-record-process-group-live" + attention_required = True + elif child_alive is True: + reason = "stopped-record-retained-child-live" + attention_required = True + elif reason != "not-running": + attention_required = True + else: + reason = "stopped" + elif not matches and child_alive is True: + reason = "retained-managed-child-live" + attention_required = True + elif not matches and reason != "not-running": + attention_required = True + result["processes"].append( + { + "pid": record.get("pid"), + "port": record.get("port"), + "node": record.get("node"), + "running": bool(matches or child_alive is True), + "verified": matches, + "reason": reason, + "attention_required": attention_required, + "state_dir": record.get("state_dir"), + "log": record.get("log"), + } + ) + result["model_fingerprint"] = manifest.get( + "model_fingerprint" + ) + result["mode"] = manifest["plan"].get("mode") + result["topology"] = manifest["plan"].get("topology") + result["ports"] = manifest.get("ports", []) + result["source_fingerprint_verified"] = source_verified + result["source_fingerprint_error"] = source_error + return result diff --git a/c/ramdisk_support/linux_ops.py b/c/ramdisk_support/linux_ops.py new file mode 100644 index 000000000..8c3e4f2bb --- /dev/null +++ b/c/ramdisk_support/linux_ops.py @@ -0,0 +1,2418 @@ +"""Linux filesystem and kernel operations used by RAM-disk discovery.""" + +from __future__ import print_function + +import contextlib +import errno +import os +import platform +import posixpath +import re +import signal +import shutil +import stat +import subprocess +import threading + +from .common import RamdiskError, _parse_range_list +from .platform_ops import ( + UNSUPPORTED_PLATFORM_REASON, + current_euid, + current_uid, + get_platform_ops, +) + + +def _read_text(path, default=""): + try: + with open(path, "r", encoding="utf-8", errors="replace") as stream: + return stream.read() + except OSError: + return default + + +def _read_proc_stat(path): + """Read a task stat record without losing arbitrary ``comm`` bytes.""" + with open( + path, + "r", + encoding="utf-8", + errors="surrogateescape", + newline="", + ) as stream: + return stream.read() + + +def _status_allowed_list(field, fallback): + """Read the kernel's effective task mask from ``/proc/self/status``.""" + status = _read_text("/proc/self/status") + match = re.search(r"^%s:\s*(.*?)\s*$" % re.escape(field), status, re.MULTILINE) + if match: + try: + return _parse_range_list(match.group(1)) + except (TypeError, ValueError): + pass + return sorted(set(int(value) for value in fallback)) + + +def _thread_sibling_groups(cpus): + """Return physical-core sibling groups clipped to the supplied CPU mask.""" + remaining = set(int(cpu) for cpu in cpus) + groups = [] + while remaining: + cpu = min(remaining) + siblings_text = _read_text( + "/sys/devices/system/cpu/cpu%d/topology/thread_siblings_list" % cpu, + str(cpu), + ) + try: + siblings = set(_parse_range_list(siblings_text)) & set(cpus) + except ValueError: + siblings = {cpu} + if not siblings: + siblings = {cpu} + groups.append(sorted(siblings)) + remaining.difference_update(siblings) + return groups + + +def _meminfo(path="/proc/meminfo"): + values = {} + for line in _read_text(path).splitlines(): + match = re.match(r"^([^:]+):\s*(\d+)(?:\s+kB)?", line) + if match: + values[match.group(1)] = int(match.group(2)) * 1024 + return values + + +def _read_cgroup_value(path): + """Read one controller file, distinguishing absence from access failure.""" + try: + with open(path, "r", encoding="utf-8", errors="strict") as stream: + return stream.read().strip() + except FileNotFoundError: + return None + except OSError as exc: + raise RamdiskError("cannot read cgroup controller file %s: %s" % (path, exc)) + + +def _read_cgroup_contract(path): + """Read a procfs cgroup contract without treating denial as absence.""" + try: + with open(path, "r", encoding="utf-8", errors="strict") as stream: + return stream.read() + except (OSError, UnicodeError) as exc: + raise RamdiskError("cannot read cgroup contract %s: %s" % (path, exc)) + + +def _node_meminfo(node): + values = {} + path = "/sys/devices/system/node/node%d/meminfo" % node + for line in _read_text(path).splitlines(): + match = re.search(r"Node\s+\d+\s+([^:]+):\s*(\d+)\s+kB", line) + if match: + values[match.group(1)] = int(match.group(2)) * 1024 + return values + + +def _physical_cores(cpus): + cores = set() + for cpu in cpus: + base = "/sys/devices/system/cpu/cpu%d/topology" % cpu + package = _read_text(os.path.join(base, "physical_package_id"), "0").strip() + core = _read_text(os.path.join(base, "core_id"), str(cpu)).strip() + cores.add((package, core)) + return max(1, len(cores)) + + +def _kernel_at_least(major, minor): + match = re.match(r"^(\d+)\.(\d+)", platform.release()) + return bool(match and (int(match.group(1)), int(match.group(2))) >= (major, minor)) + + +def _require_linux(): + if not get_platform_ops().is_linux: + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + + +def _unescape_mount(value): + return re.sub( + r"\\([0-7]{3})", + lambda match: chr(int(match.group(1), 8)), + value, + ) + + +def _split_mount_options(value): + """Split mountinfo options while preserving comma-bearing mpol masks.""" + result = [] + for token in value.split(","): + if ( + result + and result[-1].startswith("mpol=") + and re.fullmatch(r"\d+(?:-\d+)?", token) + ): + result[-1] += "," + token + else: + result.append(token) + return result + + +def _mount_table(path="/proc/self/mountinfo"): + if path == "/proc/self/mountinfo": + _require_linux() + try: + with open(path, "r", encoding="utf-8", errors="strict") as stream: + mountinfo = stream.read() + except (OSError, UnicodeError) as exc: + raise RamdiskError( + "cannot read Linux mount table %s: %s" % (path, exc) + ) from exc + result = [] + for line_number, line in enumerate(mountinfo.splitlines(), 1): + if not line.strip(): + continue + fields = line.split() + try: + separator = fields.index("-") + if separator < 6 or len(fields) <= separator + 3: + raise ValueError("incomplete mountinfo record") + result.append( + { + "mount_id": int(fields[0]), + "parent_id": int(fields[1]), + "device": fields[2], + "root": _unescape_mount(fields[3]), + "path": _unescape_mount(fields[4]), + "options": sorted( + set(_split_mount_options(fields[5])) + ), + "optional": fields[6:separator], + "filesystem": fields[separator + 1], + "source": _unescape_mount(fields[separator + 2]), + "super_options": sorted( + set(_split_mount_options(fields[separator + 3])) + ), + } + ) + except (ValueError, IndexError) as exc: + raise RamdiskError( + "cannot parse Linux mount table %s line %d: %s" + % (path, line_number, exc) + ) from exc + return result + + +def _mount_at(path, *, mount_table=None): + mount_table = _mount_table if mount_table is None else mount_table + path = posixpath.normpath(posixpath.abspath(path)) + matches = [ + mount + for mount in mount_table() + if posixpath.normpath(mount["path"]) == path + ] + if len(matches) > 1: + raise RamdiskError( + "refusing ambiguous stacked mounts at %s (mount ids %s)" + % ( + path, + ", ".join(str(item["mount_id"]) for item in matches), + ) + ) + return matches[0] if matches else None + + +def _filesystem_for_path(path, *, mount_table=None): + """Return the filesystem of the longest mountpoint containing ``path``.""" + mount_table = _mount_table if mount_table is None else mount_table + normalized = posixpath.normpath(posixpath.abspath(path)) + matches = [] + for mount in mount_table(): + root = posixpath.normpath(mount["path"]) + try: + contained = posixpath.commonpath([normalized, root]) == root + except ValueError: + contained = False + if contained: + matches.append((len(root), root, mount)) + if not matches: + return None + longest = max(item[0] for item in matches) + nearest = [item for item in matches if item[0] == longest] + if len(nearest) > 1: + raise RamdiskError( + "refusing ambiguous stacked mounts at %s (mount ids %s)" + % ( + nearest[0][1], + ", ".join( + str(item[2]["mount_id"]) + for item in nearest + ), + ) + ) + return nearest[0][2]["filesystem"] + + +def _run(command, **kwargs): + kwargs.setdefault("text", True) + kwargs.setdefault("capture_output", True) + kwargs.setdefault("check", False) + return subprocess.run(command, **kwargs) + + +def _current_gid(): + getgid = getattr(os, "getgid", None) + return int(getgid()) if getgid is not None else current_uid() + + +def _trusted_system_binary(name): + """Resolve a fixed system executable safe to place after ``sudo --``.""" + _require_linux() + candidates = [ + os.path.join(prefix, name) + for prefix in ( + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/run/current-system/sw/bin", + "/run/wrappers/bin", + "/nix/var/nix/profiles/default/bin", + ) + ] + discovered = shutil.which(name) + if discovered: + candidates.append(discovered) + system_uid = os.stat("/").st_uid + rejected = [] + for candidate in candidates: + path = os.path.realpath(candidate) + if os.path.basename(path) != name: + rejected.append(candidate) + continue + try: + info = os.stat(path) + except OSError: + continue + if ( + not stat.S_ISREG(info.st_mode) + or info.st_mode & 0o022 + or info.st_uid != system_uid + ): + rejected.append(path) + continue + parent = os.path.dirname(path) + child_info = info + unsafe_parent = False + while True: + parent_info = os.stat(parent) + sticky_protects_child = bool( + parent_info.st_uid == system_uid + and parent_info.st_mode & stat.S_ISVTX + and child_info.st_uid == system_uid + and not child_info.st_mode & 0o022 + ) + if ( + not stat.S_ISDIR(parent_info.st_mode) + or parent_info.st_uid != system_uid + or parent_info.st_mode & stat.S_IWOTH + or ( + parent_info.st_mode & stat.S_IWGRP + and not sticky_protects_child + ) + or ( + current_euid() != 0 + and os.access(parent, os.W_OK) + and not sticky_protects_child + ) + ): + unsafe_parent = True + break + next_parent = os.path.dirname(parent) + if next_parent == parent: + break + child_info = parent_info + parent = next_parent + if unsafe_parent: + rejected.append(path) + continue + return path + detail = ( + " (rejected writable candidates: %s)" % ", ".join(rejected) + if rejected + else "" + ) + raise RamdiskError( + "trusted %s executable was not found%s" % (name, detail) + ) + + +def _fresh_user_binary(name): + """Resolve an unprivileged helper now, never from serialized manifest data.""" + path = shutil.which(name) + if ( + not path + or os.path.basename(path) != name + or not os.access(path, os.X_OK) + ): + raise RamdiskError("%s was not found on PATH" % name) + return os.path.realpath(path) + + +_privilege_local = threading.local() + + +def _validate_noninteractive_sudo(sudo): + """Confirm the foreground authorization can be reused without a prompt.""" + return subprocess.run( + [sudo, "-n", "-v"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + + +def _sudo_ticket_keepalive( + stop_event, + sudo, + interval=1.0, + failure_event=None, + cancel_event=None, +): + while not stop_event.is_set(): + try: + result = subprocess.run( + [sudo, "-n", "-v"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=5.0, + ) + except (OSError, subprocess.SubprocessError): + result = None + if result is None or result.returncode: + if failure_event is not None: + failure_event.set() + if cancel_event is not None: + cancel_event.set() + return + if stop_event.wait(interval): + return + + +@contextlib.contextmanager +def _noninteractive_privilege( + keepalive=False, + cancel_event=None, + *, + trusted_system_binary=None, + sudo_ticket_keepalive=None, +): + trusted_system_binary = ( + _trusted_system_binary + if trusted_system_binary is None + else trusted_system_binary + ) + sudo_ticket_keepalive = ( + _sudo_ticket_keepalive + if sudo_ticket_keepalive is None + else sudo_ticket_keepalive + ) + previous = getattr(_privilege_local, "noninteractive", False) + _privilege_local.noninteractive = True + keepalive_stop = None + keepalive_thread = None + try: + if keepalive and not previous and current_euid() != 0: + sudo = trusted_system_binary("sudo") + keepalive_stop = threading.Event() + keepalive_failure = threading.Event() + keepalive_thread = threading.Thread( + target=sudo_ticket_keepalive, + args=( + keepalive_stop, + sudo, + 1.0, + keepalive_failure, + cancel_event, + ), + name="coli-sudo-ticket-keepalive", + daemon=True, + ) + keepalive_thread.start() + yield + finally: + if keepalive_stop is not None: + keepalive_stop.set() + if keepalive_thread is not None: + keepalive_thread.join(timeout=6.0) + _privilege_local.noninteractive = previous + + +def _privileged(command, hardware, *, trusted_system_binary=None): + del hardware + if current_euid() == 0: + return command + trusted_system_binary = ( + _trusted_system_binary + if trusted_system_binary is None + else trusted_system_binary + ) + sudo = trusted_system_binary("sudo") + options = ( + ["-n"] + if getattr(_privilege_local, "noninteractive", False) + else [] + ) + return [sudo] + options + ["--"] + command + + +def _process_start_boundary(): + """Return a conservative current process start time in boot ticks.""" + _require_linux() + try: + ticks_per_second = os.sysconf("SC_CLK_TCK") + except (OSError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot read Linux process clock tick rate: %s" % exc + ) from exc + if ( + not isinstance(ticks_per_second, int) + or isinstance(ticks_per_second, bool) + or ticks_per_second <= 0 + ): + raise RamdiskError("Linux process clock tick rate is invalid") + + uptime_path = "/proc/uptime" + try: + with open( + uptime_path, + "r", + encoding="ascii", + errors="strict", + newline="", + ) as stream: + raw = stream.read() + except (OSError, UnicodeError) as exc: + raise RamdiskError( + "cannot read Linux boot uptime %s: %s" % (uptime_path, exc) + ) from exc + + fields = raw.split() if isinstance(raw, str) else [] + match = ( + re.fullmatch(r"([0-9]+)(?:\.([0-9]+))?", fields[0]) + if len(fields) == 2 + else None + ) + if match is None: + raise RamdiskError("cannot parse Linux boot uptime %s" % uptime_path) + whole, fraction = match.groups() + fraction = fraction or "" + scale = 10 ** len(fraction) + uptime_units = int(whole, 10) * scale + int(fraction or "0", 10) + # Floor rather than round: a process starting in the current partially + # observed tick remains at/after this boundary and therefore receives the + # strict attribution checks. + return uptime_units * ticks_per_second // scale + + +def _strict_process_identity(pid): + """Read one complete process identity or report why it is unreadable.""" + _require_linux() + getpgid = getattr(os, "getpgid", None) + if getpgid is None: + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + pid = int(pid) + proc_path = "/proc/%d" % pid + stat_path = "%s/stat" % proc_path + try: + before_uid = os.stat(proc_path).st_uid + except (FileNotFoundError, ProcessLookupError): + return None + except OSError as exc: + raise RamdiskError( + "cannot read Linux process owner %s: %s; verified cleanup " + "requires complete process-table visibility" % (proc_path, exc) + ) from exc + before = _strict_proc_stat_identity(pid, stat_path) + if before is None: + return None + if before.get("state") in _INERT_PROCESS_STATES: + inert = _stable_inert_process_identity( + pid, + proc_path, + before_uid, + before, + ) + if inert is None: + return None + return { + "pid": pid, + "uid": before_uid, + "state": inert["state"], + "inert": True, + "starttime": inert["starttime"], + "cmdline": [], + "nonce": None, + "pgid": inert["pgid"], + "sid": inert["sid"], + "state_dir": None, + "weights_dir": None, + } + cmdline_path = "%s/cmdline" % proc_path + cmdline_raw = _read_proc_binary(pid, cmdline_path) + if cmdline_raw is None: + return None + environ_path = "%s/environ" % proc_path + environ = _read_proc_binary(pid, environ_path) + if environ is None: + return None + env = _managed_environment_fields(pid, environ_path, environ) + try: + observed_pgid = getpgid(pid) + except (FileNotFoundError, ProcessLookupError): + return None + except OSError as exc: + raise RamdiskError( + "cannot read Linux process group for PID %d: %s; verified " + "cleanup requires complete process-table visibility" % (pid, exc) + ) from exc + after = _strict_proc_stat_identity(pid, stat_path) + try: + after_uid = os.stat(proc_path).st_uid + except (FileNotFoundError, ProcessLookupError): + return None + except OSError as exc: + raise RamdiskError( + "cannot recheck Linux process owner %s: %s; verified cleanup " + "requires complete process-table visibility" % (proc_path, exc) + ) from exc + if ( + after is None + or before != after + or before_uid != after_uid + or observed_pgid != after["pgid"] + ): + return None + return { + "pid": pid, + "uid": after_uid, + "state": after["state"], + "inert": False, + "starttime": after["starttime"], + "cmdline": [ + value.decode("utf-8", "replace") + for value in cmdline_raw.split(b"\0") + if value + ], + "nonce": env["COLI_MANAGED_NONCE"], + "pgid": after["pgid"], + "sid": after["sid"], + "state_dir": env["COLI_STATE_DIR"], + "weights_dir": env["COLI_WEIGHTS_DIR"], + } + + +def _process_identity(pid): + """Best-effort identity read used by non-destructive status paths.""" + try: + return _strict_process_identity(pid) + except (OSError, ValueError, IndexError, RamdiskError): + return None + + +def _proc_pid_snapshot(): + """Return one unambiguous snapshot of every numeric procfs PID entry.""" + try: + entries = os.listdir("/proc") + except OSError as exc: + raise RamdiskError( + "cannot enumerate Linux process table /proc: %s; pending-launch " + "recovery requires complete process-table visibility" % exc + ) from exc + + snapshot = {} + for entry in entries: + if not isinstance(entry, str) or re.fullmatch(r"[0-9]+", entry) is None: + continue + pid = int(entry) + if pid <= 0 or entry != str(pid) or pid in snapshot: + raise RamdiskError( + "cannot trust ambiguous Linux process-table entry %r; " + "pending-launch recovery requires complete process-table " + "visibility" % entry + ) + snapshot[pid] = "/proc/%s" % entry + return snapshot + + +def _proc_pid_disappeared(pid, endpoint, missing_error): + """Accept endpoint ENOENT only when the corresponding PID is now absent.""" + proc_path = "/proc/%d" % pid + try: + os.stat(proc_path) + except (FileNotFoundError, ProcessLookupError): + return True + except OSError as exc: + raise RamdiskError( + "cannot verify Linux process identity %s after %s disappeared: " + "%s; pending-launch recovery requires complete process-table " + "visibility" % (proc_path, endpoint, exc) + ) from exc + raise RamdiskError( + "cannot read Linux process identity %s while PID %d remains; " + "pending-launch recovery requires complete process-table visibility" + % (endpoint, pid) + ) from missing_error + + +def _strict_proc_stat_identity(pid, path): + """Read stable process-group fields used by nonce-based recovery.""" + try: + raw = _read_proc_stat(path) + except (FileNotFoundError, ProcessLookupError) as exc: + if _proc_pid_disappeared(pid, path, exc): + return None + except (OSError, UnicodeError) as exc: + raise RamdiskError( + "cannot read Linux process identity %s: %s; pending-launch " + "recovery requires complete process-table visibility" % (path, exc) + ) from exc + + opening = raw.find("(") if isinstance(raw, str) else -1 + closing = raw.rfind(")") if isinstance(raw, str) else -1 + fields = raw[closing + 2 :].split() if closing >= 0 else [] + try: + declared_pid = int(raw[:opening].strip()) + state = fields[0] + pgid = int(fields[2], 10) + session = int(fields[3], 10) + num_threads = int(fields[17], 10) + starttime = int(fields[19], 10) + except (IndexError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot parse Linux process identity %s; pending-launch recovery " + "requires complete process-table visibility" % path + ) from exc + if ( + opening <= 0 + or closing <= opening + or declared_pid != pid + or len(state) != 1 + or pgid < 0 + or session < 0 + or num_threads <= 0 + or starttime < 0 + ): + raise RamdiskError( + "cannot parse Linux process identity %s; pending-launch recovery " + "requires complete process-table visibility" % path + ) + return { + "state": state, + "starttime": starttime, + "pgid": pgid, + "sid": session, + "num_threads": num_threads, + } + + +def _read_proc_binary(pid, path): + try: + with open(path, "rb") as stream: + value = stream.read() + except (FileNotFoundError, ProcessLookupError) as exc: + if _proc_pid_disappeared(pid, path, exc): + return None + except OSError as exc: + raise RamdiskError( + "cannot read Linux process identity %s: %s; pending-launch " + "recovery requires complete process-table visibility" % (path, exc) + ) from exc + if not isinstance(value, bytes): + raise RamdiskError( + "cannot parse Linux process identity %s; pending-launch recovery " + "requires complete process-table visibility" % path + ) + return value + + +def _managed_environment_fields(pid, path, raw): + """Extract launch attribution without rejecting unrelated environment data.""" + fields = { + b"COLI_MANAGED_NONCE": [], + b"COLI_STATE_DIR": [], + b"COLI_WEIGHTS_DIR": [], + } + malformed = set() + for item in raw.split(b"\0"): + if not item: + continue + if b"=" not in item: + if item in fields: + malformed.add(item) + # Linux permits arbitrary strings in ``environ``. Firefox and + # other ordinary processes use entries without ``=``; those say + # nothing about a Colibri launch and must not wedge recovery. + continue + key, value = item.split(b"=", 1) + if key in fields: + fields[key].append(value) + + result = {} + candidates = {} + ambiguous = set() + for key, values in fields.items(): + label = key.decode("ascii") + decoded = [] + invalid_value = False + for value in values: + try: + decoded.append(value.decode("utf-8", "strict")) + except UnicodeError: + invalid_value = True + candidates[label] = tuple(decoded) + if key in malformed or len(values) != 1 or invalid_value: + ambiguous.add(label) + result[label] = None + else: + result[label] = decoded[0] + result["_candidates"] = candidates + result["_ambiguous"] = tuple(sorted(ambiguous)) + return result + + +def _recheck_managed_launch_identity(pid, proc_path, expected_uid, before): + """Recheck one owner/stat pair before trusting its age or contents.""" + stat_path = "%s/stat" % proc_path + after = _strict_proc_stat_identity(pid, stat_path) + if after is None: + return None + try: + after_uid = os.stat(proc_path).st_uid + except (FileNotFoundError, ProcessLookupError): + return None + except OSError as exc: + raise RamdiskError( + "cannot recheck Linux process owner %s: %s; pending-launch " + "recovery requires complete process-table visibility" + % (proc_path, exc) + ) from exc + if after_uid != expected_uid or after != before: + raise RamdiskError( + "Linux process identity %s changed during pending-launch " + "recovery; refusing an unstable process-table view" % proc_path + ) + return after + + +_INERT_PROCESS_STATES = frozenset(("Z", "X", "x")) + + +def _stable_inert_process_identity(pid, proc_path, expected_uid, before): + """Prove a dead process identity has no live sibling task.""" + if before.get("state") not in _INERT_PROCESS_STATES: + raise RamdiskError( + "Linux process identity %s is not an inert dead task" % proc_path + ) + task_path = "%s/task" % proc_path + try: + entries = os.listdir(task_path) + except (FileNotFoundError, ProcessLookupError): + after = _recheck_managed_launch_identity( + pid, + proc_path, + expected_uid, + before, + ) + if after is None: + return None + raise RamdiskError( + "cannot enumerate Linux dead process task group %s while PID %d " + "remains" % (task_path, pid) + ) + except OSError as exc: + raise RamdiskError( + "cannot enumerate Linux dead process task group %s: %s; " + "pending-launch recovery requires complete process-table visibility" + % (task_path, exc) + ) from exc + if any(not entry.isdigit() for entry in entries): + raise RamdiskError( + "cannot parse Linux dead process task group %s" % task_path + ) + task_ids = [int(entry) for entry in entries] + if ( + before.get("num_threads") != 1 + or len(task_ids) != 1 + or set(task_ids) != {pid} + ): + raise RamdiskError( + "Linux dead process PID %d still has an incomplete or live task " + "group; refusing to treat it as inert" % pid + ) + return _recheck_managed_launch_identity( + pid, + proc_path, + expected_uid, + before, + ) + + +def _inspect_managed_launch_pid(pid, expected_uid, not_before_starttime): + """Return a stable same-UID PID observation, or ``None`` if it exited.""" + proc_path = "/proc/%d" % pid + try: + owner = os.stat(proc_path) + except (FileNotFoundError, ProcessLookupError): + return None + except OSError as exc: + raise RamdiskError( + "cannot read Linux process owner %s: %s; pending-launch recovery " + "requires complete process-table visibility" % (proc_path, exc) + ) from exc + owner_uid = getattr(owner, "st_uid", None) + if ( + not isinstance(owner_uid, int) + or isinstance(owner_uid, bool) + or owner_uid < 0 + ): + raise RamdiskError( + "cannot parse Linux process owner %s; pending-launch recovery " + "requires complete process-table visibility" % proc_path + ) + if owner_uid != expected_uid: + return {"kind": "foreign", "uid": owner_uid} + + stat_path = "%s/stat" % proc_path + before = _strict_proc_stat_identity(pid, stat_path) + if before is None: + return None + if before["starttime"] < not_before_starttime: + stable_old = _recheck_managed_launch_identity( + pid, + proc_path, + expected_uid, + before, + ) + if stable_old is None: + return None + return { + "kind": "before-boundary", + "pid": pid, + "uid": expected_uid, + "starttime": stable_old["starttime"], + "pgid": stable_old["pgid"], + "sid": stable_old["sid"], + } + if before.get("state") in _INERT_PROCESS_STATES: + inert = _stable_inert_process_identity( + pid, + proc_path, + expected_uid, + before, + ) + if inert is None: + return None + return { + "kind": "inert-dead", + "pid": pid, + "uid": expected_uid, + "state": inert["state"], + "starttime": inert["starttime"], + "pgid": inert["pgid"], + "sid": inert["sid"], + } + cmdline_path = "%s/cmdline" % proc_path + cmdline_raw = _read_proc_binary(pid, cmdline_path) + if cmdline_raw is None: + return None + environ_path = "%s/environ" % proc_path + environ_raw = _read_proc_binary(pid, environ_path) + if environ_raw is None: + return None + environment = _managed_environment_fields(pid, environ_path, environ_raw) + after = _recheck_managed_launch_identity( + pid, + proc_path, + expected_uid, + before, + ) + if after is None: + return None + return { + "kind": "same-uid", + "pid": pid, + "uid": expected_uid, + "starttime": after["starttime"], + "nonce": environment["COLI_MANAGED_NONCE"], + "pgid": after["pgid"], + "sid": after["sid"], + "cmdline": [ + item.decode("utf-8", "replace") + for item in cmdline_raw.split(b"\0") + if item + ], + "state_dir": environment["COLI_STATE_DIR"], + "weights_dir": environment["COLI_WEIGHTS_DIR"], + "environment_candidates": environment["_candidates"], + "environment_ambiguities": environment["_ambiguous"], + } + + +def _managed_launch_processes( + nonce, + uid, + *, + state_dir, + weights_dir, + not_before_starttime, + launcher_pid, + launcher_starttime, + launcher_cmdline, + expected_command, +): + """Discover every stable process attributable to one pending launch. + + Foreign-UID processes are skipped after a trustworthy procfs ownership + read. Same-UID identities are read repeatedly across independent + process-table snapshots so PID reuse, exec transitions, and new uninspected + PIDs fail closed instead of turning an incomplete scan into a false absence + proof. + """ + _require_linux() + if not isinstance(nonce, str) or not nonce or "\0" in nonce: + raise RamdiskError("managed launch nonce must be a nonempty string") + if not isinstance(uid, int) or isinstance(uid, bool) or uid < 0: + raise RamdiskError("managed launch UID must be a nonnegative integer") + if ( + not isinstance(not_before_starttime, int) + or isinstance(not_before_starttime, bool) + or not_before_starttime < 0 + ): + raise RamdiskError( + "managed launch process-start boundary must be a nonnegative " + "integer" + ) + if ( + not isinstance(launcher_pid, int) + or isinstance(launcher_pid, bool) + or launcher_pid <= 0 + ): + raise RamdiskError( + "managed launch launcher PID must be a positive integer" + ) + if ( + not isinstance(launcher_starttime, int) + or isinstance(launcher_starttime, bool) + or launcher_starttime <= 0 + ): + raise RamdiskError( + "managed launch launcher start time must be a positive integer" + ) + for label, value in ( + ("state directory", state_dir), + ("weights directory", weights_dir), + ): + if not isinstance(value, str) or not value or "\0" in value: + raise RamdiskError("managed launch %s must be a nonempty string" % label) + for label, value in ( + ("launcher command", launcher_cmdline), + ("expected command", expected_command), + ): + if ( + not isinstance(value, list) + or not value + or any(not isinstance(item, str) or not item for item in value) + ): + raise RamdiskError( + "managed launch %s must be a nonempty string list" % label + ) + + initial = _proc_pid_snapshot() + first = {} + for pid in sorted(initial): + observation = _inspect_managed_launch_pid( + pid, + uid, + not_before_starttime, + ) + if observation is not None: + first[pid] = observation + + middle = _proc_pid_snapshot() + unexpected = sorted(set(middle) - set(first)) + if unexpected: + raise RamdiskError( + "Linux process table changed during pending-launch recovery; " + "uninspected PID(s): %s" % ", ".join(str(pid) for pid in unexpected) + ) + + stable = {} + for pid in sorted(set(first) & set(middle)): + observation = _inspect_managed_launch_pid( + pid, + uid, + not_before_starttime, + ) + if observation is None: + continue + if observation != first[pid]: + raise RamdiskError( + "Linux process identity /proc/%d changed during " + "pending-launch recovery; refusing an unstable process-table " + "view" % pid + ) + stable[pid] = observation + + settled = _proc_pid_snapshot() + unexpected = sorted(set(settled) - set(stable)) + if unexpected: + raise RamdiskError( + "Linux process table changed during pending-launch recovery; " + "uninspected PID(s): %s" % ", ".join(str(pid) for pid in unexpected) + ) + + final = {} + for pid in sorted(set(stable) & set(settled)): + observation = _inspect_managed_launch_pid( + pid, + uid, + not_before_starttime, + ) + if observation is None: + continue + if observation != stable[pid]: + raise RamdiskError( + "Linux process identity /proc/%d changed during final " + "pending-launch recovery verification" % pid + ) + final[pid] = observation + + verified = _proc_pid_snapshot() + unexpected = sorted(set(verified) - set(final)) + if unexpected: + raise RamdiskError( + "Linux process table changed after final pending-launch reads; " + "unverified PID(s): %s" % ", ".join(str(pid) for pid in unexpected) + ) + + checked = {} + for pid in sorted(set(final) & set(verified)): + observation = _inspect_managed_launch_pid( + pid, + uid, + not_before_starttime, + ) + if observation is None: + continue + if observation != final[pid]: + raise RamdiskError( + "Linux process identity /proc/%d changed after the final " + "pending-launch snapshot" % pid + ) + checked[pid] = observation + + confirmation_snapshot = _proc_pid_snapshot() + unexpected = sorted(set(confirmation_snapshot) - set(checked)) + if unexpected: + raise RamdiskError( + "Linux process table changed during final pending-launch identity " + "confirmation; unverified PID(s): %s" + % ", ".join(str(pid) for pid in unexpected) + ) + + # The terminal confirmation must carry identities, not only PID names. + # Otherwise a process can exit and the same numeric PID can be reused after + # ``checked`` without changing the final key set. + confirmed = {} + for pid in sorted(set(checked) & set(confirmation_snapshot)): + observation = _inspect_managed_launch_pid( + pid, + uid, + not_before_starttime, + ) + if observation is None: + continue + if observation != checked[pid]: + raise RamdiskError( + "Linux process identity /proc/%d changed during final " + "pending-launch identity confirmation" % pid + ) + confirmed[pid] = observation + + matches = [] + for pid in sorted(confirmed): + observation = confirmed[pid] + if observation.get("kind") in { + "foreign", + "before-boundary", + "inert-dead", + }: + continue + if ( + pid == launcher_pid + and observation.get("starttime") == launcher_starttime + ): + if observation.get("cmdline") != launcher_cmdline: + raise RamdiskError( + "managed launch launcher identity changed command for PID %d" + % pid + ) + continue + candidates = observation.get("environment_candidates") or {} + target_nonce_present = ( + nonce in candidates.get("COLI_MANAGED_NONCE", ()) + ) + target_path_present = ( + state_dir in candidates.get("COLI_STATE_DIR", ()) + or weights_dir in candidates.get("COLI_WEIGHTS_DIR", ()) + ) + if pid == os.getpid() and not ( + target_nonce_present or target_path_present + ): + # The process executing recovery cannot also be an orphaned child + # from the earlier Popen attempt. This narrowly excludes a new + # Textual UI process whose argv matches the persisted launcher. + continue + associated = ( + observation.get("cmdline") in (launcher_cmdline, expected_command) + or target_nonce_present + or target_path_present + ) + if not associated: + continue + if observation.get("environment_ambiguities"): + raise RamdiskError( + "same-UID PID %d has ambiguous managed launch attribution" + % pid + ) + if observation.get("nonce") != nonce: + raise RamdiskError( + "same-UID PID %d has missing or mismatched nonce attribution" + % pid + ) + actual_state_dir = observation.get("state_dir") + actual_weights_dir = observation.get("weights_dir") + if ( + not actual_state_dir + or not actual_weights_dir + or actual_state_dir != state_dir + or actual_weights_dir != weights_dir + ): + raise RamdiskError( + "managed launch PID %d has mismatched state or weights " + "attribution" % pid + ) + if ( + observation["starttime"] <= 0 + or observation["pgid"] <= 0 + or observation["sid"] <= 0 + or observation["pgid"] != observation["sid"] + ): + raise RamdiskError( + "managed launch PID %d violates the new-session process-group " + "identity" % pid + ) + matches.append( + { + key: observation[key] + for key in ( + "pid", + "uid", + "starttime", + "nonce", + "pgid", + "sid", + "cmdline", + "state_dir", + "weights_dir", + ) + } + ) + return matches + + +def _process_group_member_pids(pgid): + """Return procfs PIDs whose stat record reports the requested PGID.""" + _require_linux() + pgid = int(pgid) + members = [] + try: + entries = os.listdir("/proc") + except OSError as exc: + raise RamdiskError( + "cannot enumerate Linux process table /proc: %s; managed " + "cleanup requires complete process-table visibility" % exc + ) from exc + for entry in entries: + if not entry.isdigit(): + continue + pid = int(entry) + stat_path = "/proc/%d/stat" % pid + try: + raw = _read_proc_stat(stat_path) + except (FileNotFoundError, ProcessLookupError): + # Exiting between listdir() and open() is ordinary procfs churn. + continue + except (OSError, UnicodeError) as exc: + raise RamdiskError( + "cannot read Linux process identity %s: %s" + "; managed cleanup requires complete process-table visibility" + % (stat_path, exc) + ) from exc + close = raw.rfind(")") + try: + if close < 0: + raise ValueError("missing process-name terminator") + fields = raw[close + 2 :].split() + member_pgid = int(fields[2]) + except (ValueError, IndexError) as exc: + raise RamdiskError( + "cannot parse Linux process identity %s: %s" + % (stat_path, exc) + ) from exc + if member_pgid == pgid: + members.append(pid) + return sorted(members) + + +def _process_group_alive(pgid): + _require_linux() + killpg = getattr(os, "killpg", None) + if killpg is None: + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + try: + killpg(int(pgid), 0) + except ProcessLookupError: + return False + except PermissionError: + pass + + # ``killpg(..., 0)`` reports a group containing only zombies as alive. + # Such tasks cannot run, retain files, or mutate usage. Prove every member + # is a stable lone-thread dead identity before treating that group as inert. + member_pids = _process_group_member_pids(pgid) + if not member_pids: + try: + killpg(int(pgid), 0) + return True + except ProcessLookupError: + return False + except PermissionError: + return True + inert = {} + for pid in member_pids: + proc_path = "/proc/%d" % pid + try: + owner_uid = os.stat(proc_path).st_uid + except (FileNotFoundError, ProcessLookupError): + continue + except OSError as exc: + raise RamdiskError( + "cannot read Linux process owner %s while checking process " + "group liveness: %s" % (proc_path, exc) + ) from exc + identity = _strict_proc_stat_identity( + pid, + "%s/stat" % proc_path, + ) + if identity is None: + continue + if identity.get("state") not in _INERT_PROCESS_STATES: + return True + stable = _stable_inert_process_identity( + pid, + proc_path, + owner_uid, + identity, + ) + if stable is not None: + inert[pid] = (owner_uid, stable) + + settled = _process_group_member_pids(pgid) + if set(settled) - set(inert): + return True + for pid in sorted(set(settled) & set(inert)): + proc_path = "/proc/%d" % pid + try: + owner_uid = os.stat(proc_path).st_uid + except (FileNotFoundError, ProcessLookupError): + continue + except OSError as exc: + raise RamdiskError( + "cannot recheck Linux process owner %s while checking process " + "group liveness: %s" % (proc_path, exc) + ) from exc + identity = _strict_proc_stat_identity( + pid, + "%s/stat" % proc_path, + ) + if identity is None: + continue + if (owner_uid, identity) != inert[pid]: + return True + return False + + +_PIDFD_REQUIRED_REASON = ( + "verified managed-process cleanup requires Linux pidfd_open and " + "pidfd_send_signal support; use a newer Python runtime or a libc/kernel " + "that provides both pidfd operations" +) + + +def _pidfd_api(pidfd_open=None, pidfd_send_signal=None): + """Resolve pidfd operations lazily, preferring the Python stdlib.""" + open_operation = ( + getattr(os, "pidfd_open", None) + if pidfd_open is None + else pidfd_open + ) + send_operation = ( + getattr(signal, "pidfd_send_signal", None) + if pidfd_send_signal is None + else pidfd_send_signal + ) + if callable(open_operation) and callable(send_operation): + return open_operation, send_operation + + try: + import ctypes + + libc = ctypes.CDLL(None, use_errno=True) + except (ImportError, OSError): + libc = None + + if not callable(open_operation) and libc is not None: + libc_open = getattr(libc, "pidfd_open", None) + if libc_open is not None: + libc_open.argtypes = (ctypes.c_int, ctypes.c_uint) + libc_open.restype = ctypes.c_int + + def open_operation(pid, flags): + result = libc_open(int(pid), int(flags)) + if result < 0: + error_number = ctypes.get_errno() + raise OSError( + error_number, + os.strerror(error_number), + ) + return result + + if not callable(send_operation) and libc is not None: + libc_send = getattr(libc, "pidfd_send_signal", None) + if libc_send is not None: + libc_send.argtypes = ( + ctypes.c_int, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_uint, + ) + libc_send.restype = ctypes.c_int + + def send_operation(pidfd, signum, siginfo, flags): + if siginfo is not None: + raise ValueError( + "verified cleanup does not accept siginfo payloads" + ) + result = libc_send( + int(pidfd), + int(signum), + None, + int(flags), + ) + if result < 0: + error_number = ctypes.get_errno() + raise OSError( + error_number, + os.strerror(error_number), + ) + + if not callable(open_operation) or not callable(send_operation): + raise RamdiskError(_PIDFD_REQUIRED_REASON) + return open_operation, send_operation + + +def _pidfd_process_control_supported(): + """Probe that this runtime and kernel can bind and signal a pidfd.""" + descriptor = None + try: + open_operation, send_operation = _pidfd_api() + descriptor = open_operation(os.getpid(), 0) + send_operation(descriptor, 0, None, 0) + return True + except (OSError, RamdiskError, TypeError, ValueError): + return False + finally: + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + pass + + +def _pidfd_exited(pidfd): + """Return whether one pidfd has become readable because its task exited.""" + try: + import select + + readable, _, _ = select.select([int(pidfd)], [], [], 0) + except (ImportError, OSError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot poll a pinned Linux process identity: %s; verified " + "cleanup requires working pidfd polling" % exc + ) from exc + return bool(readable) + + +def _verified_group_member_mismatch(record, identity, pid, expected_pgid): + """Return the persisted-attribution mismatch for one pinned task.""" + if not isinstance(identity, dict) or identity.get("pid") != pid: + return "unreadable-process-identity" + if ( + not isinstance(identity.get("starttime"), int) + or isinstance(identity.get("starttime"), bool) + or identity["starttime"] <= 0 + ): + return "unreadable-process-identity" + if identity.get("uid") != record.get("uid"): + return "foreign-uid" + if pid == int(record["pid"]) and ( + identity.get("starttime") != record.get("starttime") + ): + return "reused-pid" + if identity.get("pgid") != expected_pgid: + return "foreign-process-group" + if identity.get("sid") != expected_pgid: + return "foreign-session" + if identity.get("inert") is True: + return None + if identity.get("inert") is not False: + return "unreadable-process-identity" + if identity.get("nonce") != record.get("nonce"): + return "foreign-nonce" + if identity.get("state_dir") != record.get("state_dir"): + return "foreign-state-directory" + if identity.get("weights_dir") != record.get("weights_dir"): + return "foreign-weights-directory" + return None + + +def _signal_verified_process_group( + record, + signum, + *, + process_group_member_pids=None, + process_identity=None, + process_group_alive=None, + pidfd_open=None, + pidfd_send_signal=None, + pidfd_exited=None, + close_fd=None, +): + """Pin, validate, then signal each exact member without numeric PGID use. + + The result status is one of ``signaled``, ``absent``, ``inconclusive``, + or ``foreign``. Inconclusive membership churn is deliberately distinct + from absence so callers can retry it only inside a bounded deadline. + """ + _require_linux() + if not isinstance(record, dict): + raise RamdiskError("verified cleanup requires a process record") + try: + expected_pid = int(record["pid"]) + expected_pgid = int(record.get("pgid", expected_pid)) + except (KeyError, TypeError, ValueError) as exc: + raise RamdiskError( + "verified cleanup process identity is incomplete" + ) from exc + if expected_pid <= 0 or expected_pgid <= 0: + raise RamdiskError("verified cleanup process identity is invalid") + + member_pids = ( + _process_group_member_pids + if process_group_member_pids is None + else process_group_member_pids + ) + identity_reader = ( + _strict_process_identity + if process_identity is None + else process_identity + ) + group_alive = ( + _process_group_alive + if process_group_alive is None + else process_group_alive + ) + poll_exited = _pidfd_exited if pidfd_exited is None else pidfd_exited + close_operation = os.close if close_fd is None else close_fd + + def inconclusive(reason, members=None): + return { + "status": "inconclusive", + "reason": reason, + "members": [] if members is None else list(members), + } + + try: + initial = list(member_pids(expected_pgid)) + except (OSError, RamdiskError) as exc: + return inconclusive("process-group-enumeration-failed: %s" % exc) + if initial != sorted(set(initial)) or any(pid <= 0 for pid in initial): + return { + "status": "inconclusive", + "reason": "ambiguous-membership", + "members": initial, + } + if not initial: + try: + alive_before = group_alive(expected_pgid) + confirmed = list(member_pids(expected_pgid)) + alive_after = group_alive(expected_pgid) + except (OSError, RamdiskError) as exc: + return inconclusive( + "empty-group-confirmation-failed: %s" % exc + ) + if not confirmed and not alive_before and not alive_after: + return {"status": "absent", "members": []} + return { + "status": "inconclusive", + "reason": "empty-live-process-group", + "members": confirmed, + } + + open_operation, send_operation = _pidfd_api( + pidfd_open=pidfd_open, + pidfd_send_signal=pidfd_send_signal, + ) + + pidfds = {} + readiness_before = {} + identities = {} + try: + # Bind the entire candidate set before reading any numeric PID state. + # A later PID/PGID reuse can therefore never redirect a signal. + for pid in initial: + try: + pidfds[pid] = open_operation(pid, 0) + except ProcessLookupError: + return { + "status": "inconclusive", + "reason": "member-exited-before-pidfd-open", + "members": initial, + } + except OSError as exc: + if exc.errno == errno.ESRCH: + return { + "status": "inconclusive", + "reason": "member-exited-before-pidfd-open", + "members": initial, + } + if exc.errno in (errno.ENOSYS, errno.EINVAL): + raise RamdiskError( + "%s (pidfd_open failed: %s)" + % (_PIDFD_REQUIRED_REASON, exc) + ) from exc + raise RamdiskError( + "cannot pin Linux PID %d for verified cleanup: %s" + % (pid, exc) + ) from exc + + try: + readiness_before = { + pid: bool(poll_exited(pidfd)) + for pid, pidfd in pidfds.items() + } + except (OSError, RamdiskError) as exc: + return inconclusive( + "pidfd-poll-failed: %s" % exc, + initial, + ) + for pid in initial: + try: + identity = identity_reader(pid) + except (OSError, RamdiskError) as exc: + return inconclusive( + "process-identity-read-failed for PID %d: %s" + % (pid, exc), + initial, + ) + if identity is None: + return { + "status": "inconclusive", + "reason": "unreadable-process-identity", + "members": initial, + } + identities[pid] = identity + + try: + readiness_after_identity = { + pid: bool(poll_exited(pidfd)) + for pid, pidfd in pidfds.items() + } + settled = list(member_pids(expected_pgid)) + except (OSError, RamdiskError) as exc: + return inconclusive( + "process-group-revalidation-failed: %s" % exc, + initial, + ) + if settled != initial: + return { + "status": "inconclusive", + "reason": "membership-changed", + "members": sorted(set(initial) | set(settled)), + } + + for pid in initial: + mismatch = _verified_group_member_mismatch( + record, + identities[pid], + pid, + expected_pgid, + ) + if mismatch is not None: + return { + "status": "foreign", + "reason": mismatch, + "members": initial, + } + if identities[pid].get("inert") is not True and ( + readiness_before[pid] or readiness_after_identity[pid] + ): + return { + "status": "inconclusive", + "reason": "pinned-member-exited-during-validation", + "members": initial, + } + + try: + confirmed = list(member_pids(expected_pgid)) + except (OSError, RamdiskError) as exc: + return inconclusive( + "process-group-confirmation-failed: %s" % exc, + initial, + ) + if confirmed != initial: + return { + "status": "inconclusive", + "reason": "membership-changed", + "members": sorted(set(initial) | set(confirmed)), + } + try: + readiness_before_signal = { + pid: bool(poll_exited(pidfd)) + for pid, pidfd in pidfds.items() + } + except (OSError, RamdiskError) as exc: + return inconclusive( + "final-pidfd-poll-failed: %s" % exc, + initial, + ) + if any( + identities[pid].get("inert") is not True + and readiness_before_signal[pid] + for pid in initial + ): + return { + "status": "inconclusive", + "reason": "pinned-member-exited-before-signal", + "members": initial, + } + + live = [ + pid + for pid in initial + if identities[pid].get("inert") is not True + ] + if not live: + try: + alive_before = group_alive(expected_pgid) + inert_confirmation = list(member_pids(expected_pgid)) + alive_after = group_alive(expected_pgid) + except (OSError, RamdiskError) as exc: + return inconclusive( + "inert-group-confirmation-failed: %s" % exc, + initial, + ) + if ( + inert_confirmation == initial + and not alive_before + and not alive_after + ): + return {"status": "absent", "members": initial} + return { + "status": "inconclusive", + "reason": "inert-membership-not-stable", + "members": sorted(set(initial) | set(inert_confirmation)), + } + + signaled = [] + exited = [] + for pid in live: + try: + send_operation(pidfds[pid], signum, None, 0) + signaled.append(pid) + except ProcessLookupError: + exited.append(pid) + except OSError as exc: + if exc.errno == errno.ESRCH: + exited.append(pid) + continue + raise RamdiskError( + "pidfd signal %s failed for verified PID %d after " + "signaling PID(s) %s: %s; cleanup authority remains " + "persisted and numeric fallback is forbidden" + % ( + signum, + pid, + ", ".join(str(value) for value in signaled) or "none", + exc, + ) + ) from exc + return { + "status": "signaled", + "members": initial, + "signaled": signaled, + "exited": exited, + } + finally: + for pidfd in pidfds.values(): + try: + close_operation(pidfd) + except OSError: + pass + + +def _process_status(pid, *, read_text=None): + _require_linux() + read_text = _read_text if read_text is None else read_text + return read_text("/proc/%d/status" % int(pid)) + + +def _busy_mount_references_proc(path): + """Strict root-only procfs scan for references below ``path``.""" + _require_linux() + path = os.path.normpath(path) + os.sep + found = [] + + def visibility_error(action, proc_path, error): + raise RamdiskError( + "cannot %s %s: %s; managed cleanup requires complete /proc " + "visibility (hidepid or a security policy may deny it)" + % (action, proc_path, error) + ) from error + + def reference_below(target): + if target.endswith(" (deleted)"): + target = target[: -len(" (deleted)")] + if not os.path.isabs(target): + return False + return (os.path.normpath(target) + os.sep).startswith(path) + + def missing_endpoint_is_inert(entry, endpoint, missing_error): + """Corroborate endpoint ENOENT without overlooking a live task.""" + stat_path = "/proc/%s/stat" % entry + try: + process_stat = _read_proc_stat(stat_path) + except (FileNotFoundError, ProcessLookupError): + # The PID itself is now absent, so it cannot retain the mount. + return True + except (OSError, UnicodeError) as exc: + visibility_error("verify process identity", stat_path, exc) + + close = process_stat.rfind(")") + fields = process_stat[close + 2 :].split() if close >= 0 else [] + try: + state = fields[0] + flags = int(fields[6], 10) + except (IndexError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot parse process identity %s after missing endpoint %s; " + "managed cleanup requires complete /proc visibility" + % (stat_path, endpoint) + ) from exc + if len(state) != 1: + raise RamdiskError( + "cannot parse process identity %s after missing endpoint %s; " + "managed cleanup requires complete /proc visibility" + % (stat_path, endpoint) + ) + + # PF_KTHREAD tasks have no userspace mm, files, or cwd. + if flags & 0x00200000: + return True + if state in ("Z", "X", "x"): + # A multithreaded process can retain a zombie group leader while + # live siblings still share its mm/files/fs. Only a complete task + # snapshot with no nonleader TID proves this dead leader inert. + task_dir = "/proc/%s/task" % entry + try: + task_entries = os.listdir(task_dir) + except (FileNotFoundError, ProcessLookupError): + try: + _read_proc_stat(stat_path) + except (FileNotFoundError, ProcessLookupError): + return True + except (OSError, UnicodeError) as exc: + visibility_error("recheck process identity", stat_path, exc) + raise RamdiskError( + "cannot enumerate task group %s while PID %s remains; " + "managed cleanup requires complete /proc visibility" + % (task_dir, entry) + ) from missing_error + except OSError as exc: + visibility_error("enumerate process task group", task_dir, exc) + invalid_tasks = [ + task for task in task_entries if not task.isdigit() + ] + if invalid_tasks: + raise RamdiskError( + "cannot parse process task group %s; managed cleanup " + "requires complete /proc visibility" % task_dir + ) + try: + num_threads = int(fields[17], 10) + except (IndexError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot parse process thread count %s; managed cleanup " + "requires complete /proc visibility" % stat_path + ) from exc + task_ids = [int(task) for task in task_entries] + unique_tasks = set(task_ids) + if ( + num_threads <= 0 + or len(task_ids) != num_threads + or len(unique_tasks) != num_threads + or int(entry) not in unique_tasks + ): + raise RamdiskError( + "incomplete process task snapshot %s: stat declares %d " + "threads but task entries are %s; managed cleanup " + "requires complete /proc visibility" + % ( + task_dir, + num_threads, + ",".join(str(task) for task in sorted(unique_tasks)) + or "none", + ) + ) + live_siblings = sorted( + task for task in unique_tasks if task != int(entry) + ) + if not live_siblings: + return True + raise RamdiskError( + "cannot trust missing process endpoint %s: zombie/dead " + "leader PID %s still has sibling tasks %s; managed cleanup " + "requires complete /proc visibility" + % ( + endpoint, + entry, + ",".join(str(task) for task in live_siblings), + ) + ) from missing_error + raise RamdiskError( + "cannot trust missing process endpoint %s while PID %s remains " + "a live userspace task; managed cleanup requires complete /proc " + "visibility" % (endpoint, entry) + ) from missing_error + + try: + entries = os.listdir("/proc") + except OSError as exc: + visibility_error("enumerate", "/proc", exc) + + maps_line = re.compile( + r"^[0-9A-Fa-f]+-[0-9A-Fa-f]+\s+" + r"[r-][w-][x-][ps]\s+[0-9A-Fa-f]+\s+" + r"[0-9A-Fa-f]+:[0-9A-Fa-f]+\s+\d+" + r"(?:\s+(.*))?$" + ) + + def task_group_snapshot(entry): + """Return one complete task-membership snapshot for a live TGID.""" + stat_path = "/proc/%s/stat" % entry + try: + process_stat = _read_proc_stat(stat_path) + except (FileNotFoundError, ProcessLookupError): + return None + except (OSError, UnicodeError) as exc: + visibility_error("read process identity", stat_path, exc) + + close = process_stat.rfind(")") + fields = process_stat[close + 2 :].split() if close >= 0 else [] + try: + state = fields[0] + flags = int(fields[6], 10) + num_threads = int(fields[17], 10) + start_time = int(fields[19], 10) + except (IndexError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot parse process task identity %s; managed cleanup " + "requires complete /proc visibility" % stat_path + ) from exc + if len(state) != 1 or num_threads <= 0 or start_time < 0: + raise RamdiskError( + "cannot parse process task identity %s; managed cleanup " + "requires complete /proc visibility" % stat_path + ) + + task_dir = "/proc/%s/task" % entry + try: + task_entries = os.listdir(task_dir) + except (FileNotFoundError, ProcessLookupError) as exc: + try: + _read_proc_stat(stat_path) + except (FileNotFoundError, ProcessLookupError): + return None + except (OSError, UnicodeError) as recheck_exc: + visibility_error( + "recheck process identity", + stat_path, + recheck_exc, + ) + raise RamdiskError( + "cannot enumerate task group %s while PID %s remains; " + "managed cleanup requires complete /proc visibility" + % (task_dir, entry) + ) from exc + except OSError as exc: + visibility_error("enumerate process task group", task_dir, exc) + + if any(not task.isdigit() for task in task_entries): + raise RamdiskError( + "cannot parse process task group %s; managed cleanup " + "requires complete /proc visibility" % task_dir + ) + task_ids = [int(task) for task in task_entries] + unique_tasks = set(task_ids) + if ( + len(task_ids) != num_threads + or len(unique_tasks) != num_threads + or int(entry) not in unique_tasks + ): + raise RamdiskError( + "incomplete process task snapshot %s: stat declares %d " + "threads but task entries are %s; managed cleanup requires " + "complete /proc visibility" + % ( + task_dir, + num_threads, + ",".join(str(task) for task in sorted(unique_tasks)) + or "none", + ) + ) + return { + "flags": flags, + "start_time": start_time, + "tasks": tuple(sorted(unique_tasks)), + } + + def missing_task_endpoint_is_inert(entry, task, endpoint, missing_error): + """Reject a partial live-task view; tolerate only proven inert tasks.""" + task_stat_path = "/proc/%s/task/%s/stat" % (entry, task) + try: + task_stat = _read_proc_stat(task_stat_path) + except (FileNotFoundError, ProcessLookupError): + leader_stat_path = "/proc/%s/stat" % entry + try: + _read_proc_stat(leader_stat_path) + except (FileNotFoundError, ProcessLookupError): + return True + except (OSError, UnicodeError) as exc: + visibility_error( + "recheck process identity", + leader_stat_path, + exc, + ) + raise RamdiskError( + "incomplete process task snapshot: task %s disappeared at %s " + "while PID %s remains; managed cleanup requires complete " + "/proc visibility" % (task, endpoint, entry) + ) from missing_error + except (OSError, UnicodeError) as exc: + visibility_error("verify process task identity", task_stat_path, exc) + + close = task_stat.rfind(")") + fields = task_stat[close + 2 :].split() if close >= 0 else [] + try: + state = fields[0] + flags = int(fields[6], 10) + except (IndexError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot parse process task identity %s after missing endpoint " + "%s; managed cleanup requires complete /proc visibility" + % (task_stat_path, endpoint) + ) from exc + if flags & 0x00200000 or state in ("Z", "X", "x"): + return True + raise RamdiskError( + "cannot trust missing task endpoint %s while task %s in PID %s " + "remains live; managed cleanup requires complete /proc visibility" + % (endpoint, task, entry) + ) from missing_error + + def scan_task_references(entry, pid, endpoint_root, missing_is_inert): + """Return ``(busy, inert)`` for one leader or nonleader task.""" + for leaf in ("cwd", "root", "exe"): + proc_path = "%s/%s" % (endpoint_root, leaf) + try: + target = os.readlink(proc_path) + if reference_below(target): + return True, False + except (FileNotFoundError, ProcessLookupError) as exc: + if missing_is_inert(proc_path, exc): + return False, True + except OSError as exc: + visibility_error("read process reference", proc_path, exc) + + maps_path = "%s/maps" % endpoint_root + try: + with open( + maps_path, + "r", + encoding="utf-8", + errors="surrogateescape", + ) as stream: + mappings = stream.read() + except (FileNotFoundError, ProcessLookupError) as exc: + if missing_is_inert(maps_path, exc): + return False, True + except OSError as exc: + visibility_error("read process mappings", maps_path, exc) + for line_number, line in enumerate(mappings.splitlines(), 1): + match = maps_line.fullmatch(line) + if match is None: + raise RamdiskError( + "cannot parse process mappings %s line %d; managed cleanup " + "requires complete /proc visibility" + % (maps_path, line_number) + ) + mapped = match.group(1) + if not mapped or not mapped.startswith("/"): + continue + if reference_below(_unescape_mount(mapped)): + return True, False + + fd_dir = "%s/fd" % endpoint_root + try: + descriptors = os.listdir(fd_dir) + except (FileNotFoundError, ProcessLookupError) as exc: + if missing_is_inert(fd_dir, exc): + return False, True + except OSError as exc: + visibility_error("enumerate process descriptors", fd_dir, exc) + for descriptor in descriptors: + descriptor_path = os.path.join(fd_dir, descriptor) + try: + target = os.readlink(descriptor_path) + if reference_below(target): + return True, False + except (FileNotFoundError, ProcessLookupError): + # Descriptor closure after listdir() releases that reference. + continue + except OSError as exc: + visibility_error( + "read process descriptor", + descriptor_path, + exc, + ) + return False, False + + for entry in entries: + if not entry.isdigit(): + continue + pid = int(entry) + leader_root = "/proc/%s" % entry + busy, inert = scan_task_references( + entry, + pid, + leader_root, + lambda endpoint, error: missing_endpoint_is_inert( + entry, + endpoint, + error, + ), + ) + if inert: + continue + if busy: + found.append(pid) + continue + + initial = task_group_snapshot(entry) + if initial is None or initial["flags"] & 0x00200000: + continue + + for task in initial["tasks"]: + if task == pid: + continue + task_root = "/proc/%s/task/%s" % (entry, task) + busy, _ = scan_task_references( + entry, + pid, + task_root, + lambda endpoint, error, task=task: ( + missing_task_endpoint_is_inert( + entry, + task, + endpoint, + error, + ) + ), + ) + if busy: + found.append(pid) + break + if found and found[-1] == pid: + continue + + final = task_group_snapshot(entry) + if final is None: + continue + if ( + final["start_time"] != initial["start_time"] + or final["tasks"] != initial["tasks"] + ): + raise RamdiskError( + "incomplete process task snapshot /proc/%s/task changed while " + "it was inspected; managed cleanup requires complete /proc " + "visibility" % entry + ) + return sorted(set(found)) + + +def _fuser_failure(path, result): + detail = " ".join( + value.strip() + for value in ( + getattr(result, "stdout", "") or "", + getattr(result, "stderr", "") or "", + ) + if value.strip() + ) + if len(detail) > 1000: + detail = detail[:997] + "..." + suffix = ": %s" % detail if detail else "" + return RamdiskError( + "trusted fuser could not inspect managed mount %s (exit %s)%s" + % (path, getattr(result, "returncode", "unknown"), suffix) + ) + + +def _parse_fuser_mount_references(path, result): + """Parse PSmisc fuser's intentionally split stdout/stderr contract.""" + stdout = getattr(result, "stdout", "") or "" + stderr = getattr(result, "stderr", "") or "" + returncode = getattr(result, "returncode", None) + if returncode == 0: + tokens = stdout.split() + if not tokens or any( + re.fullmatch(r"[0-9]+", token) is None or int(token) <= 0 + for token in tokens + ): + raise RamdiskError( + "trusted fuser returned an invalid PID list for %s" % path + ) + # Without --verbose, PSmisc writes exactly the requested mount name + # followed by the per-PID access letters c/e/f/F/r/m to stderr. Match + # the exact path independently of its whitespace or regex syntax; + # arbitrary diagnostics would mean the scan may be incomplete. + annotation = re.fullmatch( + re.escape(path) + r":[ \tcefFrm]*(?:\r?\n)?", + stderr, + ) + if stderr and annotation is None: + raise _fuser_failure(path, result) + return sorted(set(int(token) for token in tokens)) + if returncode == 1 and not stdout.strip() and not stderr.strip(): + return [] + raise _fuser_failure(path, result) + + +def _trusted_fuser_binary(trusted_system_binary): + """Resolve PSmisc fuser with install guidance for unprivileged cleanup.""" + try: + return trusted_system_binary("fuser") + except RamdiskError as exc: + raise RamdiskError( + "unprivileged managed cleanup requires trusted PSmisc fuser: " + "%s; install the psmisc package and retry" % exc + ) from exc + + +def _busy_mount_references( + path, + hardware=None, + *, + run=None, + trusted_system_binary=None, + privileged=None, +): + """Return a complete busy set via root procfs or trusted privileged fuser.""" + _require_linux() + path = os.path.normpath(os.path.abspath(os.fspath(path))) + if current_euid() == 0: + return _busy_mount_references_proc(path) + run = _run if run is None else run + trusted_system_binary = ( + _trusted_system_binary + if trusted_system_binary is None + else trusted_system_binary + ) + fuser = _trusted_fuser_binary(trusted_system_binary) + command = [fuser, "-mM", path] + if privileged is None: + command = _privileged( + command, + hardware, + trusted_system_binary=trusted_system_binary, + ) + else: + command = privileged(command, hardware) + try: + result = run(command, timeout=10.0) + except Exception as exc: + raise RamdiskError( + "trusted fuser could not inspect managed mount %s: %s" + % (path, exc) + ) from exc + return _parse_fuser_mount_references(path, result) + + +def _ensure_busy_mount_scan_available( + path, + hardware=None, + *, + trusted_system_binary=None, + run=None, +): + """Prove cleanup discovery and unmount exist before mount mutation.""" + _require_linux() + trusted_system_binary = ( + _trusted_system_binary + if trusted_system_binary is None + else trusted_system_binary + ) + run = _run if run is None else run + if current_euid() == 0: + _busy_mount_references_proc(path) + else: + # The planned path is not a mountpoint yet, so ``fuser -M`` cannot + # probe it. Root is always a mountpoint and exercises the exact trusted + # PSmisc binary, privileged command shape, option support, and current + # sudo authorization that cleanup will require later. + _busy_mount_references( + os.path.sep, + hardware=hardware, + run=run, + trusted_system_binary=trusted_system_binary, + ) + umount = trusted_system_binary("umount") + help_command = [umount, "--help"] + if current_euid() != 0: + help_command = _privileged( + help_command, + hardware, + trusted_system_binary=trusted_system_binary, + ) + try: + help_result = run(help_command, timeout=5.0) + except Exception as exc: + raise RamdiskError( + "trusted umount helper could not be verified: %s; install the " + "util-linux package" % exc + ) from exc + help_output = "%s\n%s" % ( + getattr(help_result, "stdout", "") or "", + getattr(help_result, "stderr", "") or "", + ) + if ( + getattr(help_result, "returncode", None) != 0 + or "--no-canonicalize" not in help_output + ): + raise RamdiskError( + "trusted privileged umount helper is incompatible or " + "unauthorized: cleanup requires the util-linux " + "--no-canonicalize option" + ) + + +class LinuxPlatformOps: + """Narrow Linux discovery operations with no import-time probes.""" + + is_linux = True + + def __init__(self, platform_name="linux"): + self.platform_name = platform_name + + @property + def process_control_supported(self): + """Whether managed tasks can be pinned and safely signalled.""" + return ( + callable(getattr(os, "getpgid", None)) + and callable(getattr(os, "killpg", None)) + and getattr(signal, "SIGTERM", None) is not None + and getattr(signal, "SIGKILL", None) is not None + and _pidfd_process_control_supported() + ) + + @property + def process_control_reason(self): + return _PIDFD_REQUIRED_REASON + + def capabilities(self): + return { + "platform": self.platform_name, + "hardware_discovery": True, + "cgroup_memory": True, + "numa": True, + "ramdisk_lifecycle": True, + "reason": None, + } + + read_text = staticmethod(_read_text) + status_allowed_list = staticmethod(_status_allowed_list) + thread_sibling_groups = staticmethod(_thread_sibling_groups) + meminfo = staticmethod(_meminfo) + read_cgroup_value = staticmethod(_read_cgroup_value) + read_cgroup_contract = staticmethod(_read_cgroup_contract) + node_meminfo = staticmethod(_node_meminfo) + physical_cores = staticmethod(_physical_cores) + process_start_boundary = staticmethod(_process_start_boundary) + process_identity = staticmethod(_process_identity) + managed_launch_processes = staticmethod(_managed_launch_processes) + process_group_member_pids = staticmethod(_process_group_member_pids) + process_group_alive = staticmethod(_process_group_alive) + signal_verified_process_group = staticmethod( + _signal_verified_process_group + ) + process_status = staticmethod(_process_status) + busy_mount_references = staticmethod(_busy_mount_references) + + @staticmethod + def path_exists(path): + return os.path.exists(path) + + @staticmethod + def cpu_affinity(): + get_affinity = getattr(os, "sched_getaffinity", None) + if get_affinity is None: + return None + try: + return sorted(int(cpu) for cpu in get_affinity(0)) + except OSError: + return None + + @staticmethod + def kernel_release(): + return platform.release() + + @staticmethod + def kernel_at_least(major, minor): + return _kernel_at_least(major, minor) + + @staticmethod + def executable_path(name): + return shutil.which(name) diff --git a/c/ramdisk_support/model.py b/c/ramdisk_support/model.py new file mode 100644 index 000000000..d003b45fd --- /dev/null +++ b/c/ramdisk_support/model.py @@ -0,0 +1,306 @@ +"""Safetensors model discovery and immutable model identity helpers.""" + +from __future__ import print_function + +import hashlib +import json +import os +import re +import struct + +from .common import MIB, RamdiskError + + +MAX_ST_HEADER = 512 * MIB + +EXPERT_RE = re.compile( + r"^model\.layers\.(\d+)\.mlp\.experts\.(\d+)\." + r"(gate_proj|up_proj|down_proj)\.weight(\.qs)?$" +) + +def _read_safetensors_header(path): + size = os.path.getsize(path) + with open(path, "rb") as stream: + raw = stream.read(8) + if len(raw) != 8: + raise RamdiskError("truncated safetensors file: %s" % path) + header_size = struct.unpack(" MAX_ST_HEADER or header_size > size - 8: + raise RamdiskError("invalid safetensors header length in %s" % path) + raw_header = stream.read(header_size) + if len(raw_header) != header_size: + raise RamdiskError("truncated safetensors header: %s" % path) + try: + header = json.loads(raw_header.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + raise RamdiskError("invalid safetensors header in %s: %s" % (path, exc)) + if not isinstance(header, dict): + raise RamdiskError("safetensors header is not an object: %s" % path) + data_start = 8 + header_size + tensors = {} + for name, record in header.items(): + if name == "__metadata__": + continue + if not isinstance(record, dict) or "data_offsets" not in record: + raise RamdiskError("invalid tensor record %r in %s" % (name, path)) + offsets = record["data_offsets"] + if ( + not isinstance(offsets, list) + or len(offsets) != 2 + or not all(isinstance(value, int) for value in offsets) + or offsets[0] < 0 + or offsets[1] < offsets[0] + or data_start + offsets[1] > size + ): + raise RamdiskError("invalid tensor offsets for %s in %s" % (name, path)) + tensors[name] = { + "dtype": record.get("dtype"), + "shape": record.get("shape"), + "offset": data_start + offsets[0], + "bytes": offsets[1] - offsets[0], + } + return raw_header, tensors + +def _sha256_file(path): + digest = hashlib.sha256() + with open(path, "rb") as stream: + while True: + chunk = stream.read(8 * MIB) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() + +def _shape_numel(shape): + if not isinstance(shape, list) or not shape or not all(isinstance(value, int) and value >= 0 for value in shape): + return None + result = 1 + for value in shape: + result *= value + return result + +def _resolve_direct_format(rows, columns, weight_bytes, scale_bytes): + """Mirror ``qt_resolve_fmt`` for an unstamped routed-expert tensor. + + Return ``(fmt, group_size)`` only when the engine can decode the exact + weight/scale geometry. Ambiguous E8 layouts and recognized-but-unsupported + FP8 UE8M0 sidecars fail closed, while the established unstamped + int8-versus-FP8 collision rule continues to select int8. + """ + int8_bytes = rows * columns + int4_bytes = rows * ((columns + 1) // 2) + int2_bytes = rows * ((columns + 3) // 4) + int3_groups = (columns + 63) // 64 + int3_bytes = rows * int3_groups * 24 + e8_bytes = rows * ((columns + 255) // 256) * 98 + fp8_blocks = ((rows + 127) // 128) * ((columns + 127) // 128) + + # qt_resolve_fmt's SECOND DESIGN LANDMINE: an unstamped I=98 tensor can + # satisfy E8 and one or more raw-byte formats simultaneously. + if scale_bytes == 4 and weight_bytes == e8_bytes: + raw_bytes_also = weight_bytes == int8_bytes + if raw_bytes_also and (fp8_blocks in (1, 4) or rows == 1): + return None + return 6, 0 + + # Keep the engine's row-format precedence for small-shape byte collisions. + if weight_bytes == int8_bytes: + fmt, group_size = 1, 0 + elif weight_bytes == int4_bytes: + fmt, group_size = 2, 0 + if scale_bytes > rows * 4: + for candidate in (16, 32, 48, 64, 96, 128, 192, 256): + if candidate > columns: + break + if scale_bytes == rows * ((columns + candidate - 1) // candidate) * 4: + fmt, group_size = 4, candidate + break + elif weight_bytes == int2_bytes: + fmt, group_size = 3, 0 + elif weight_bytes == int3_bytes: + fmt, group_size = 5, 0 + else: + return None + + if fmt == 1: + is_row = scale_bytes == rows * 4 + is_fp8_f32 = scale_bytes == fp8_blocks * 4 + is_fp8_ue8m0 = scale_bytes == fp8_blocks + if is_row and is_fp8_f32: + pass # Unstamped collision: qt_resolve_fmt selects incumbent int8. + elif is_fp8_ue8m0: + return None + elif is_fp8_f32 and not is_row: + fmt = 8 + + if fmt == 4: + expected_scales = rows * ((columns + group_size - 1) // group_size) + elif fmt == 5: + expected_scales = rows * int3_groups + elif fmt == 8: + expected_scales = fp8_blocks + else: + expected_scales = rows + if scale_bytes != expected_scales * 4: + return None + return fmt, group_size + +def _direct_tensor_set_eligible(entry, config): + hidden = int(config["hidden_size"]) + intermediate = int(config["moe_intermediate_size"]) + prefix = "model.layers.%d.mlp.experts.%d." % (entry["layer"], entry["expert"]) + for projection, rows, columns in ( + ("gate_proj", intermediate, hidden), + ("up_proj", intermediate, hidden), + ("down_proj", hidden, intermediate), + ): + weight = entry["tensors"][prefix + projection + ".weight"] + scale = entry["tensors"][prefix + projection + ".weight.qs"] + if ( + weight["dtype"] not in ("U8", "I8") + or scale["dtype"] != "F32" + or weight["offset"] % 4 + or scale["offset"] % 4 + or _shape_numel(weight["shape"]) != weight["bytes"] + or _shape_numel(scale["shape"]) != scale["bytes"] // 4 + or scale["bytes"] % 4 + ): + return False + weight_bytes = weight["bytes"] + if _resolve_direct_format(rows, columns, weight_bytes, scale["bytes"]) is None: + return False + return True + +def scan_model(model_dir): + """Index shards and each expert's complete six-tensor direct-map closure.""" + model_dir = os.path.realpath(os.path.abspath(os.path.expanduser(model_dir))) + if not os.path.isdir(model_dir): + raise RamdiskError("model directory not found: %s" % model_dir) + names = sorted(name for name in os.listdir(model_dir) if name.endswith(".safetensors")) + if not names: + raise RamdiskError("no .safetensors shards found in %s" % model_dir) + fingerprint = hashlib.sha256() + identity_files = {} + required_metadata = ("config.json", "tokenizer.json") + optional_metadata = ("generation_config.json", "tokenizer_config.json") + for name in required_metadata + optional_metadata: + path = os.path.join(model_dir, name) + if not os.path.isfile(path): + if name in required_metadata: + raise RamdiskError("required model metadata is missing: %s" % path) + continue + digest = _sha256_file(path) + size = os.path.getsize(path) + identity_files[name] = {"size_bytes": size, "sha256": digest} + fingerprint.update(("metadata\0%s\0%d\0%s\n" % (name, size, digest)).encode("utf-8")) + config_path = os.path.join(model_dir, "config.json") + try: + with open(config_path, "r", encoding="utf-8") as stream: + config = json.load(stream) + except (OSError, ValueError) as exc: + raise RamdiskError("cannot parse %s: %s" % (config_path, exc)) + if not isinstance(config, dict): + raise RamdiskError("config.json must contain a JSON object") + required_positive = ( + "hidden_size", + "num_hidden_layers", + "num_attention_heads", + "n_routed_experts", + "num_experts_per_tok", + "moe_intermediate_size", + "intermediate_size", + "kv_lora_rank", + "qk_nope_head_dim", + "qk_rope_head_dim", + "v_head_dim", + "n_shared_experts", + "vocab_size", + ) + missing = [name for name in required_positive if not isinstance(config.get(name), int) or config[name] <= 0] + if missing: + raise RamdiskError("config.json is missing positive engine fields: %s" % ", ".join(missing)) + shards = [] + experts = {} + tensor_bytes = 0 + expert_tensor_bytes = 0 + for name in names: + path = os.path.join(model_dir, name) + if not os.path.isfile(path): + raise RamdiskError("shard is not a regular file: %s" % path) + st = os.stat(path, follow_symlinks=True) + raw_header, tensors = _read_safetensors_header(path) + header_digest = hashlib.sha256(raw_header).hexdigest() + identity = "%s\0%d\0%d\0%d\0%s\n" % ( + name, + st.st_size, + st.st_mtime_ns, + st.st_ino, + header_digest, + ) + fingerprint.update(identity.encode("utf-8")) + shards.append( + { + "name": name, + "path": path, + "size_bytes": st.st_size, + "device": st.st_dev, + "inode": st.st_ino, + "mtime_ns": st.st_mtime_ns, + "header_sha256": header_digest, + "tensor_count": len(tensors), + } + ) + for tensor_name, tensor in tensors.items(): + tensor_bytes += tensor["bytes"] + match = EXPERT_RE.match(tensor_name) + if not match: + continue + layer, expert = int(match.group(1)), int(match.group(2)) + key = "%d:%d" % (layer, expert) + entry = experts.setdefault( + key, + { + "layer": layer, + "expert": expert, + "tensors": {}, + "shards": set(), + "tensor_bytes": 0, + }, + ) + entry["tensors"][tensor_name] = { + "shard": name, + "bytes": tensor["bytes"], + "dtype": tensor["dtype"], + "shape": tensor["shape"], + "offset": tensor["offset"], + } + entry["shards"].add(name) + entry["tensor_bytes"] += tensor["bytes"] + expert_tensor_bytes += tensor["bytes"] + complete = {} + for key, entry in experts.items(): + prefix = "model.layers.%d.mlp.experts.%d." % (entry["layer"], entry["expert"]) + expected = set() + for projection in ("gate_proj", "up_proj", "down_proj"): + weight = prefix + projection + ".weight" + expected.add(weight) + expected.add(weight + ".qs") + if expected == set(entry["tensors"]): + entry["shards"] = sorted(entry["shards"]) + entry["direct_map_eligible"] = _direct_tensor_set_eligible(entry, config) + complete[key] = entry + total_bytes = sum(shard["size_bytes"] for shard in shards) + return { + "path": model_dir, + "fingerprint": "sha256:" + fingerprint.hexdigest(), + "fingerprint_algorithm": "metadata content plus sorted shard name,size,mtime,inode,header-sha256", + "identity_files": identity_files, + "shards": shards, + "shard_names": names, + "total_shard_bytes": total_bytes, + "tensor_bytes": tensor_bytes, + "dense_tensor_bytes": max(0, tensor_bytes - expert_tensor_bytes), + "experts": complete, + "complete_experts": len(complete), + "config": config, + } diff --git a/c/ramdisk_support/mounts.py b/c/ramdisk_support/mounts.py new file mode 100644 index 000000000..b2362c234 --- /dev/null +++ b/c/ramdisk_support/mounts.py @@ -0,0 +1,1014 @@ +"""tmpfs mounting, shard staging, and NUMA namespace validation.""" + +from __future__ import print_function + +import concurrent.futures +import hashlib +import mmap +import os +import re +import secrets +import stat +import subprocess +import sys +import threading +import time + +from .common import ( + GIB, + MIB, + RamdiskError, + _MountHelperCompletedError, + _raise_if_cancelled, +) +from .discovery import _discover_cgroup_memory +from .linux_ops import ( + _current_gid, + _meminfo, + _mount_at, + _node_meminfo, + _privileged, + _run, + _trusted_system_binary, +) +from .model import _read_safetensors_header, scan_model +from .platform_ops import ( + UNSUPPORTED_PLATFORM_REASON, + current_uid, + get_platform_ops, +) + + +def _busy_mount_references(path, *, ops=None, hardware=None): + """Return processes that keep a managed mount busy.""" + ops = get_platform_ops() if ops is None else ops + return ops.busy_mount_references(path, hardware=hardware) + + +def _reusable_empty_mountpoint(path): + """Recognize an empty root-owned leaf left by X-mount.mkdir=0755.""" + try: + info = os.stat(path, follow_symlinks=False) + return ( + stat.S_ISDIR(info.st_mode) + and info.st_uid == os.stat("/").st_uid + and not (info.st_mode & 0o022) + and not os.listdir(path) + ) + except OSError: + return False + + +def _mount_option_list(plan, mount, thp=None, include_noswap=None): + thp = thp or plan["mount_options"]["thp"] + if include_noswap is None: + include_noswap = plan["mount_options"]["noswap"] + options = [ + "size=%d" % mount["size_bytes"], + "huge=%s" % thp, + "noatime", + "nodev", + "nosuid", + "noexec", + "mode=0700", + "uid=%d" % current_uid(), + "gid=%d" % _current_gid(), + "mpol=%s" % mount["policy"], + "X-mount.mkdir=0755", + ] + if include_noswap: + options.insert(1, "noswap") + return options + + +def _mount_tmpfs( + plan, + mount, + *, + trusted_system_binary=None, + run=None, + privileged=None, + rollback_interrupted_mount=None, +): + hardware = plan["hardware"] + trusted_system_binary = ( + _trusted_system_binary + if trusted_system_binary is None + else trusted_system_binary + ) + run = _run if run is None else run + privileged = _privileged if privileged is None else privileged + # ``rollback_interrupted_mount`` remains in the private compatibility + # signature while the facade is reconstructed, but it must never run here. + # The lifecycle persists pending ownership before invoking this helper and + # is the only layer that can make a durable, exact-identity cleanup choice. + mount_bin = trusted_system_binary("mount") + attempts = [] + thp = plan["mount_options"]["thp"] + noswap = plan["mount_options"]["noswap"] + attempts.append((thp, noswap)) + if thp == "within_size": + attempts.append(("advise", noswap)) + if plan["mount_options"]["allow_swappable"] and noswap: + attempts.append((thp, False)) + if thp == "within_size": + attempts.append(("advise", False)) + seen = set() + errors = [] + for try_thp, try_noswap in attempts: + if (try_thp, try_noswap) in seen: + continue + seen.add((try_thp, try_noswap)) + options = _mount_option_list( + plan, + mount, + try_thp, + try_noswap, + ) + command = [ + mount_bin, + "-t", + "tmpfs", + "-o", + ",".join(options), + "tmpfs", + mount["path"], + ] + result = run(privileged(command, hardware)) + if result.returncode == 0: + mount["effective_thp"] = try_thp + mount["effective_noswap"] = try_noswap + return + errors.append( + result.stderr.strip() + or result.stdout.strip() + or "mount failed" + ) + message = (result.stderr + result.stdout).lower() + if not any( + word in message + for word in ( + "invalid argument", + "unknown", + "not supported", + "wrong fs", + ) + ): + break + raise _MountHelperCompletedError( + "cannot mount tmpfs at %s: %s" + % (mount["path"], "; ".join(errors)) + ) + + +def _umount_path( + path, + hardware, + *, + trusted_system_binary=None, + run=None, + privileged=None, +): + trusted_system_binary = ( + _trusted_system_binary + if trusted_system_binary is None + else trusted_system_binary + ) + run = _run if run is None else run + privileged = _privileged if privileged is None else privileged + umount = trusted_system_binary("umount") + # Lifecycle mutations are serialized and CAP_SYS_ADMIN is delegated only + # to this trusted util-linux helper. Avoid its userspace canonicalization + # pass so the latest identity checks and the kernel pathname lookup are as + # close together as the path-based API permits. + result = run( + privileged( + [umount, "--no-canonicalize", "--", path], + hardware, + ) + ) + if result.returncode: + message = ( + result.stderr.strip() + or result.stdout.strip() + or "umount failed" + ) + raise RamdiskError("cannot unmount %s: %s" % (path, message)) + + +def _rollback_interrupted_mount( + plan, + mount, + effective_thp, + effective_noswap, + cause, + *, + mount_at=None, + validate_mount=None, + umount_path=None, +): + """Compatibility shim that never performs pathname-only rollback. + + Interrupted mount helpers are recovered only by the durable lifecycle's + pending/identified ownership state machine. Propagate the original event + without inspecting, validating, or unmounting whatever now occupies the + pathname. + """ + raise cause + + +def _option_present(options, name): + return any( + option == name or option.startswith(name + "=") + for option in options + ) + + +def _validate_mount(mount, plan, *, mount_at=None): + mount_at = _mount_at if mount_at is None else mount_at + actual = mount_at(mount["path"]) + if not actual: + raise RamdiskError("expected mount is absent: %s" % mount["path"]) + if actual["filesystem"] != "tmpfs" or actual["source"] != "tmpfs": + raise RamdiskError("refusing foreign mount at %s" % mount["path"]) + options = set(actual["options"] + actual["super_options"]) + required = ("noatime", "nodev", "nosuid", "noexec") + missing = [ + name + for name in required + if not _option_present(options, name) + ] + if mount.get( + "effective_noswap", + plan["mount_options"]["noswap"], + ) and not _option_present(options, "noswap"): + missing.append("noswap") + if missing: + raise RamdiskError( + "tmpfs at %s is missing options: %s" + % (mount["path"], ", ".join(missing)) + ) + mode_ok = any( + option in ("mode=700", "mode=0700") + for option in options + ) + huge = mount.get( + "effective_thp", + plan["mount_options"]["thp"], + ) + policy = mount["policy"].replace("\\,", ",") + normalized_options = { + option.replace("\\,", ",") + for option in options + } + if not mode_ok or not _option_present( + normalized_options, + "huge", + ): + raise RamdiskError( + "tmpfs at %s is missing managed mode/THP options" + % mount["path"] + ) + if "huge=%s" % huge not in normalized_options: + raise RamdiskError( + "tmpfs at %s has an unexpected THP policy" + % mount["path"] + ) + if "mpol=%s" % policy not in normalized_options: + raise RamdiskError( + "tmpfs at %s has an unexpected NUMA policy" + % mount["path"] + ) + actual["all_options"] = sorted(options) + return actual + + +def _default_cgroup_available_memory(*, discover_cgroup_memory=None): + discover_cgroup_memory = ( + _discover_cgroup_memory + if discover_cgroup_memory is None + else discover_cgroup_memory + ) + cgroup = discover_cgroup_memory() + if cgroup.get("error"): + raise RamdiskError( + "cannot validate cgroup memory headroom: %s" + % cgroup["error"] + ) + return cgroup.get("available_bytes") + + +def _available_memory(*, meminfo=None, cgroup_available_memory=None): + meminfo = _meminfo if meminfo is None else meminfo + cgroup_available_memory = ( + _default_cgroup_available_memory + if cgroup_available_memory is None + else cgroup_available_memory + ) + values = meminfo() + available = values.get( + "MemAvailable", + values.get("MemFree", 0), + ) + cgroup_available = cgroup_available_memory() + return ( + min(available, cgroup_available) + if cgroup_available is not None + else available + ) + + +def _host_available_for_mount( + mount, + plan=None, + *, + meminfo=None, + node_meminfo=None, +): + """Return host/NUMA availability without shared cgroup headroom.""" + meminfo = _meminfo if meminfo is None else meminfo + node_meminfo = ( + _node_meminfo + if node_meminfo is None + else node_meminfo + ) + if mount.get("node") is None: + nodes = (plan or {}).get("placement", {}).get( + "memory_nodes" + ) + if nodes: + available = 0 + for node in nodes: + values = node_meminfo(int(node)) + available += values.get( + "MemFree", + values.get("MemAvailable", 0), + ) + return available + values = meminfo() + return values.get( + "MemAvailable", + values.get("MemFree", 0), + ) + values = node_meminfo(int(mount["node"])) + return values.get( + "MemFree", + values.get("MemAvailable", 0), + ) + + +def _available_for_mount( + mount, + plan=None, + *, + host_available_for_mount=None, + cgroup_available_memory=None, +): + host_available_for_mount = ( + _host_available_for_mount + if host_available_for_mount is None + else host_available_for_mount + ) + cgroup_available_memory = ( + _default_cgroup_available_memory + if cgroup_available_memory is None + else cgroup_available_memory + ) + available = host_available_for_mount(mount, plan=plan) + cgroup_available = cgroup_available_memory() + return ( + min(available, cgroup_available) + if cgroup_available is not None + else available + ) + + +def _copy_stream(src, tmp, expected_size, cancel_event=None): + binary_flag = getattr(os, "O_BINARY", 0) + source_fd = os.open(src, os.O_RDONLY | binary_flag) + try: + destination_fd = os.open( + tmp, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | binary_flag, + 0o400, + ) + try: + copied = 0 + while copied < expected_size: + _raise_if_cancelled(cancel_event) + data = os.read( + source_fd, + min(8 * MIB, expected_size - copied), + ) + if not data: + raise RamdiskError( + "source shard was truncated while copying: %s" + % src + ) + view = memoryview(data) + while view: + written = os.write(destination_fd, view) + if written <= 0: + raise RamdiskError( + "short write while staging %s" % src + ) + view = view[written:] + copied += len(data) + if os.read(source_fd, 1): + raise RamdiskError( + "source shard grew while copying: %s" % src + ) + os.fsync(destination_fd) + finally: + os.close(destination_fd) + if hasattr(os, "posix_fadvise") and hasattr( + os, + "POSIX_FADV_DONTNEED", + ): + try: + os.posix_fadvise( + source_fd, + 0, + 0, + os.POSIX_FADV_DONTNEED, + ) + except OSError: + pass + finally: + os.close(source_fd) + + +def _copy_one( + src, + destination, + expected_size, + reserve_floor, + progress=None, + available=None, + cancel_event=None, +): + available = available or _available_memory + _raise_if_cancelled(cancel_event) + if available() < reserve_floor: + raise RamdiskError( + "available memory reached the protected reserve before %s" + % os.path.basename(src) + ) + tmp = destination + ".coli-copy-%d-%s" % ( + os.getpid(), + secrets.token_hex(4), + ) + started = time.monotonic() + try: + _copy_stream( + src, + tmp, + expected_size, + cancel_event=cancel_event, + ) + os.chmod(tmp, 0o400) + if os.path.getsize(tmp) != expected_size: + raise RamdiskError( + "staged size mismatch for %s" + % os.path.basename(src) + ) + _read_safetensors_header(tmp) + os.replace(tmp, destination) + if progress: + progress( + os.path.basename(src), + expected_size, + time.monotonic() - started, + ) + finally: + try: + os.unlink(tmp) + except OSError: + pass + + +def _copy_worker_main(src, tmp, expected_size): + _copy_stream(src, tmp, int(expected_size)) + os.chmod(tmp, 0o400) + return 0 + + +def _copy_one_affined( + src, + destination, + expected_size, + node, + numactl, + cpu_list, + reserve_floor, + progress=None, + available=None, + cancel_event=None, + *, + run=None, + worker_entrypoint=None, +): + if not get_platform_ops().is_linux: + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + run = _run if run is None else run + if worker_entrypoint is None: + raise RamdiskError("copy worker entrypoint is unavailable") + available = available or _available_memory + _raise_if_cancelled(cancel_event) + if available() < reserve_floor: + raise RamdiskError( + "available memory reached the protected reserve before replica copy" + ) + tmp = destination + ".coli-copy-%d-%s" % ( + os.getpid(), + secrets.token_hex(4), + ) + started = time.monotonic() + command = [ + numactl, + "--physcpubind=%s" % cpu_list, + "--membind=%d" % node, + sys.executable, + worker_entrypoint, + "--copy-worker", + src, + tmp, + str(expected_size), + ] + try: + if cancel_event is None: + result = run(command) + else: + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + while process.poll() is None: + if cancel_event.wait(0.1): + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + _raise_if_cancelled(cancel_event) + stdout, stderr = process.communicate() + except BaseException: + if process.poll() is None: + process.kill() + process.wait() + raise + result = subprocess.CompletedProcess( + command, + process.returncode, + stdout, + stderr, + ) + if result.returncode: + raise RamdiskError( + "node-affined replica copy failed: %s" + % ( + result.stderr.strip() + or result.stdout.strip() + ) + ) + if os.path.getsize(tmp) != expected_size: + raise RamdiskError( + "replica size mismatch for %s" + % os.path.basename(src) + ) + _read_safetensors_header(tmp) + os.replace(tmp, destination) + if progress: + progress( + os.path.basename(src), + expected_size, + time.monotonic() - started, + ) + finally: + try: + os.unlink(tmp) + except OSError: + pass + + +def _populate_mount( + plan, + mount, + source_root=None, + progress=None, + cancel_event=None, + *, + available_for_mount=None, + copy_one=None, + copy_one_affined=None, + engine_cpu_list=None, +): + available_for_mount = ( + _available_for_mount + if available_for_mount is None + else available_for_mount + ) + copy_one = _copy_one if copy_one is None else copy_one + copy_one_affined = ( + _copy_one_affined + if copy_one_affined is None + else copy_one_affined + ) + source_root = source_root or plan["model"]["path"] + selected = plan["staging"]["selected_shards"] + linked = plan["staging"]["linked_shards"] + identities = { + item["name"]: item + for item in plan["source_shards"] + } + if mount.get("node") is None: + reserve_floor = ( + plan["reserve"]["runtime_bytes"] + + plan["reserve"]["page_table_bytes"] + + plan["reserve"]["os_margin_bytes"] + ) + else: + node_info = next( + item + for item in plan["hardware"]["nodes"] + if item["id"] == mount["node"] + ) + reserve_floor = ( + plan["reserve"]["runtime_bytes"] + + plan["reserve"]["page_table_bytes"] + + node_info.get("reserve_bytes", 8 * GIB) + ) + + def available(): + return available_for_mount(mount, plan=plan) + + workers = max( + 1, + min(plan["parallel"], len(selected) or 1), + ) + admission_lock = threading.Lock() + inflight = [0] + + def copy_name(name): + _raise_if_cancelled(cancel_event) + source = os.path.join(source_root, name) + destination = os.path.join(mount["path"], name) + expected = identities[name]["size_bytes"] + with admission_lock: + observed = available() + if ( + observed - inflight[0] - expected + < reserve_floor + ): + raise RamdiskError( + "projected shard copies would breach the protected " + "memory reserve" + ) + inflight[0] += expected + try: + if ( + source_root != plan["model"]["path"] + and mount["node"] is not None + ): + if engine_cpu_list is None: + raise RamdiskError( + "node-affined copy CPU selection is unavailable" + ) + return copy_one_affined( + source, + destination, + expected, + mount["node"], + plan["hardware"]["numactl"], + engine_cpu_list(plan, node=mount["node"]), + reserve_floor, + progress, + available, + cancel_event, + ) + return copy_one( + source, + destination, + expected, + reserve_floor, + progress, + available, + cancel_event, + ) + finally: + with admission_lock: + inflight[0] -= expected + + with concurrent.futures.ThreadPoolExecutor( + max_workers=workers + ) as executor: + futures = [ + executor.submit(copy_name, name) + for name in selected + ] + for future in concurrent.futures.as_completed(futures): + future.result() + _raise_if_cancelled(cancel_event) + for name in linked: + _raise_if_cancelled(cancel_event) + target = os.path.join(plan["model"]["path"], name) + destination = os.path.join(mount["path"], name) + os.symlink(target, destination) + + +def _mix_sample_value(value): + """Return a stable, well-distributed unsigned 64-bit value.""" + mask = (1 << 64) - 1 + value = (value + 0x9E3779B97F4A7C15) & mask + value = ( + (value ^ (value >> 30)) + * 0xBF58476D1CE4E5B9 + ) & mask + value = ( + (value ^ (value >> 27)) + * 0x94D049BB133111EB + ) & mask + return value ^ (value >> 31) + + +def _sample_page_indices(total_pages, sample_pages, node_count): + sample_pages = max(1, min(sample_pages, total_pages)) + if sample_pages == total_pages: + return list(range(total_pages)) + node_count = max(1, node_count) + if node_count == 1: + return [ + ((2 * sample + 1) * total_pages) + // (2 * sample_pages) + for sample in range(sample_pages) + ] + + eligible_orders = [ + order + for order in range(10) + if ( + total_pages + (1 << order) - 1 + ) // (1 << order) >= 7 * node_count + ] + residue_counts = { + order: [0] * node_count + for order in eligible_orders + } + seed = _mix_sample_value( + total_pages + ^ (sample_pages << 32) + ^ node_count + ) + indices = [] + for sample in range(sample_pages): + lower = sample * total_pages // sample_pages + upper = (sample + 1) * total_pages // sample_pages + target = float(sample + 1) / node_count + best = None + for salt in range(32): + value = lower + ( + _mix_sample_value( + seed + sample * 32 + salt + ) + % (upper - lower) + ) + maximum_distance = 0.0 + squared_distance = 0.0 + for order in eligible_orders: + residue = (value >> order) % node_count + for node, count in enumerate( + residue_counts[order] + ): + projected = count + ( + 1 if node == residue else 0 + ) + distance = abs(projected - target) + maximum_distance = max( + maximum_distance, + distance, + ) + squared_distance += distance * distance + score = ( + maximum_distance, + squared_distance, + salt, + ) + if best is None or score < best[0]: + best = (score, value) + value = best[1] + indices.append(value) + for order in eligible_orders: + residue_counts[order][ + (value >> order) % node_count + ] += 1 + return indices + + +def _sample_numa_allocation( + path, + max_pages=1024, + node_count=1, +): + """Touch a bounded page sample and report its Linux NUMA nodes.""" + if not get_platform_ops().is_linux: + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + counts = {} + size = os.path.getsize(path) + if size <= 0: + return counts + with open(path, "rb") as stream: + with mmap.mmap( + stream.fileno(), + 0, + access=mmap.ACCESS_READ, + ) as mapping: + total_pages = max(1, (size + 4095) // 4096) + pages = max(1, min(max_pages, total_pages)) + for page in _sample_page_indices( + total_pages, + pages, + node_count, + ): + mapping[min(size - 1, page * 4096)] + needle = "file=" + path.replace(" ", "\\040") + from .linux_ops import _read_text + + for line in _read_text( + "/proc/self/numa_maps" + ).splitlines(): + if needle not in line and path not in line: + continue + for node, value in re.findall( + r"\bN(\d+)=(\d+)\b", + line, + ): + counts[node] = ( + counts.get(node, 0) + int(value) + ) + return counts + + +def _validate_namespace( + plan, + mount, + sample_numa=True, + *, + sample_numa_allocation=None, +): + sample_numa_allocation = ( + _sample_numa_allocation + if sample_numa_allocation is None + else sample_numa_allocation + ) + expected_names = sorted( + item["name"] + for item in plan["source_shards"] + ) + actual_names = sorted( + name + for name in os.listdir(mount["path"]) + if name.endswith(".safetensors") + ) + if actual_names != expected_names: + raise RamdiskError( + "staged namespace filenames do not match the canonical model" + ) + identities = { + item["name"]: item + for item in plan["source_shards"] + } + selected = set(plan["staging"]["selected_shards"]) + linked = set(plan["staging"]["linked_shards"]) + for name in expected_names: + path = os.path.join(mount["path"], name) + if name in selected: + if ( + os.path.islink(path) + or not stat.S_ISREG(os.stat(path).st_mode) + ): + raise RamdiskError( + "staged shard is not a regular tmpfs file: %s" + % name + ) + if os.path.getsize(path) != identities[name]["size_bytes"]: + raise RamdiskError( + "staged shard size mismatch: %s" % name + ) + if os.stat(path).st_mode & 0o222: + raise RamdiskError( + "staged shard is writable: %s" % name + ) + raw, _ = _read_safetensors_header(path) + if ( + hashlib.sha256(raw).hexdigest() + != identities[name]["header_sha256"] + ): + raise RamdiskError( + "staged shard header mismatch: %s" % name + ) + elif name in linked: + if not os.path.islink(path): + raise RamdiskError( + "unstaged shard is not an SSD fallback symlink: %s" + % name + ) + canonical = os.path.join( + plan["model"]["path"], + name, + ) + if os.path.realpath(path) != os.path.realpath(canonical): + raise RamdiskError( + "fallback symlink does not target the canonical " + "shard: %s" % name + ) + allocation = {} + if not sample_numa: + return allocation + selected_names = plan["staging"]["selected_shards"] + placement_nodes = plan.get("placement", {}).get( + "memory_nodes", + plan["hardware"]["online_nodes"], + ) + pages_per_shard = max( + 32, + min(1024, 4096 // max(1, len(selected_names))), + ) + for name in selected_names: + path = os.path.join(mount["path"], name) + for node, count in sample_numa_allocation( + path, + pages_per_shard, + node_count=len(placement_nodes), + ).items(): + allocation[node] = allocation.get(node, 0) + count + online_nodes = plan.get("hardware", {}).get( + "online_nodes", + placement_nodes, + ) + verify_numa = ( + len(placement_nodes) > 1 + or len(online_nodes) > 1 + ) + if verify_numa: + total = sum(allocation.values()) + if not total: + raise RamdiskError( + "could not verify actual NUMA allocation for staged shards" + ) + outside = sum( + count + for node, count in allocation.items() + if int(node) not in placement_nodes + ) + if float(outside) / total > 0.01: + raise RamdiskError( + "tmpfs sample escaped the reviewed NUMA " + "memory-node mask" + ) + if mount["node"] is not None: + local = allocation.get(str(mount["node"]), 0) + if float(local) / total < 0.95: + raise RamdiskError( + "node-local tmpfs sample is below 95% local allocation" + ) + elif len(placement_nodes) > 1: + ideal = float(total) / len(placement_nodes) + deviations = [ + abs(allocation.get(str(node), 0) - ideal) + / ideal + for node in placement_nodes + ] + maximum_deviation = max(deviations) + if maximum_deviation > 0.15: + node_pages = ", ".join( + "%s=%d" + % ( + node, + allocation.get(str(node), 0), + ) + for node in placement_nodes + ) + raise RamdiskError( + "interleaved tmpfs sample is imbalanced: " + "node pages %s; maximum deviation %.1f%% " + "exceeds 15%%" + % ( + node_pages, + maximum_deviation * 100.0, + ) + ) + return allocation + + +def _source_still_matches(plan, *, scan_model_fn=None): + scan_model_fn = scan_model if scan_model_fn is None else scan_model_fn + current = scan_model_fn(plan["model"]["path"]) + if current["fingerprint"] != plan["model"]["fingerprint"]: + raise RamdiskError( + "canonical model changed while staging; refusing to " + "publish the manifest" + ) diff --git a/c/ramdisk_support/planning.py b/c/ramdisk_support/planning.py new file mode 100644 index 000000000..c035e5348 --- /dev/null +++ b/c/ramdisk_support/planning.py @@ -0,0 +1,1228 @@ +"""Pure profile, capacity, and placement planning primitives.""" + +from __future__ import print_function + +import copy +import json +import math +import os +import re +import subprocess + +from .accelerator import ( + GPU_LAYOUT_EXPERTS_ONLY, + GPU_LAYOUT_DENSE_ATTENTION_SHARDED, + GPU_VRAM_RESERVE_BYTES, + _normalize_gpu_layout, + _same_gpu_identity, + apply_gpu_selection, +) +from .common import ( + DEFAULT_MOUNT_ROOT, + GIB, + MANIFEST_VERSION, + MIB, + PLAN_SCHEMA, + PROFILE_LINE_RE, + RamdiskError, + _format_range_list, + _parse_range_list, + _path_is_below, + _path_without_symlinks, + _usage_engine_id, + _usage_engine_name, + _validated_usage_header, + _utc_now, +) + + +def _load_profile(path, model): + if not path: + path = os.path.join(model["path"], ".coli_usage") + if not os.path.isfile(path): + raise RamdiskError( + "partial staging requires .coli_usage or an explicit compatible --profile" + ) + counts = {} + header_records = [] + fingerprint = None + try: + with open(path, "r", encoding="utf-8") as stream: + text = stream.read() + if path.endswith(".json") or text.lstrip().startswith("{"): + payload = json.loads(text) + if not isinstance(payload, dict): + raise RamdiskError("profile JSON must contain an object") + fingerprint = payload.get("model_fingerprint") + rows = payload.get("counts", []) + if not isinstance(rows, list): + raise RamdiskError("profile JSON counts must be a list") + for row in rows: + if isinstance(row, dict): + layer, expert, count = row.get("layer"), row.get("expert"), row.get("count") + else: + layer, expert, count = row + counts["%d:%d" % (int(layer), int(expert))] = int(count) + else: + for number, line in enumerate(text.splitlines(), 1): + if not line.strip() or line.lstrip().startswith("#"): + continue + match = PROFILE_LINE_RE.match(line) + if not match: + raise RamdiskError("invalid profile line %d in %s" % (number, path)) + layer, expert, count = (int(value) for value in match.groups()) + if layer < 0: + header_records.append((layer, expert, count)) + continue + counts["%d:%d" % (layer, expert)] = count + except (OSError, ValueError, TypeError) as exc: + if isinstance(exc, RamdiskError): + raise + raise RamdiskError("cannot parse profile %s: %s" % (path, exc)) + if fingerprint and fingerprint != model["fingerprint"]: + raise RamdiskError("profile model fingerprint does not match the selected model") + if header_records: + config = model.get("config", {}) + engine = _usage_engine_name(config.get("model_type")) + _validated_usage_header( + header_records, + source="profile %s" % path, + expected_dimensions=( + int(config.get("num_hidden_layers", 0)), + int(config.get("n_routed_experts", 0)), + ), + expected_engine_id=_usage_engine_id(engine), + ) + compatible = {key: count for key, count in counts.items() if key in model["experts"] and count > 0} + if not compatible: + raise RamdiskError("profile contains no experts compatible with this model") + return os.path.realpath(path), compatible + +def _select_partial(model, counts, budget_bytes): + shard_sizes = {item["name"]: item["size_bytes"] for item in model["shards"]} + # Experts commonly share the same one- or two-shard closure. Grouping those + # closures makes the greedy score both exact and cheap: each candidate gets + # credit for every newly completed profiled expert, not just the expert that + # happened to nominate the shard set. + closure_groups = {} + for key in sorted(counts): + closure = frozenset(model["experts"][key]["shards"]) + group = closure_groups.setdefault(closure, {"keys": [], "benefit": 0}) + group["keys"].append(key) + group["benefit"] += counts[key] * model["experts"][key]["tensor_bytes"] + selected = set() + covered_closures = set() + while True: + candidates = [] + for closure in sorted(closure_groups, key=lambda value: tuple(sorted(value))): + if closure in covered_closures: + continue + trial = selected | set(closure) + added = trial - selected + cost = sum(shard_sizes[name] for name in added) + if not added or sum(shard_sizes[name] for name in selected) + cost > budget_bytes: + continue + newly_covered = [ + other + for other in closure_groups + if other not in covered_closures and other.issubset(trial) + ] + benefit = sum(closure_groups[item]["benefit"] for item in newly_covered) + ratio = float(benefit) / float(cost) + # The sorted closure tuple is the final deterministic tie-breaker. + candidates.append((ratio, benefit, -cost, tuple(sorted(closure)), trial)) + if not candidates: + break + _, _, _, _, selected = max(candidates) + covered_closures = { + closure for closure in closure_groups if closure.issubset(selected) + } + staged_experts = sorted( + key + for key, expert in model["experts"].items() + if set(expert["shards"]).issubset(selected) and expert["direct_map_eligible"] + ) + return sorted(selected), staged_experts + +def _runtime_reserve(model, ctx, direct_experts, cache_cap=8, kv_slots=1): + config = model.get("config") or {} + layers = int(config.get("num_hidden_layers", 0) or 0) + kv_lora = int(config.get("kv_lora_rank", 0) or 0) + rope = int(config.get("qk_rope_head_dim", 0) or 0) + index_dim = int(config.get("index_head_dim", 0) or 0) + qk_nope = int(config.get("qk_nope_head_dim", 0) or 0) + v_head = int(config.get("v_head_dim", 0) or 0) + heads = int(config.get("num_attention_heads", 0) or 0) + kv_bytes = (layers + 1) * max(1, ctx) * (kv_lora + rope) * 4 * kv_slots + index_bytes = layers * max(1, ctx) * index_dim * 4 * kv_slots + attention_scratch = max(1, ctx) * heads * (qk_nope + v_head) * 4 + dense = model["dense_tensor_bytes"] + direct = set(direct_experts) + fallback_by_layer = {} + for key, expert in model["experts"].items(): + if key not in direct: + fallback_by_layer.setdefault(expert["layer"], []).append(expert["tensor_bytes"]) + fallback_cache = sum( + min(cache_cap, len(sizes)) * max(sizes) + for sizes in fallback_by_layer.values() + if sizes + ) + max_expert = max((entry["tensor_bytes"] for entry in model["experts"].values()), default=0) + working_set = min(64, max((len(sizes) for sizes in fallback_by_layer.values()), default=0)) * max_expert + engine_overhead = max(int(1.2e9), dense // 100) + return { + "dense_bytes": dense, + "kv_bytes": kv_bytes, + "index_bytes": index_bytes, + "attention_scratch_bytes": attention_scratch, + "fallback_cache_bytes": fallback_cache, + "working_set_bytes": working_set, + "engine_overhead_bytes": engine_overhead, + } + +def _requested_ids(value, label, allowed, default): + """Normalize an operator range list without ever widening its effective mask.""" + allowed = sorted(set(int(item) for item in allowed)) + if value is None or value == "": + selected = sorted(set(int(item) for item in default)) + elif isinstance(value, str): + if len(value) > 4096: + raise RamdiskError("%s range list is unreasonably long" % label) + for token in value.split(","): + token = token.strip() + match = re.fullmatch(r"(\d+)(?:-(\d+))?", token) + if not match: + raise RamdiskError("%s must be a CPU/NUMA range list such as 0-3,8" % label) + start = int(match.group(1)) + end = int(match.group(2) or start) + if end < start: + raise RamdiskError("%s contains a descending range" % label) + if allowed and (start > allowed[-1] or end > allowed[-1]): + raise RamdiskError("%s requests IDs outside the effective host mask" % label) + selected = _parse_range_list(value) + elif isinstance(value, (list, tuple, set)): + if any( + isinstance(item, bool) or not isinstance(item, int) or item < 0 + for item in value + ): + raise RamdiskError("%s must contain non-negative integer IDs" % label) + selected = sorted(set(value)) + else: + raise RamdiskError("%s must be a CPU/NUMA range list" % label) + if not selected: + raise RamdiskError("%s resolves to an empty effective mask" % label) + outside = sorted(set(selected) - set(allowed)) + if outside: + raise RamdiskError( + "%s requests IDs outside the effective host mask: %s" + % (label, _format_range_list(outside)) + ) + return selected + +def _build_placement(args, hardware, topology): + """Resolve selected memory nodes and whole-core CPU masks for one plan.""" + online_nodes = sorted(set(int(node) for node in hardware.get("online_nodes", []))) + node_rows = { + int(node["id"]): node + for node in hardware.get("nodes", []) + if isinstance(node, dict) and isinstance(node.get("id"), int) + } + all_cpus = sorted( + { + int(cpu) + for node in node_rows.values() + for cpu in node.get("cpus", []) + if isinstance(cpu, int) and not isinstance(cpu, bool) and cpu >= 0 + } + ) + effective_nodes = sorted( + set(int(node) for node in hardware.get("effective_nodes", online_nodes)) + & set(online_nodes) + ) + effective_cpus = sorted( + set(int(cpu) for cpu in hardware.get("effective_cpus", all_cpus)) + & set(all_cpus) + ) + if not effective_nodes: + raise RamdiskError("the effective cpuset exposes no NUMA memory nodes") + if not effective_cpus: + raise RamdiskError("the effective cpuset exposes no CPUs") + + default_nodes = [ + node + for node in effective_nodes + if set(node_rows.get(node, {}).get("cpus", [])) & set(effective_cpus) + ] or effective_nodes + memory_nodes = _requested_ids( + getattr(args, "memory_nodes", None), + "--memory-nodes", + effective_nodes, + default_nodes, + ) + missing_rows = sorted(set(memory_nodes) - set(node_rows)) + if missing_rows: + raise RamdiskError( + "hardware discovery has no details for selected NUMA node(s): %s" + % _format_range_list(missing_rows) + ) + default_cpus = sorted( + set(effective_cpus) + & { + int(cpu) + for node in memory_nodes + for cpu in node_rows.get(node, {}).get("cpus", []) + } + ) or effective_cpus + cpus = _requested_ids( + getattr(args, "cpu_list", None), + "--cpu-list", + effective_cpus, + default_cpus, + ) + memory_node_cpus = { + int(cpu) + for node in memory_nodes + for cpu in node_rows.get(node, {}).get("cpus", []) + } + remote_cpus = sorted(set(cpus) - memory_node_cpus) + if topology == "per-node" and remote_cpus: + raise RamdiskError( + "per-node --cpu-list includes CPUs outside the selected replica nodes: %s" + % _format_range_list(remote_cpus) + ) + + raw_groups = hardware.get("core_groups") or [[cpu] for cpu in effective_cpus] + core_groups = [] + covered = set() + for raw_group in raw_groups: + group = sorted(set(int(cpu) for cpu in raw_group) & set(effective_cpus)) + if group and not (set(group) & covered): + core_groups.append(group) + covered.update(group) + core_groups.extend([[cpu] for cpu in effective_cpus if cpu not in covered]) + selected = set(cpus) + split_groups = [ + group + for group in core_groups + if selected.intersection(group) and not set(group).issubset(selected) + ] + if split_groups: + raise RamdiskError( + "--cpu-list must select whole effective physical cores; split sibling group(s): %s" + % ", ".join(_format_range_list(group) for group in split_groups) + ) + + engine_cpu_sets = [] + if topology == "interleaved": + targets = [(None, cpus)] + else: + targets = [ + ( + node, + sorted(set(cpus) & set(node_rows.get(node, {}).get("cpus", []))), + ) + for node in memory_nodes + ] + for node, engine_cpus in targets: + physical_cores = sum( + 1 for group in core_groups if set(group).issubset(set(engine_cpus)) + ) + engine_cpu_sets.append( + { + "node": node, + "cpus": engine_cpus, + "cpu_list": _format_range_list(engine_cpus), + "physical_cores": physical_cores, + } + ) + return { + "memory_nodes": memory_nodes, + "memory_node_list": _format_range_list(memory_nodes), + "cpus": cpus, + "cpu_list": _format_range_list(cpus), + "engine_cpu_sets": engine_cpu_sets, + "effective_nodes": effective_nodes, + "effective_node_list": _format_range_list(effective_nodes), + "effective_cpus": effective_cpus, + "effective_cpu_list": _format_range_list(effective_cpus), + "remote_cpus": remote_cpus, + "remote_cpu_list": _format_range_list(remote_cpus), + "memory_policy": ( + "equal-interleave" + if topology == "interleaved" and len(memory_nodes) > 1 + else "strict-bind" + ), + "dimm_control": "informational-only", + } + + +def _node_core_count(plan, node=None): + for target in plan.get("placement", {}).get("engine_cpu_sets", []): + if target.get("node") == node: + return max(1, int(target.get("physical_cores", 0))) + if node is None: + return max(1, int(plan["hardware"]["physical_cores"])) + try: + return max( + 1, + int( + next( + item["physical_cores"] + for item in plan["hardware"]["nodes"] + if int(item["id"]) == int(node) + ) + ), + ) + except StopIteration: + raise RamdiskError( + "NUMA node %s is absent from the recorded hardware plan" % node + ) + + +def _engine_cpu_list(plan, node=None): + for target in plan.get("placement", {}).get("engine_cpu_sets", []): + if target.get("node") == node: + value = target.get("cpu_list") + if isinstance(value, str) and value: + return value + cpus = target.get("cpus") + if isinstance(cpus, list) and cpus: + return _format_range_list(cpus) + if node is None: + cpus = plan.get("hardware", {}).get("effective_cpus") + if not cpus: + cpus = [ + cpu + for row in plan.get("hardware", {}).get("nodes", []) + for cpu in row.get("cpus", []) + ] + else: + cpus = next( + ( + row.get("cpus", []) + for row in plan.get("hardware", {}).get("nodes", []) + if row.get("id") == node + ), + [], + ) + if not cpus: + raise RamdiskError("managed engine CPU mask is empty") + return _format_range_list(cpus) + + +def _memory_node_list(plan, node=None): + if node is not None: + return str(int(node)) + nodes = plan.get("placement", {}).get( + "memory_nodes", + plan.get("hardware", {}).get("online_nodes", []), + ) + if not nodes: + raise RamdiskError("managed memory-node mask is empty") + return _format_range_list(nodes) + + +def _managed_numa_enabled(plan, node=None): + """Use the engine policy for every shared plan, including one-node binds.""" + if node is not None: + return False + nodes = plan.get("placement", {}).get( + "memory_nodes", + plan.get("hardware", {}).get("online_nodes", []), + ) + return bool(nodes) + + +def _normalize_managed_accelerator(args, hardware, placement): + """Normalize an internal preset accelerator draft into a plan contract.""" + raw = getattr(args, "managed_accelerator", None) + if raw is None: + return { + "mode": "cpu", + "layout": GPU_LAYOUT_EXPERTS_ONLY, + "devices": [], + "mmap": False, + "rammap": True, + "async_copy": False, + "vram_budget": None, + "capability": "not-requested", + } + if not isinstance(raw, dict) or raw.get("mode") != "cuda": + raise RamdiskError("managed accelerator draft is malformed") + layout = _normalize_gpu_layout(raw.get("layout")) + devices = raw.get("devices") + if not isinstance(devices, list) or not devices: + raise RamdiskError("managed CUDA staging requires at least one GPU") + discovered = { + int(device["index"]): device + for device in hardware.get("gpus", []) + if isinstance(device, dict) + and isinstance(device.get("index"), int) + and not isinstance(device.get("index"), bool) + } + selected_nodes = set(placement["memory_nodes"]) + normalized = [] + seen = set() + for cuda_ordinal, device in enumerate(devices): + if not isinstance(device, dict): + raise RamdiskError("managed accelerator device record is malformed") + index = device.get("index") + node = device.get("numa_node") + if ( + isinstance(index, bool) + or not isinstance(index, int) + or index < 0 + or index in seen + ): + raise RamdiskError("managed accelerator GPU indices are invalid") + if index not in discovered: + raise RamdiskError( + "managed accelerator GPU %d is absent from hardware discovery" + % index + ) + observed = discovered[index] + if ( + not isinstance(node, int) + or isinstance(node, bool) + or node not in selected_nodes + or observed.get("numa_node") != node + ): + raise RamdiskError( + "managed accelerator GPU %d is outside the reviewed NUMA placement" + % index + ) + if not _same_gpu_identity(device, observed): + raise RamdiskError( + "managed accelerator GPU %d identity changed" % index + ) + pci_bus_id = str(observed.get("pci_bus_id") or "") + uuid = str(observed.get("uuid") or device.get("uuid") or "") + if not uuid: + raise RamdiskError( + "managed accelerator GPU %d has no stable UUID" % index + ) + seen.add(index) + normalized.append( + { + "index": index, + "cuda_ordinal": cuda_ordinal, + "name": str(observed.get("name") or device.get("name") or ""), + "uuid": uuid, + "pci_bus_id": pci_bus_id, + "numa_node": node, + } + ) + if raw.get("vram_budget") != "auto": + raise RamdiskError("managed CUDA VRAM budget must be auto") + if raw.get("mmap") is not True or raw.get("rammap") is not False: + raise RamdiskError( + "managed CUDA staging requires mmap and disables direct RAM-map" + ) + if ( + layout == GPU_LAYOUT_DENSE_ATTENTION_SHARDED + and len(normalized) < 2 + ): + raise RamdiskError( + "dense-attention-sharded requires at least two selected GPUs" + ) + return { + "mode": "cuda", + "layout": layout, + "devices": normalized, + "mmap": True, + "rammap": False, + "async_copy": bool(raw.get("async_copy", True)), + "vram_budget": "auto", + "capability": str(raw.get("capability") or "unverified"), + } + + +def _accelerator_projection(contract, hardware, model): + discovered = { + int(device["index"]): device + for device in hardware.get("gpus") or [] + if isinstance(device, dict) + and isinstance(device.get("index"), int) + and not isinstance(device.get("index"), bool) + } + selected = [ + discovered.get(device["index"], {}) + for device in contract["devices"] + ] + selected_free = sum(int(device.get("free_bytes") or 0) for device in selected) + selected_total = sum( + int(device.get("total_bytes") or 0) for device in selected + ) + dense_tensor_bytes = int(model.get("dense_tensor_bytes") or 0) + dense_gpu_bytes = ( + dense_tensor_bytes + if contract["mode"] == "cuda" + and contract["layout"] != GPU_LAYOUT_EXPERTS_ONLY + else 0 + ) + reserve_bytes = ( + GPU_VRAM_RESERVE_BYTES * len(selected) + if contract["mode"] == "cuda" + else 0 + ) + per_device = [] + device_count = len(selected) + for position, (contract_device, device) in enumerate( + zip(contract["devices"], selected) + ): + dense_share = 0 + if dense_gpu_bytes and device_count: + dense_share = ( + dense_gpu_bytes // device_count + + ( + 1 + if position < dense_gpu_bytes % device_count + else 0 + ) + ) + free_bytes = int(device.get("free_bytes") or 0) + required_bytes = ( + GPU_VRAM_RESERVE_BYTES + dense_share + if contract["mode"] == "cuda" + else 0 + ) + per_device.append( + { + "index": contract_device["index"], + "uuid": contract_device.get("uuid", ""), + "free_bytes": free_bytes, + "total_bytes": int(device.get("total_bytes") or 0), + "projected_dense_bytes": dense_share, + "reserve_bytes": ( + GPU_VRAM_RESERVE_BYTES + if contract["mode"] == "cuda" + else 0 + ), + "expert_headroom_bytes": max( + 0, + free_bytes - required_bytes, + ), + "admission_ok": free_bytes >= required_bytes, + } + ) + return { + "dense_tensor_bytes": dense_tensor_bytes, + "dense_gpu_bytes": dense_gpu_bytes, + "vram_reserve_per_device_bytes": ( + GPU_VRAM_RESERVE_BYTES + if contract["mode"] == "cuda" + else 0 + ), + "selected_free_bytes": selected_free, + "selected_total_bytes": selected_total, + "expert_headroom_bytes": max( + 0, + selected_free - reserve_bytes - dense_gpu_bytes, + ), + "per_device": per_device, + "admission_scope": ( + "balanced-estimate" + if dense_gpu_bytes + else "exact-reserve-only" + ), + "exact_per_device_at_runtime": contract["mode"] == "cuda", + } + + +def _preset_metadata(args): + preset_id = getattr(args, "ramdisk_preset", None) + if not preset_id: + return None + label = getattr(args, "ramdisk_preset_label", None) or str(preset_id) + return { + "id": str(preset_id), + "label": str(label), + "state": "custom" if preset_id == "custom" else "selected", + "reason": str(getattr(args, "ramdisk_preset_reason", "") or ""), + "fallback": getattr(args, "ramdisk_preset_fallback", None), + } + + +def build_plan( + args, + hardware=None, + model=None, + *, + discover_hardware, + scan_model, + load_profile=_load_profile, + select_partial=_select_partial, + runtime_reserve=_runtime_reserve, + build_placement=_build_placement, + reusable_empty_mountpoint, + filesystem_for_path, + state_root, + manifest_path, + benchmarks_path, + current_euid, + get_platform_ops, +): + """Build a RAM-disk deployment plan using facade-supplied host services.""" + hardware = copy.deepcopy(hardware or discover_hardware()) + requested_gpu = getattr(args, "gpu", None) + if requested_gpu is not None: + requested_cpu = ( + isinstance(requested_gpu, str) + and requested_gpu.strip().lower() == "none" + ) + if ( + not requested_cpu + and getattr(args, "topology", "interleaved") + != "interleaved" + ): + raise RamdiskError( + "managed CUDA staging requires interleaved topology" + ) + planning_args = copy.deepcopy(args) + existing = getattr(args, "managed_accelerator", None) or {} + apply_gpu_selection( + planning_args, + hardware, + selector=requested_gpu, + layout=getattr(args, "gpu_layout", None), + cuda_capable=( + True + if existing.get("capability") == "available" + else None + ), + reset_placement=( + getattr(args, "memory_nodes", None) in (None, "") + and getattr(args, "cpu_list", None) in (None, "") + ), + ) + args = planning_args + elif ( + getattr(args, "gpu_layout", GPU_LAYOUT_EXPERTS_ONLY) + != GPU_LAYOUT_EXPERTS_ONLY + and getattr(args, "managed_accelerator", None) is None + ): + raise RamdiskError("--gpu-layout requires --gpu auto or a device list") + model = model or scan_model(args.model) + mode = getattr(args, "mode", "full") + topology = getattr(args, "topology", "interleaved") + capacity_gb = getattr(args, "capacity_gb", None) + if mode not in ("full", "partial") or topology not in ("interleaved", "per-node"): + raise RamdiskError("invalid RAM-disk mode or topology") + placement = build_placement(args, hardware, topology) + managed_accelerator = _normalize_managed_accelerator( + args, + hardware, + placement, + ) + if ( + managed_accelerator["mode"] == "cuda" + and topology != "interleaved" + ): + raise RamdiskError( + "managed CUDA staging requires interleaved topology" + ) + accelerator_projection = _accelerator_projection( + managed_accelerator, + hardware, + model, + ) + if capacity_gb is not None and ( + isinstance(capacity_gb, bool) + or not isinstance(capacity_gb, (int, float)) + or not math.isfinite(capacity_gb) + or capacity_gb <= 0 + ): + raise RamdiskError("--capacity-gb must be a finite positive number") + raw_ctx = getattr(args, "ctx", 0) + if isinstance(raw_ctx, bool) or not isinstance(raw_ctx, int) or raw_ctx < 0: + raise RamdiskError("--ctx must be zero (default) or a positive integer") + raw_parallel = getattr(args, "parallel", 2) + if ( + isinstance(raw_parallel, bool) + or not isinstance(raw_parallel, int) + or not 1 <= raw_parallel <= 64 + ): + raise RamdiskError("--parallel must be an integer between 1 and 64") + capacity_bytes = int(capacity_gb * GIB) if capacity_gb is not None else model["total_shard_bytes"] + profile_path = None + counts = None + if mode == "full": + selected = list(model["shard_names"]) + direct_experts = sorted( + key for key, entry in model["experts"].items() if entry["direct_map_eligible"] + ) + else: + if not capacity_gb or capacity_gb <= 0: + raise RamdiskError("partial staging requires a positive --capacity-gb budget") + profile_path, counts = load_profile(getattr(args, "profile", None), model) + selected, direct_experts = select_partial(model, counts, capacity_bytes) + if not selected: + raise RamdiskError("no complete shard closure fits the partial staging budget") + resident_experts = sorted( + key + for key, expert in model["experts"].items() + if set(expert["shards"]).issubset(selected) + ) + shard_sizes = {item["name"]: item["size_bytes"] for item in model["shards"]} + staged_bytes = sum(shard_sizes[name] for name in selected) + managed_ctx = int(raw_ctx or 4096) + managed_cache_cap = 8 + managed_kv_slots = 1 + managed_reserve = runtime_reserve( + model, + managed_ctx, + direct_experts, + cache_cap=managed_cache_cap, + kv_slots=managed_kv_slots, + ) + # The benchmark contract includes SSD and tmpfs-through-slab baselines even + # for a fully staged model. Those paths need the ordinary cap/LRU working + # set in addition to the resident tmpfs copy, so admission uses the larger + # of managed-direct and non-RAMMAP benchmark runtime projections. + benchmark_reserve = runtime_reserve( + model, + managed_ctx, + [], + cache_cap=managed_cache_cap, + kv_slots=managed_kv_slots, + ) + managed_runtime_bytes = sum(managed_reserve.values()) + benchmark_runtime_bytes = sum(benchmark_reserve.values()) + runtime_bytes = max(managed_runtime_bytes, benchmark_runtime_bytes) + memory = hardware["memory"] + selected_nodes = list(placement["memory_nodes"]) + selected_node_rows = [ + node for node in hardware.get("nodes", []) if node.get("id") in selected_nodes + ] + selected_total = sum( + int(node.get("memory_total_bytes", 0)) for node in selected_node_rows + ) or int(memory["total_bytes"]) + selected_available = sum( + int(node.get("memory_available_bytes", 0)) for node in selected_node_rows + ) or int(memory["available_bytes"]) + cgroup_memory = hardware.get("cgroup_memory") or {} + cgroup_available = cgroup_memory.get("available_bytes") + if ( + not isinstance(cgroup_available, int) + or isinstance(cgroup_available, bool) + or cgroup_available < 0 + ): + cgroup_available = None + cgroup_high_available = cgroup_memory.get("high_available_bytes") + if ( + not isinstance(cgroup_high_available, int) + or isinstance(cgroup_high_available, bool) + or cgroup_high_available < 0 + ): + cgroup_high_available = None + effective_available = ( + min(selected_available, cgroup_available) + if cgroup_available is not None + else selected_available + ) + global_margin = max(selected_total // 10, 16 * GIB) + page_tables = int(math.ceil(float(staged_bytes + runtime_bytes) / 512.0)) + required_global = staged_bytes + runtime_bytes + page_tables + global_margin + blockers = [] + warnings = [] + projection = accelerator_projection + if ( + managed_accelerator["mode"] == "cuda" + and managed_accelerator["layout"] != GPU_LAYOUT_EXPERTS_ONLY + and ( + projection["dense_gpu_bytes"] + + ( + len(managed_accelerator["devices"]) + * projection["vram_reserve_per_device_bytes"] + ) + > projection["selected_free_bytes"] + or any( + not device["admission_ok"] + for device in projection["per_device"] + ) + ) + ): + blockers.append( + "selected GPU free VRAM cannot hold the projected dense tensors " + "and per-device reserve" + ) + if ( + managed_accelerator["mode"] == "cuda" + and managed_accelerator["layout"] != GPU_LAYOUT_EXPERTS_ONLY + ): + warnings.append( + "per-card dense VRAM admission is a balanced estimate; " + "Operate reports exact tensor placement and any CPU fallback" + ) + if ( + managed_accelerator["mode"] == "cuda" + and managed_accelerator["capability"] != "available" + ): + warnings.append( + "CUDA engine capability was not proven during planning; " + "managed start will fail closed if the backend is unavailable" + ) + if cgroup_memory.get("error"): + blockers.append( + "cannot validate cgroup memory headroom: %s" + % cgroup_memory["error"] + ) + if not hardware["linux"]: + blockers.append("coli ramdisk is supported only on Linux") + if not hardware["tmpfs"]["supported"]: + blockers.append("tmpfs is not available in /proc/filesystems") + allow_swappable = bool(getattr(args, "allow_swappable", False)) + if not hardware["tmpfs"]["noswap_supported"] and not allow_swappable: + blockers.append("this kernel does not advertise tmpfs noswap; use --allow-swappable only if accepted") + if hardware["swap"]["used_bytes"]: + warnings.append("swap is already in use; managed commands never run swapoff") + if topology == "per-node" and not hardware.get("numactl"): + blockers.append("per-node topology requires numactl") + if topology == "per-node": + engine_cpu_sets = { + item["node"]: item for item in placement["engine_cpu_sets"] + } + for node in selected_node_rows: + if not node.get("cpus"): + blockers.append( + "NUMA node %d has no online CPUs and cannot host a node-local engine" + % node["id"] + ) + elif not engine_cpu_sets[node["id"]]["cpus"]: + blockers.append( + "NUMA node %d has no selected whole-core CPUs for its replica" + % node["id"] + ) + if capacity_bytes < staged_bytes: + blockers.append("selected shard closures exceed the staging budget") + if topology == "interleaved": + if selected_available < required_global: + blockers.append( + "selected NUMA nodes would breach the runtime/OS reserve" + ) + if placement["remote_cpus"]: + warnings.append( + "selected engine CPUs outside the memory-node mask will perform " + "intentional remote NUMA access: %s" % placement["remote_cpu_list"] + ) + if len(selected_nodes) > 1: + warnings.append( + "Linux interleave may fall back outside the selected nodes under severe " + "memory pressure; Colibri reserves headroom and verifies initial page placement" + ) + else: + for node in selected_node_rows: + margin = max(node["memory_total_bytes"] // 10, 8 * GIB) + node_page_tables = int(math.ceil(float(staged_bytes + runtime_bytes) / 512.0)) + required = staged_bytes + runtime_bytes + node_page_tables + margin + node["required_bytes"] = required + node["reserve_bytes"] = margin + if node["memory_available_bytes"] < required: + blockers.append("NUMA node %d cannot hold its replica and reserve" % node["id"]) + if mode == "partial": + total_count = sum(counts.values()) + covered_profile = [ + key + for key in counts + if set(model["experts"][key]["shards"]).issubset(selected) + ] + staged_count = sum(counts[key] for key in covered_profile) + coverage = float(staged_count) / total_count if total_count else 0.0 + predicted_avoided = sum( + counts[key] * model["experts"][key]["tensor_bytes"] + for key in covered_profile + ) + pin_selected = [] + pin_bytes = 0 + for key in sorted(counts, key=lambda item: (-counts[item], item)): + expert_bytes = model["experts"][key]["tensor_bytes"] + if pin_bytes + expert_bytes <= capacity_bytes: + pin_selected.append(key) + pin_bytes += expert_bytes + pin_count = sum(counts[key] for key in pin_selected) + pin_comparison = { + "budget_bytes": capacity_bytes, + "selected_experts": pin_selected, + "resident_expert_bytes": pin_bytes, + "coverage": float(pin_count) / total_count if total_count else 0.0, + } + else: + covered_profile = [] + coverage = 1.0 + predicted_avoided = sum( + model["experts"][key]["tensor_bytes"] for key in resident_experts + ) + pin_comparison = None + direct_bytes = sum(model["experts"][key]["tensor_bytes"] for key in direct_experts) + resident_expert_bytes = sum( + model["experts"][key]["tensor_bytes"] for key in resident_experts + ) + efficiency = float(resident_expert_bytes) / staged_bytes if staged_bytes else 0.0 + mount_root = os.path.abspath(os.path.expanduser(getattr(args, "mount_root", DEFAULT_MOUNT_ROOT))) + invoking_euid = current_euid() + mount_root_preexisting = os.path.isdir(mount_root) and not os.path.islink(mount_root) + forbidden_roots = { + "/", + "/bin", + "/boot", + "/dev", + "/etc", + "/home", + "/lib", + "/lib64", + "/mnt", + "/opt", + "/proc", + "/root", + "/run", + "/sbin", + "/srv", + "/sys", + "/tmp", + "/usr", + "/var", + os.path.normpath(os.path.expanduser("~")), + } + if os.path.normpath(mount_root) in forbidden_roots: + blockers.append("mount root is a protected broad directory") + try: + under_mnt = os.path.commonpath([mount_root, "/mnt"]) == "/mnt" and mount_root != "/mnt" + except ValueError: + under_mnt = False + if not under_mnt: + blockers.append("v1 managed mount roots must be below /mnt") + if os.path.realpath(mount_root) != mount_root: + blockers.append("mount root path must not traverse symbolic links") + if os.path.lexists(mount_root): + if os.path.islink(mount_root) or not os.path.isdir(mount_root): + blockers.append("mount root exists but is not a real directory") + elif os.stat(mount_root).st_mode & 0o022: + blockers.append("existing mount root must not be group/world writable") + elif invoking_euid != 0 and os.access(mount_root, os.W_OK): + blockers.append("existing mount root must not be writable by the invoking user") + elif topology == "interleaved": + try: + entries = os.listdir(mount_root) + reusable = bool(entries) and all( + re.fullmatch(r"node\d+", name) + and reusable_empty_mountpoint(os.path.join(mount_root, name)) + for name in entries + ) + if entries and not reusable: + blockers.append("interleaved mount root must be absent or empty") + elif reusable: + warnings.append( + "interleaved mount will temporarily cover verified empty node mountpoints from an earlier topology" + ) + except OSError as exc: + blockers.append("cannot inspect mount root: %s" % exc) + # Every existing parent below /mnt must be non-writable by this user, so a + # second process cannot exchange a directory for a symlink between review + # and the privileged mount(8) call. X-mount.mkdir creates absent parents. + if under_mnt: + parent = os.path.dirname(mount_root) + while parent.startswith("/mnt"): + if os.path.lexists(parent): + if os.path.islink(parent) or not os.path.isdir(parent): + blockers.append("mount root has an unsafe parent: %s" % parent) + break + if invoking_euid != 0 and os.access(parent, os.W_OK): + blockers.append("mount root parent is writable by the invoking user: %s" % parent) + break + if parent == "/mnt": + break + parent = os.path.dirname(parent) + model_path = os.path.normpath(model["path"]) + try: + if os.path.commonpath([mount_root, model_path]) in (mount_root, model_path): + blockers.append("mount root must not contain or be contained by the canonical model") + except ValueError: + blockers.append("mount root and model path are on incompatible path roots") + nodes = selected_nodes + requested_thp = getattr(args, "thp", "auto") or "auto" + thp = ( + "within_size" if hardware["thp"]["within_size_supported"] else "advise" + ) if requested_thp == "auto" else requested_thp + if thp == "within_size" and not hardware["thp"]["within_size_supported"]: + warnings.append("THP within_size is not advertised; mount will fall back to advise if rejected") + if thp == "advise" and not hardware["thp"]["advise_supported"]: + blockers.append("tmpfs THP advise mode is not available") + replicas = [None] if topology == "interleaved" else nodes + replica_count = len(replicas) + mounts = [] + for node in replicas: + path = mount_root if node is None else os.path.join(mount_root, "node%d" % node) + # A contiguous range avoids an option-separator comma in mount(8)'s + # ``-o`` string on the overwhelmingly common 0..N online-node layout. + node_list = _format_range_list(nodes) + if "," in node_list: + node_list = node_list.replace(",", "\\,") + # Prevent ordinal remapping while reviewed nodes remain allowed. + # Without ``static``, Linux maps the policy's ordinal nodes into a new + # cpuset; Start/Benchmark separately refuse every effective-mask drift. + policy = ( + "interleave=static:" + node_list + if node is None and len(nodes) > 1 + else "bind=static:%d" % (nodes[0] if node is None else node) + ) + mounts.append( + { + "node": node, + "path": path, + "path_preexisting": os.path.isdir(path) and not os.path.islink(path), + "policy": policy, + "size_bytes": max(staged_bytes + max(64 * MIB, staged_bytes // 100), 64 * MIB), + } + ) + if os.path.lexists(path): + if os.path.islink(path) or not os.path.isdir(path): + blockers.append("managed mount path exists but is not a real directory: %s" % path) + elif invoking_euid != 0 and os.access(path, os.W_OK): + blockers.append("managed mount path is writable by the invoking user: %s" % path) + else: + try: + if os.listdir(path): + blockers.append("managed mount path is not empty: %s" % path) + except OSError as exc: + blockers.append("cannot inspect managed mount path %s: %s" % (path, exc)) + durable_state = {} + try: + durable_state = { + "root": state_root(), + "manifest": manifest_path(), + "benchmarks": benchmarks_path(), + } + for label, durable_path in durable_state.items(): + if not _path_without_symlinks(durable_path): + blockers.append("durable %s path must not traverse symbolic links" % label) + if _path_is_below(durable_path, mount_root, allow_equal=True): + blockers.append("durable %s path must be outside every volatile mount" % label) + if hardware["linux"] and get_platform_ops().is_linux: + filesystem = filesystem_for_path(durable_path) + if filesystem in ("tmpfs", "ramfs"): + blockers.append( + "durable %s path is on volatile %s; use an SSD-backed XDG state directory" + % (label, filesystem) + ) + except (RamdiskError, OSError, subprocess.SubprocessError) as exc: + blockers.append(str(exc)) + total_staged_bytes = staged_bytes * replica_count + total_runtime_bytes = runtime_bytes * replica_count + total_page_table_bytes = page_tables * replica_count + if topology == "per-node": + total_os_margin = sum( + int(node.get("reserve_bytes", 0)) for node in selected_node_rows + ) + total_required = sum( + int(node.get("required_bytes", 0)) for node in selected_node_rows + ) + if selected_available < total_required: + blockers.append("available memory cannot hold all per-node replicas and reserves") + else: + total_os_margin = global_margin + total_required = required_global + if cgroup_available is not None and cgroup_available < total_required: + blockers.append( + "cgroup memory hard-limit headroom cannot hold the staged copies, " + "managed runtime, and reserve" + ) + if ( + cgroup_high_available is not None + and cgroup_high_available < total_required + ): + warnings.append( + "cgroup memory.high headroom is below the projected deployment; " + "staging or runtime may be heavily reclaimed/throttled" + ) + return { + "schema": PLAN_SCHEMA, + "version": MANIFEST_VERSION, + "created_at": _utc_now(), + "mode": mode, + "topology": topology, + "mount_root": mount_root, + "capacity_bytes": capacity_bytes, + "model": { + "path": model["path"], + "fingerprint": model["fingerprint"], + "fingerprint_algorithm": model["fingerprint_algorithm"], + "shard_count": len(model["shards"]), + "total_shard_bytes": model["total_shard_bytes"], + "dense_tensor_bytes": model["dense_tensor_bytes"], + "complete_experts": model["complete_experts"], + }, + "profile": { + "path": profile_path, + "coverage": coverage, + "staging_efficiency": efficiency, + "predicted_expert_bytes_avoided": predicted_avoided, + "predicted_expert_bytes_avoided_per_staged_byte": ( + float(predicted_avoided) / staged_bytes if staged_bytes else 0.0 + ), + "covered_experts": covered_profile, + "pin_comparison": pin_comparison, + }, + "staging": { + "selected_shards": selected, + "linked_shards": sorted(set(model["shard_names"]) - set(selected)), + "staged_bytes": staged_bytes, + "staged_experts": resident_experts, + "staged_expert_count": len(resident_experts), + "staged_expert_bytes": resident_expert_bytes, + "direct_mapped_experts": direct_experts, + "direct_mapped_expert_count": len(direct_experts), + "direct_mapped_bytes": direct_bytes, + "replica_count": replica_count, + "total_staged_bytes": total_staged_bytes, + }, + "reserve": { + "runtime": managed_reserve, + "benchmark_runtime": benchmark_reserve, + "managed_runtime_bytes": managed_runtime_bytes, + "benchmark_runtime_bytes": benchmark_runtime_bytes, + "runtime_bytes": runtime_bytes, + "page_table_bytes": page_tables, + "os_margin_bytes": global_margin, + "required_global_bytes": required_global, + "available_bytes": effective_available, + "host_available_bytes": selected_available, + "cgroup_available_bytes": cgroup_available, + "cgroup_high_available_bytes": cgroup_high_available, + "total_runtime_bytes": total_runtime_bytes, + "total_page_table_bytes": total_page_table_bytes, + "total_os_margin_bytes": total_os_margin, + "total_required_bytes": total_required, + }, + "placement": placement, + "hardware": hardware, + "mounts": mounts, + "mount_root_preexisting": mount_root_preexisting, + "mount_options": { + "noswap": hardware["tmpfs"]["noswap_supported"], + "allow_swappable": allow_swappable, + "thp": thp, + "fixed": ["noatime", "nodev", "nosuid", "noexec", "mode=0700"], + }, + "prefault": int( + getattr(args, "prefault", None) + if getattr(args, "prefault", None) is not None + else mode == "full" + ), + "parallel": raw_parallel, + "managed_runtime": { + "ctx": managed_ctx, + "kv_slots": managed_kv_slots, + "cache_cap": managed_cache_cap, + "autopin": 0, + "cap_raise": 0, + }, + "managed_accelerator": managed_accelerator, + "accelerator_projection": accelerator_projection, + "preset": _preset_metadata(args), + "blockers": sorted(set(blockers)), + "warnings": warnings, + "durable_state": durable_state, + # Internal source identities are retained in a plan used by prepare but + # omitted by the compact human renderer only. + "source_shards": model["shards"], + } diff --git a/c/ramdisk_support/platform_ops.py b/c/ramdisk_support/platform_ops.py new file mode 100644 index 000000000..99a8d9e2c --- /dev/null +++ b/c/ramdisk_support/platform_ops.py @@ -0,0 +1,85 @@ +"""Platform selection and explicit RAM-disk capability reports.""" + +from __future__ import print_function + +import os +import platform +import sys + +from .common import RamdiskError + + +UNSUPPORTED_PLATFORM_REASON = "coli ramdisk is supported only on Linux" + + +def current_uid(): + """Return the invoking UID without assuming a POSIX ``os`` module.""" + getuid = getattr(os, "getuid", None) + return int(getuid()) if getuid is not None else 1000 + + +def current_euid(): + """Return the effective UID, falling back to the portable invoking UID.""" + geteuid = getattr(os, "geteuid", None) + return int(geteuid()) if geteuid is not None else current_uid() + + +def _capabilities(platform_name, supported, reason=None): + return { + "platform": platform_name, + "hardware_discovery": bool(supported), + "cgroup_memory": bool(supported), + "numa": bool(supported), + "ramdisk_lifecycle": bool(supported), + "reason": reason, + } + + +def _unsupported_process_operation(*args, **kwargs): + del args, kwargs + raise RamdiskError(UNSUPPORTED_PLATFORM_REASON) + + +class UnsupportedPlatformOps: + """Portable facts for a host without a RAM-disk lifecycle backend.""" + + is_linux = False + process_control_supported = False + process_control_reason = UNSUPPORTED_PLATFORM_REASON + + def __init__(self, platform_name): + self.platform_name = platform_name + + def capabilities(self): + return _capabilities( + self.platform_name, + supported=False, + reason=UNSUPPORTED_PLATFORM_REASON, + ) + + def cpu_count(self): + return max(1, int(os.cpu_count() or 1)) + + def kernel_release(self): + return platform.release() + + process_start_boundary = staticmethod(_unsupported_process_operation) + process_identity = staticmethod(_unsupported_process_operation) + managed_launch_processes = staticmethod(_unsupported_process_operation) + process_group_member_pids = staticmethod(_unsupported_process_operation) + process_group_alive = staticmethod(_unsupported_process_operation) + signal_verified_process_group = staticmethod( + _unsupported_process_operation + ) + process_status = staticmethod(_unsupported_process_operation) + busy_mount_references = staticmethod(_unsupported_process_operation) + + +def get_platform_ops(platform_name=None): + """Select an operations backend without probing host facilities.""" + selected = sys.platform if platform_name is None else str(platform_name) + if selected.startswith("linux"): + from .linux_ops import LinuxPlatformOps + + return LinuxPlatformOps(selected) + return UnsupportedPlatformOps(selected) diff --git a/c/ramdisk_support/presentation.py b/c/ramdisk_support/presentation.py new file mode 100644 index 000000000..312d2b884 --- /dev/null +++ b/c/ramdisk_support/presentation.py @@ -0,0 +1,1434 @@ +"""Human-readable RAM-disk reports, review tokens, and view projections.""" + +from __future__ import print_function + +import hashlib +import json + +from .common import GIB, _format_range_list +from .presets import PRESET_CHOICES +from ramdisk_ui import ( + ActionPolicy, + DeploymentHealth, + HealthLevel, + PlacementContract, +) + + +def _placement_summary(plan, base_port=8000): + """Describe placement in user terms instead of implementation terms.""" + contract = PlacementContract.from_plan(plan, base_port) + copies = contract.copy_count + engines = contract.engine_count + nodes = list(contract.numa_nodes) + ports = list(contract.ports) + each_gib = contract.staged_bytes_per_copy / float(GIB) + total_gib = contract.total_staged_bytes / float(GIB) + full = contract.mode == "full" + copy_name = "complete model" if full else "selected shard set" + copy_word = "copy" if copies == 1 else "copies" + engine_word = "engine" if engines == 1 else "independent engines" + port_word = "port" if len(ports) == 1 else "ports" + endpoints = "%s %s" % ( + port_word, + ", ".join(str(port) for port in ports), + ) + node_labels = ["N%s" % node for node in nodes] + selected_cpus = plan.get("placement", {}).get("cpu_list") + cpu_clause = ( + " Selected engine CPUs: %s." % selected_cpus + if selected_cpus + else "" + ) + + if contract.is_shared: + title = "Single shared model (recommended)" + cost = "%d %s %s (%.2f GiB) · %d %s" % ( + copies, + copy_name, + copy_word, + total_gib, + engines, + engine_word, + ) + explanation = ( + "Stored once; RAM pages are spread across %d NUMA %s selected " + "for this plan and one engine serves one endpoint.%s" + % ( + len(nodes), + "node" if len(nodes) == 1 else "nodes", + cpu_clause, + ) + ) + rail = "MODEL x1 -> RAM [%s] -> ENGINE x1" % ( + " | ".join(node_labels) if node_labels else "host" + ) + else: + title = ( + "Independent full-model replicas (advanced)" + if full + else "Independent staged-set replicas (advanced)" + ) + cost = "%d %s %s (%d x %.2f GiB = %.2f GiB) · %d %s" % ( + copies, + copy_name, + copy_word, + copies, + each_gib, + total_gib, + engines, + engine_word, + ) + explanation = ( + "This is replication, not model sharding: every NUMA node " + "stores the entire staged set and serves a separate endpoint.%s" + % cpu_clause + ) + rail = "MODEL x%d -> %s -> ENGINES x%d" % ( + copies, + " ".join("[%s]" % label for label in node_labels) + or "[host]", + engines, + ) + return { + "title": title, + "cost": cost, + "explanation": explanation, + "rail": rail, + "endpoints": endpoints, + "copy_count": copies, + "engine_count": engines, + "ports": ports, + } + + +def _accelerator_review(plan): + accelerator = plan.get("managed_accelerator") or {} + if accelerator.get("mode") != "cuda": + return None + devices = accelerator.get("devices") or [] + projection = plan.get("accelerator_projection") or {} + dense_gpu_bytes = projection.get("dense_gpu_bytes") + expert_headroom_bytes = projection.get("expert_headroom_bytes") + reserve_per_device = projection.get( + "vram_reserve_per_device_bytes" + ) + has_projection = all( + isinstance(value, int) + for value in ( + dense_gpu_bytes, + expert_headroom_bytes, + reserve_per_device, + ) + ) + return { + "devices": devices, + "indices": ",".join(str(device["index"]) for device in devices), + "layout": str(accelerator.get("layout") or "experts-only"), + "dense_gpu_gib": ( + float(dense_gpu_bytes) / GIB + if has_projection + else None + ), + "expert_headroom_gib": ( + float(expert_headroom_bytes) / GIB + if has_projection + else None + ), + "reserve_per_device_gib": ( + float(reserve_per_device) / GIB + if has_projection + else None + ), + } + + +def _human_plan(plan): + placement = _placement_summary(plan) + print("RAM-disk plan: %s" % placement["title"]) + preset = plan.get("preset") or {} + if preset: + print( + " preset: %s%s" + % ( + preset.get("label", preset.get("id", "Custom")), + " (Custom)" if preset.get("state") == "custom" else "", + ) + ) + if preset.get("reason"): + print(" preset decision: %s" % preset["reason"]) + print(" model: %s" % plan["model"]["path"]) + print(" placement: %s" % placement["cost"]) + print(" endpoints after start: %s" % placement["endpoints"]) + print( + " NUMA memory nodes: %s; managed engine CPUs: %s" + % ( + plan.get("placement", {}).get( + "memory_node_list", + "all", + ), + plan.get("placement", {}).get("cpu_list", "all"), + ) + ) + print( + " DIMM/channel placement: informational only; " + "Linux allocates by NUMA node" + ) + print(" %s" % placement["explanation"]) + print( + " staged set: %d shard(s); %d direct expert(s)" + % ( + len(plan["staging"]["selected_shards"]), + plan["staging"]["direct_mapped_expert_count"], + ) + ) + accelerator = _accelerator_review(plan) + if accelerator is not None: + print( + " accelerator: CUDA selected GPU indices %s; devices %s; " + "layout %s; GPU-local NUMA %s; mmap upload; VRAM budget auto" + % ( + accelerator["indices"], + ", ".join( + "%s (%s)" + % ( + device["index"], + device.get("name") or "unnamed", + ) + for device in accelerator["devices"] + ), + accelerator["layout"], + _format_range_list( + sorted( + { + int(device["numa_node"]) + for device in accelerator["devices"] + } + ) + ), + ) + ) + if accelerator["dense_gpu_gib"] is not None: + print( + " GPU projection: dense %.2f GiB; expert headroom %.2f GiB; " + "reserve %.2f GiB/card" + % ( + accelerator["dense_gpu_gib"], + accelerator["expert_headroom_gib"], + accelerator["reserve_per_device_gib"], + ) + ) + print( + " total staged + OS/runtime projection: %.2f GiB; " + "available: %.2f GiB" + % ( + plan["reserve"]["total_required_bytes"] / float(GIB), + plan["reserve"]["available_bytes"] / float(GIB), + ) + ) + if plan["mode"] == "partial": + print( + " profile coverage: %.1f%%; staging efficiency: %.1f%%" + % ( + plan["profile"]["coverage"] * 100, + plan["profile"]["staging_efficiency"] * 100, + ) + ) + pin = plan["profile"]["pin_comparison"] + print( + " same-budget hot PIN comparison: %.1f%% profile coverage " + "with %d expert(s)" + % ( + pin["coverage"] * 100, + len(pin["selected_experts"]), + ) + ) + for warning in plan["warnings"]: + print(" warning: %s" % warning) + for blocker in plan["blockers"]: + print(" BLOCKED: %s" % blocker) + + +def _human_status(report): + print("RAM-disk state: %s" % report["state"]) + if not report["present"]: + return + for mount in report["mounts"]: + print( + " %s: %s" + % ( + mount["path"], + "verified tmpfs" + if mount["verified"] + else "missing/unverified", + ) + ) + for process in report["processes"]: + print( + " port %s PID %s: %s" + % ( + process["port"], + process["pid"], + process["reason"], + ) + ) + recovery = report.get("recovery") + if not isinstance(recovery, dict): + return + print( + " recovery: %s / %s" + % (recovery.get("operation"), recovery.get("state")) + ) + for path in recovery.get("retained_mounts", []): + print(" retained mount: %s" % path) + for path in recovery.get("released_mounts", []): + print(" released mount: %s" % path) + for process in recovery.get("retained_processes", []): + print( + " retained PID %s: %s (%s)" + % ( + process.get("pid"), + process.get("state_dir"), + process.get("error") or "absence unproven", + ) + ) + for pending in recovery.get("pending_launches", []): + print( + " outcome-unknown launch node %s port %s: %s" + % ( + pending.get("node"), + pending.get("port"), + pending.get("state_dir"), + ) + ) + errors = recovery.get("errors", {}) + if isinstance(errors, dict): + for name, value in errors.items(): + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + print( + " %s PID %s: %s" + % (name, item.get("pid"), item.get("error")) + ) + else: + print(" %s: %s" % (name, item)) + else: + print(" %s: %s" % (name, value)) + if recovery.get("action"): + print(" action: %s" % recovery["action"]) + + +def _human_benchmark(result): + print( + "RAM-disk benchmark (%s / %s)" + % (result["mode"], result["topology"]) + ) + for variant in result["variants"]: + if variant.get("status") != "ok": + print( + " %-30s %s" + % (variant["name"], variant.get("status")) + ) + continue + interactive = variant["interactive"] + print( + " %-30s TTFT %s ms tok/s p50 %s p95 %s RAM %.1f%%" + % ( + variant["name"], + "%.1f" % interactive["ttft_ms"] + if interactive["ttft_ms"] is not None + else "n/a", + "%.3f" % interactive["p50_tokens_per_second"] + if interactive["p50_tokens_per_second"] is not None + else "n/a", + "%.3f" % interactive["p95_tokens_per_second"] + if interactive["p95_tokens_per_second"] is not None + else "n/a", + interactive["ram_map_coverage"] * 100, + ) + ) + print( + " forward p50/p99 %s/%s ms SSD %s bytes/token" + % ( + "%.1f" % interactive["forward_p50_ms"] + if interactive["forward_p50_ms"] is not None + else "n/a", + "%.1f" % interactive["forward_p99_ms"] + if interactive["forward_p99_ms"] is not None + else "n/a", + "%.0f" % interactive["ssd_bytes_per_token"] + if interactive["ssd_bytes_per_token"] is not None + else "n/a", + ) + ) + aggregate = result["aggregate"] + print( + " aggregate: %s slowest %s tok/s total %s tok/s" + % ( + aggregate.get("status"), + "%.3f" % aggregate["slowest_node_tokens_per_second"] + if aggregate.get("slowest_node_tokens_per_second") + is not None + else "n/a", + "%.3f" % aggregate["total_tokens_per_second"] + if aggregate.get("total_tokens_per_second") is not None + else "n/a", + ) + ) + system = result["system"] + print( + " system: stage %s s prefault %s s RSS %s GiB " + "mount shmem %.2f GiB" + % ( + "%.1f" % system["stage_seconds"] + if system.get("stage_seconds") is not None + else "n/a", + "%.2f" % system["prefault_seconds"] + if system.get("prefault_seconds") is not None + else "n/a", + "%.2f" % (system["rss_bytes"] / GIB) + if system.get("rss_bytes") is not None + else "n/a", + system["shmem_bytes"] / float(GIB), + ) + ) + print( + " swap +%.3f GiB host huge-page coverage %.1f%% NUMA %s" + % ( + system["swap_delta_bytes"] / float(GIB), + system["huge_page_coverage"] * 100, + system["numa_page_placement"], + ) + ) + print( + " acceptance: paths=%s outputs=%s no-swap-growth=%s " + "within-budget=%s" + % ( + result["acceptance"].get( + "all_required_paths_succeeded" + ), + result["acceptance"]["greedy_outputs_identical"], + result["acceptance"]["no_swap_growth"], + result["acceptance"]["staging_within_budget"], + ) + ) + if ( + result["acceptance"].get( + "full_zero_physical_ssd_reads_verified" + ) + is not None + ): + print( + " full direct physical SSD reads measured zero: %s" + % result["acceptance"][ + "full_zero_physical_ssd_reads_verified" + ] + ) + print( + " best knobs for this topology: %s" + % result["best_runtime_knobs"] + ) + + +def _plan_confirmation_token(plan): + """Stable identity for exactly the plan a user reviewed in the TUI.""" + reviewed = { + "schema": plan.get("schema"), + "version": plan.get("version"), + "model_fingerprint": plan.get("model", {}).get("fingerprint"), + "mode": plan.get("mode"), + "topology": plan.get("topology"), + "placement": plan.get("placement"), + "mount_root": plan.get("mount_root"), + "capacity_bytes": plan.get("capacity_bytes"), + "selected_shards": plan.get("staging", {}).get( + "selected_shards" + ), + "linked_shards": plan.get("staging", {}).get( + "linked_shards" + ), + "total_staged_bytes": plan.get("staging", {}).get( + "total_staged_bytes" + ), + "total_required_bytes": plan.get("reserve", {}).get( + "total_required_bytes" + ), + "mounts": plan.get("mounts"), + "mount_options": plan.get("mount_options"), + "prefault": plan.get("prefault"), + "parallel": plan.get("parallel"), + "managed_runtime": plan.get("managed_runtime"), + "managed_accelerator": plan.get("managed_accelerator"), + "preset": plan.get("preset"), + } + payload = json.dumps( + reviewed, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _manifest_confirmation_token( + manifest, + *, + persisted_base_port, +): + """Bind a destructive confirmation to one prepared deployment.""" + mounts = [] + for record in manifest.get("mounts", []): + identity = record.get("identity", {}) + mounts.append( + { + "path": record.get("path"), + "node": record.get("node"), + "mount_id": identity.get("mount_id"), + "device": identity.get("device"), + } + ) + processes = [] + for record in manifest.get("processes", []): + processes.append( + { + "pid": record.get("pid"), + "pgid": record.get("pgid"), + "uid": record.get("uid"), + "starttime": record.get("starttime"), + "nonce": record.get("nonce"), + "port": record.get("port"), + "node": record.get("node"), + } + ) + reviewed = { + "version": manifest.get("version"), + "deployment_id": manifest.get("deployment_id"), + "created_at": manifest.get("created_at"), + "state": manifest.get("state"), + "base_port": persisted_base_port(manifest), + "model_fingerprint": manifest.get("model_fingerprint"), + "plan_token": _plan_confirmation_token( + manifest.get("plan", {}) + ), + "mounts": mounts, + "processes": processes, + } + payload = json.dumps( + reviewed, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _prepare_confirmation(plan, base_port=8000): + contract = PlacementContract.from_plan(plan, base_port) + placement = _placement_summary(plan, base_port) + copies = contract.copy_count + each_gib = contract.staged_bytes_per_copy / float(GIB) + total_gib = contract.total_staged_bytes / float(GIB) + copy_name = ( + "complete model" + if contract.mode == "full" + else "selected shard set" + ) + if contract.is_shared: + nodes = len(contract.numa_nodes) + accelerator = _accelerator_review(plan) + accelerator_text = "" + if accelerator is not None: + accelerator_text = ( + "GPU(s) %s use layout %s. " + % ( + accelerator["indices"], + accelerator["layout"], + ) + ) + if accelerator["dense_gpu_gib"] is not None: + accelerator_text += ( + "Projected dense VRAM %.2f GiB and expert headroom " + "%.2f GiB after %.2f GiB/card reserve. " + % ( + accelerator["dense_gpu_gib"], + accelerator["expert_headroom_gib"], + accelerator["reserve_per_device_gib"], + ) + ) + return ( + "CONFIRM SHARED PLAN: stage %d %s copy (%.2f GiB) at %s, " + "spread across %d NUMA %s. Memory nodes %s; engine CPUs %s. " + "%sStart will launch 1 engine on %s. tmpfs size is a cap, THP " + "is requested rather than guaranteed, and copy workers do not " + "create replicas. Press p again within 10s." + % ( + copies, + copy_name, + total_gib, + plan["mount_root"], + nodes, + "node" if nodes == 1 else "nodes", + plan.get("placement", {}).get( + "memory_node_list", + "all", + ), + plan.get("placement", {}).get("cpu_list", "all"), + accelerator_text, + placement["endpoints"], + ) + ) + return ( + "CONFIRM REPLICA PLAN: stage %d %s copies " + "(%d x %.2f GiB = %.2f GiB) at %s. Memory nodes %s; " + "selected CPUs %s. Start will launch %d independent engines on %s. " + "This is replication, not sharding, and does not accelerate one " + "request. Press p again within 10s." + % ( + copies, + copy_name, + copies, + each_gib, + total_gib, + plan["mount_root"], + plan.get("placement", {}).get( + "memory_node_list", + "all", + ), + plan.get("placement", {}).get("cpu_list", "all"), + placement["engine_count"], + placement["endpoints"], + ) + ) + + +def _prepare_confirmation_rows(plan, base_port=8000): + """Put the irreversible topology facts in the first three TUI rows.""" + contract = PlacementContract.from_plan(plan, base_port) + placement = _placement_summary(plan, base_port) + copies = contract.copy_count + engines = contract.engine_count + if contract.mode == "full": + copies_text = "%d complete model %s" % ( + copies, + "copy" if copies == 1 else "copies", + ) + else: + copies_text = "%d selected shard-set %s" % ( + copies, + "copy" if copies == 1 else "copies", + ) + accelerator = None + if contract.is_replication: + placement_text = "DANGER · replication, not sharding" + engines_text = "%d independent engines" % engines + else: + nodes = len(contract.numa_nodes) + placement_text = "SHARED · pages span %d NUMA %s" % ( + nodes, + "node" if nodes == 1 else "nodes", + ) + engines_text = "%d engine on %s" % ( + engines, + placement["endpoints"], + ) + accelerator = _accelerator_review(plan) + if accelerator is not None: + engines_text += " · GPU(s) %s" % accelerator["indices"] + placement_text += " · layout %s" % accelerator["layout"] + rows = [ + ("warn", "REVIEW · %s" % copies_text), + ("warn", "START · %s" % engines_text), + ( + "bad" if contract.is_replication else "accent", + placement_text, + ), + ] + if ( + not contract.is_replication + and accelerator is not None + and accelerator["dense_gpu_gib"] is not None + ): + rows.append( + ( + "normal", + "VRAM · dense %.2f GiB · expert headroom %.2f GiB · " + "reserve %.2f GiB/card" + % ( + accelerator["dense_gpu_gib"], + accelerator["expert_headroom_gib"], + accelerator["reserve_per_device_gib"], + ), + ) + ) + return rows + + +def _tui_plan_rows( + plan, + report, + active=False, + base_port=8000, + confirmation=None, +): + placement = _placement_summary(plan, base_port) + rows = [] + if confirmation: + rows.extend(_prepare_confirmation_rows(plan, base_port)) + rows.append(("normal", "")) + rows.extend( + [ + ( + "dim", + "ACTIVE DEPLOYMENT" + if active + else "DRAFT PLAN · nothing has been changed yet", + ), + ( + "warn" + if plan["topology"] == "per-node" + else "accent", + placement["title"], + ), + ("accent", placement["rail"]), + ("normal", placement["cost"]), + ( + "normal", + "After Start: %s" % placement["endpoints"], + ), + ( + "warn" if plan["topology"] == "per-node" else "dim", + placement["explanation"], + ), + ("normal", ""), + *( + [ + ( + "accent", + "PRESET · %s%s" + % ( + plan["preset"].get("label", "Custom"), + ( + " · CUSTOM" + if plan["preset"].get("state") == "custom" + else "" + ), + ), + ), + ( + "dim", + plan["preset"].get("reason") or "Reviewed draft values.", + ), + ("normal", ""), + ] + if plan.get("preset") + else [] + ), + ( + "heading", + "STAGING · %s" + % ( + "full model" + if plan["mode"] == "full" + else "profile-selected shard set" + ), + ), + ( + "normal", + "%d of %d shards in RAM · %d direct-mapped experts · " + "prefault %s" + % ( + len(plan["staging"]["selected_shards"]), + plan["model"]["shard_count"], + plan["staging"]["direct_mapped_expert_count"], + "on" if plan["prefault"] else "off", + ), + ), + ( + "normal", + "%s host memory %.2f GiB · %s available %.2f GiB" + % ( + "Planned" if active else "Projected", + plan["reserve"]["total_required_bytes"] + / float(GIB), + "at preparation" if active else "currently", + plan["reserve"]["available_bytes"] / float(GIB), + ), + ), + ] + ) + accelerator = _accelerator_review(plan) + if accelerator is not None: + devices = accelerator["devices"] + rows.extend( + [ + ( + "heading", + "GPU STAGING · one shared source, one multi-GPU engine", + ), + ( + "accent", + "GPU(s) %s · PCI/NUMA %s" + % ( + " · ".join( + "%s %s" + % ( + device["index"], + device.get("name") or "unnamed", + ) + for device in devices + ), + " · ".join( + "%s → N%s" + % ( + device.get("pci_bus_id", "?"), + device.get("numa_node", "?"), + ) + for device in devices + ), + ), + ), + ( + "normal", + "Selected GPU indices %s · layout %s · " + "COLI_MMAP upload · async CUDA copies" + % ( + accelerator["indices"], + accelerator["layout"], + ), + ), + ] + ) + if accelerator["dense_gpu_gib"] is not None: + rows.append( + ( + "normal", + "Projected dense VRAM %.2f GiB · expert headroom " + "%.2f GiB · reserve %.2f GiB/card" + % ( + accelerator["dense_gpu_gib"], + accelerator["expert_headroom_gib"], + accelerator["reserve_per_device_gib"], + ), + ) + ) + if plan["mode"] == "partial": + rows.append( + ( + "normal", + "Profile coverage %.1f%% · staging efficiency %.1f%%" + % ( + plan["profile"]["coverage"] * 100, + plan["profile"]["staging_efficiency"] * 100, + ), + ) + ) + if confirmation: + rows.extend( + [ + ("normal", ""), + ("warn", "FULL PREPARATION DETAIL"), + ("warn", confirmation), + ] + ) + if active: + health = DeploymentHealth.from_report(plan, report) + if health.level is HealthLevel.VERIFIED: + health_style = "good" + health_title = "DEPLOYMENT VERIFIED" + health_detail = ( + "Persisted settings are locked. Activity shows current " + "mount and engine health." + ) + elif health.level is HealthLevel.FAST_CHECK: + health_style = "warn" + health_title = ( + "FAST CHECK PASSED · DEEP VERIFICATION PENDING" + ) + health_detail = ( + "Press R for source and NUMA verification; Start also " + "revalidates before launch." + ) + else: + health_style = "bad" + health_title = "DEPLOYMENT NEEDS ATTENTION" + health_detail = ( + "Open Activity and press R before Start; Destroy " + "revalidates every exact mount." + ) + rows.extend( + [ + ("normal", ""), + (health_style, health_title), + ( + "dim" if health.fast_check_ok else "bad", + health_detail, + ), + ] + ) + elif plan["blockers"]: + rows.extend([("normal", ""), ("bad", "NOT READY")]) + rows.extend( + ("bad", blocker) + for blocker in plan["blockers"] + ) + else: + rows.extend([("normal", ""), ("good", "READY")]) + if confirmation: + rows.append( + ( + "warn", + "Press p again before the confirmation expires, " + "or any change cancels it.", + ) + ) + else: + rows.append( + ( + "dim", + "Review the copy count and memory total above, " + "then press p to prepare.", + ) + ) + rows.extend( + ("warn", warning) + for warning in plan["warnings"] + ) + if report.get("present"): + rows.append( + ( + "dim", + "Lifecycle state: %s" + % report.get("state", "unknown"), + ) + ) + return rows + + +def _tui_hardware_rows(hardware): + nodes = hardware.get("nodes", []) + rows = [ + ("heading", "HOST MEMORY TOPOLOGY"), + ( + "normal", + "%.1f GiB available / %.1f GiB total · %d physical cores · " + "%d NUMA %s" + % ( + hardware["memory"]["available_bytes"] / float(GIB), + hardware["memory"]["total_bytes"] / float(GIB), + hardware["physical_cores"], + len(nodes), + "node" if len(nodes) == 1 else "nodes", + ), + ), + ( + "dim", + "NUMA nodes determine RAM placement. CPU cores do not " + "create model copies.", + ), + ( + "normal", + "Kernel %s · tmpfs %s · noswap %s · THP %s" + % ( + hardware["kernel_release"], + "available" + if hardware["tmpfs"]["supported"] + else "missing", + "available" + if hardware["tmpfs"]["noswap_supported"] + else "missing", + hardware["thp"]["shmem_enabled"] or "unknown", + ), + ), + ( + "warn" if hardware["swap"]["used_bytes"] else "dim", + "Swap in use %.2f GiB" + % ( + hardware["swap"]["used_bytes"] + / float(GIB) + ), + ), + ("normal", ""), + ] + for node in nodes: + rows.extend( + [ + ( + "accent", + "NUMA %s · CPUs %s · %d physical cores" + % ( + node["id"], + node["cpu_list"], + node["physical_cores"], + ), + ), + ( + "normal", + " %.1f GiB available / %.1f GiB total · " + "distance %s" + % ( + node["memory_available_bytes"] + / float(GIB), + node["memory_total_bytes"] / float(GIB), + node["distance"], + ), + ), + ] + ) + return rows + + +def _tui_activity_rows( + report, + hardware, + process_metrics=None, + *, + meminfo, +): + rows = [ + ( + "heading", + "LIFECYCLE · %s" + % report.get("state", "unknown").upper(), + ) + ] + if not report.get("present"): + rows.extend( + [ + ("dim", "No RAM workspace exists yet."), + ( + "normal", + "Review the Plan page, then prepare it with p.", + ), + ] + ) + return rows + rows.append( + ( + "dim", + "%s validation · manifest %s" + % ( + "deep" + if report.get("deep_validation") + else "fast", + report.get("manifest_path"), + ), + ) + ) + rows.extend( + [("normal", ""), ("heading", "RAM MOUNTS")] + ) + for mount in report.get("mounts", []): + rows.append( + ( + "good" if mount.get("verified") else "bad", + "%s · %s · NUMA pages %s" + % ( + mount["path"], + "verified" + if mount.get("verified") + else "missing or unverified", + mount.get("numa_allocation") + or "not sampled", + ), + ) + ) + rows.extend( + [("normal", ""), ("heading", "MANAGED ENGINES")] + ) + processes = report.get("processes", []) + if not processes: + rows.append( + ( + "dim", + "No engine is running. Prepared weights stay " + "resident until Destroy.", + ) + ) + for process in processes: + rows.append( + ( + "good" if process.get("running") else "dim", + "port %s · PID %s · node %s · %s" + % ( + process.get("port"), + process.get("pid"), + process.get("node"), + process.get("reason"), + ), + ) + ) + metrics = (process_metrics or {}).get( + process.get("pid"), + {}, + ) + if metrics.get("rss_bytes") is not None: + rows.append( + ( + "dim", + " RSS %.2f GiB across %d processes · " + "RAM map %s experts / %s GiB" + % ( + metrics["rss_bytes"] / float(GIB), + metrics["rss_processes"], + metrics["rammap_experts"] + if metrics["rammap_experts"] is not None + else "n/a", + "%.2f" + % (metrics["rammap_bytes"] / GIB) + if metrics["rammap_bytes"] is not None + else "n/a", + ), + ) + ) + mem = meminfo() + rows.extend( + [ + ("normal", ""), + ( + "dim", + "Host shared memory %.2f GiB · swap %.3f GiB" + % ( + mem.get("Shmem", 0) / float(GIB), + hardware["swap"]["used_bytes"] / float(GIB), + ), + ), + ] + ) + return rows + + +def _tui_benchmark_rows(history): + rows = [("heading", "PERSISTENT PATH SCORECARD")] + results = (history or {}).get("results", []) + if not results: + rows.extend( + [ + ("dim", "No benchmark history yet."), + ( + "normal", + "Prepare the workspace, stop managed engines, " + "then press b here.", + ), + ] + ) + return rows + latest = results[-1] + rows.append( + ( + "accent", + "Latest %s · best %s" + % ( + latest.get("created_at"), + latest.get("best_variant"), + ), + ) + ) + for variant in latest.get("variants", []): + if variant.get("status") != "ok": + rows.append( + ( + "warn", + "%s · %s" + % ( + variant.get("name"), + variant.get("status"), + ), + ) + ) + continue + score = variant.get("interactive", {}) + rows.append( + ( + "normal", + "%s · TTFT %s ms · %.2f tok/s p50 · RAM %.1f%% · " + "SSD %s B/token" + % ( + variant.get("name"), + "%.1f" % score["ttft_ms"] + if score.get("ttft_ms") is not None + else "n/a", + score.get("p50_tokens_per_second") or 0.0, + ( + score.get("ram_map_coverage") + or 0.0 + ) + * 100, + "%.0f" % score["ssd_bytes_per_token"] + if score.get("ssd_bytes_per_token") + is not None + else "n/a", + ), + ) + ) + aggregate = latest.get("aggregate", {}) + rows.append( + ( + "dim", + "Aggregate %s · slowest %s tok/s · total %s tok/s" + % ( + aggregate.get("status", "n/a"), + aggregate.get( + "slowest_node_tokens_per_second", + "n/a", + ), + aggregate.get( + "total_tokens_per_second", + "n/a", + ), + ), + ) + ) + return rows + + +def _tui_settings_rows( + args, + plan, + report, + base_port=8000, +): + rows = [("heading", "WORKSPACE SETTINGS")] + preset = plan.get("preset") or {} + if preset: + rows.extend( + [ + ( + "accent" if preset.get("state") != "custom" else "warn", + "Preset · %s%s" + % ( + preset.get("label", preset.get("id", "Custom")), + " · Custom" if preset.get("state") == "custom" else "", + ), + ), + ("dim", preset.get("reason") or "Advanced draft values."), + ] + ) + if report.get("present"): + placement = _placement_summary(plan, base_port) + can_change_port = report.get("state") in ( + "ready", + "stopped", + ) + rows.extend( + [ + ("warn", "LOCKED BY ACTIVE DEPLOYMENT"), + ("normal", placement["title"]), + ("normal", placement["cost"]), + ( + "normal" if can_change_port else "dim", + ( + "[P] Next Start base port %s" + if can_change_port + else "Current base port %s" + ) + % base_port, + ), + ( + "dim", + "Start uses the persisted weights plan shown here. " + "Stop before changing its next endpoint; Destroy " + "before changing placement or staging.", + ), + ] + ) + return rows + placement = _placement_summary(plan, base_port) + rows.extend( + [ + ( + "warn" + if plan["topology"] == "per-node" + else "accent", + "Placement · %s" % placement["title"], + ), + ("normal", placement["cost"]), + ("dim", placement["explanation"]), + ] + ) + if plan["topology"] == "per-node": + rows.append( + ("good", "[i] Return to one shared copy") + ) + else: + rows.append( + ( + "dim", + "Replica mode is explicit-only: choose Multiple NUMA " + "replicas at startup or pass --topology per-node.", + ) + ) + rows.extend( + [ + ("normal", ""), + ("normal", "[m] Staging mode %s" % args.mode), + ( + "normal", + "[c] Per-copy budget %s" + % ( + "%.1f GiB" % args.capacity_gb + if args.capacity_gb + else "full model size" + ), + ), + ( + "normal", + "[r] Usage profile %s" + % (args.profile or "/.coli_usage"), + ), + ( + "normal", + "[o] Mount root %s" % args.mount_root, + ), + ( + "normal", + "[P] Base port %s" % args.base_port, + ), + ( + "normal", + "[w] Copy workers %s " + "(copy concurrency only)" % args.parallel, + ), + ( + "normal", + "[H] Huge pages %s" % args.thp, + ), + ( + "normal", + "[f] Prefault %s" + % ("on" if plan["prefault"] else "off"), + ), + ( + "normal", + "[y] Swappable tmpfs %s" + % ( + "allowed" + if args.allow_swappable + else "refused" + ), + ), + ("normal", ""), + ( + "dim", + "Full mode always stages the full model; capacity " + "changes only apply to partial mode.", + ), + ] + ) + return rows + + +def _tui_preset_rows(): + rows = [ + ("heading", "WHAT SHOULD COLIBRI OPTIMIZE?"), + ( + "dim", + "Choose once to prepopulate the draft. Nothing is mounted or copied.", + ), + ("normal", ""), + ] + for index, (_preset_id, label, description) in enumerate( + PRESET_CHOICES, + 1, + ): + rows.append( + ( + "accent" if index == 1 else "normal", + "[%d] %s%s" + % ( + index, + label, + " · default" if index == 1 else "", + ), + ) + ) + rows.append(("dim", " %s" % description)) + rows.extend( + [ + ("normal", ""), + ( + "dim", + "Enter selects Fastest GPU staging. Advanced settings remain editable.", + ), + ] + ) + return rows + + +def _tui_help_rows(): + return [ + ("heading", "HOW THIS WORKS"), + ( + "normal", + "1. Plan shows exactly how many model copies, engines, " + "ports, and GiB will be created.", + ), + ( + "normal", + "2. Prepare mounts tmpfs and copies weights. It does not " + "start an engine.", + ), + ( + "normal", + "3. Start launches the persisted deployment; Stop keeps " + "RAM weights; Destroy unmounts them.", + ), + ("normal", ""), + ( + "accent", + "Shared placement is the normal path: one model copy and " + "one engine across all NUMA nodes.", + ), + ( + "warn", + "Per-node means independent full replicas, not a model " + "split. It is never enabled by a TUI toggle.", + ), + ("normal", ""), + ("heading", "KEYS"), + ("normal", "Left/Right or h/l · change page"), + ("normal", "Up/Down or j/k · scroll"), + ( + "normal", + "p · review/prepare s · start " + "x · stop d · destroy", + ), + ( + "normal", + "b · benchmark R · deep refresh " + "? · close help", + ), + ( + "normal", + "c · cancel prepare/start/benchmark at a safe " + "cleanup checkpoint", + ), + ( + "normal", + "Settings page · edit draft settings before preparation", + ), + ( + "normal", + "q or Esc · quit; long operations cancel safely first, " + "cleanup finishes before exit", + ), + ] + + +def _tui_idle_action_hint(screen, plan, report): + """Return only actions the shared lifecycle policy permits.""" + policy = ActionPolicy.from_state(plan, report) + if ( + screen == 0 + and report + and not report.get("present") + and policy.prepare.enabled + ): + return "[p] review / prepare" + if screen == 3 and policy.benchmark.enabled: + return "[b] benchmark" + if policy.start.enabled: + return "[s] start [d] destroy" + if policy.stop.enabled: + return "[x] stop" + if policy.destroy.enabled: + return "[d] destroy" + return "[R] refresh" diff --git a/c/ramdisk_support/presets.py b/c/ramdisk_support/presets.py new file mode 100644 index 000000000..b93bb7a8b --- /dev/null +++ b/c/ramdisk_support/presets.py @@ -0,0 +1,385 @@ +"""Pure first-run RAM-workspace preset resolution.""" + +from __future__ import print_function + +import argparse +import copy +import os + +from .accelerator import apply_gpu_selection, eligible_gpu_devices +from .common import GIB, MIB, RamdiskError + + +PRESET_GPU_FASTEST = "gpu-fastest" +PRESET_SINGLE = "single" +PRESET_MINIMAL = "minimal" +PRESET_REPLICAS = "replicas" + +PRESET_CHOICES = ( + ( + PRESET_GPU_FASTEST, + "Fastest GPU staging", + "One shared copy on GPU-local NUMA nodes; one multi-GPU engine.", + ), + ( + PRESET_SINGLE, + "Single RAM copy", + "One full shared copy using the normal effective NUMA placement.", + ), + ( + PRESET_MINIMAL, + "Minimal RAM", + "Largest safe profile-guided partial staging set.", + ), + ( + PRESET_REPLICAS, + "Multiple NUMA replicas", + "Advanced: one complete copy and independent engine per NUMA node.", + ), +) + +_PRESET_LABELS = { + preset_id: label for preset_id, label, _description in PRESET_CHOICES +} + + +def _engine_cuda_capable(engine_path): + """Return whether a local engine contains the CUDA backend marker.""" + if not engine_path: + return None + try: + with open(os.path.realpath(engine_path), "rb") as stream: + while True: + block = stream.read(1024 * 1024) + if not block: + return False + if b"[CUDA] mode: routed experts" in block: + return True + except OSError: + return False + + +def _namespace(args): + return argparse.Namespace(**copy.deepcopy(vars(args))) + + +def _set_common(draft, preset_id): + draft.ramdisk_preset = preset_id + draft.ramdisk_preset_label = _PRESET_LABELS[preset_id] + draft.ramdisk_preset_reason = "" + draft.ramdisk_preset_fallback = None + draft.managed_accelerator = None + draft.gpu = "none" + draft.gpu_layout = "experts-only" + draft.prefault = None + return draft + + +def _single_draft(args, preset_id=PRESET_SINGLE): + draft = _set_common(_namespace(args), preset_id) + draft.mode = "full" + draft.topology = "interleaved" + draft.capacity_gb = None + draft.memory_nodes = None + draft.cpu_list = None + return draft + + +def _replica_draft(args): + draft = _set_common(_namespace(args), PRESET_REPLICAS) + draft.mode = "full" + draft.topology = "per-node" + draft.capacity_gb = None + draft.memory_nodes = None + draft.cpu_list = None + draft.ramdisk_preset_reason = ( + "Explicit replication: every selected NUMA node receives a complete " + "copy and an independent managed engine." + ) + return draft + + +def _gpu_local_draft(args, hardware, selector, cuda_capable): + layout = getattr(args, "gpu_layout", None) + draft = _set_common(_namespace(args), PRESET_GPU_FASTEST) + draft.mode = "full" + draft.capacity_gb = None + apply_gpu_selection( + draft, + hardware, + selector=selector, + layout=layout, + cuda_capable=cuda_capable, + reset_placement=True, + ) + devices = draft.managed_accelerator["devices"] + draft.ramdisk_preset_reason = ( + "One shared model copy across GPU-local NUMA node(s) %s; " + "one managed engine uses GPU(s) %s." + % ( + draft.memory_nodes, + ",".join(str(device["index"]) for device in devices), + ) + ) + return draft + + +def _memory_admitted(plan): + reserve = plan.get("reserve") or {} + available = reserve.get("available_bytes") + required = reserve.get("total_required_bytes") + if not isinstance(available, int) or not isinstance(required, int): + return False + if available < required: + return False + memory_blockers = ( + "memory hard-limit", + "memory headroom", + "cannot hold", + "breach the runtime/OS reserve", + "replicas and reserves", + ) + return not any( + any(marker in blocker for marker in memory_blockers) + for blocker in plan.get("blockers", []) + ) + + +def _partial_upper_bytes(full_plan, model): + reserve = full_plan.get("reserve") or {} + available = int(reserve.get("available_bytes") or 0) + runtime = int(reserve.get("runtime_bytes") or 0) + margin = int(reserve.get("os_margin_bytes") or 0) + upper = max(0, available - runtime - margin) + return min(int(model["total_shard_bytes"]), upper) + + +def _minimum_profile_closure(model, counts): + shard_sizes = { + item["name"]: int(item["size_bytes"]) + for item in model["shards"] + } + costs = [] + for key in counts: + expert = model["experts"].get(key) + if expert is None: + continue + costs.append( + sum(shard_sizes[name] for name in set(expert["shards"])) + ) + return min(costs) if costs else 0 + + +def _blocked_from(plan, message, draft): + blocked = copy.deepcopy(plan) + blocked["blockers"] = sorted( + set(list(blocked.get("blockers", [])) + [message]) + ) + blocked["preset"] = { + "id": draft.ramdisk_preset, + "label": draft.ramdisk_preset_label, + "state": "selected", + "reason": draft.ramdisk_preset_reason, + "fallback": draft.ramdisk_preset_fallback, + } + return blocked + + +def _partial_plan( + draft, + full_plan, + hardware, + model, + *, + build_plan, + load_profile, +): + upper = _partial_upper_bytes(full_plan, model) + draft.mode = "partial" + draft.capacity_gb = max(float(MIB) / GIB, float(upper) / GIB) + try: + _profile_path, counts = load_profile( + getattr(draft, "profile", None), + model, + ) + except RamdiskError as exc: + return draft, _blocked_from( + full_plan, + "profile-guided staging is unavailable: %s" % exc, + draft, + ) + minimum = _minimum_profile_closure(model, counts) + if upper < minimum or minimum <= 0: + return draft, _blocked_from( + full_plan, + "no complete profile-guided shard closure fits the safe RAM budget", + draft, + ) + + # The planner owns reserve arithmetic. Start from its projected staging + # ceiling, then remove exactly the reported deficit until admitted. + budget = upper + last_plan = None + for _attempt in range(16): + draft.capacity_gb = max( + float(MIB) / GIB, + float(budget + 1023) / GIB, + ) + try: + candidate = build_plan( + draft, + hardware=hardware, + model=model, + ) + except RamdiskError: + candidate = None + if candidate is not None: + last_plan = candidate + if _memory_admitted(candidate): + return draft, candidate + reserve = candidate.get("reserve") or {} + deficit = max( + MIB, + int(reserve.get("total_required_bytes") or 0) + - int(reserve.get("available_bytes") or 0), + ) + else: + deficit = MIB + budget -= deficit + if budget < minimum: + break + return draft, _blocked_from( + last_plan or full_plan, + "no complete profile-guided shard closure fits the safe RAM budget", + draft, + ) + + +def _annotate(plan, draft): + plan = copy.deepcopy(plan) + plan["preset"] = { + "id": draft.ramdisk_preset, + "label": draft.ramdisk_preset_label, + "state": ( + "custom" + if draft.ramdisk_preset == "custom" + else "selected" + ), + "reason": draft.ramdisk_preset_reason, + "fallback": draft.ramdisk_preset_fallback, + } + return plan + + +def mark_preset_custom(args): + """Mark a selected draft custom without discarding accelerator settings.""" + selected = getattr(args, "ramdisk_preset", None) + if not selected or selected == "custom": + return args + previous = getattr(args, "ramdisk_preset_label", None) or str(selected) + args.ramdisk_preset = "custom" + args.ramdisk_preset_label = "Custom" + args.ramdisk_preset_reason = "Advanced values edited from %s." % previous + args.ramdisk_preset_fallback = None + return args + + +def resolve_preset( + preset_id, + args, + *, + hardware, + model, + build_plan, + load_profile, + cuda_capable=None, +): + """Return populated draft arguments and the authoritative reviewed plan.""" + if preset_id not in _PRESET_LABELS: + raise RamdiskError("unknown RAM-workspace preset: %s" % preset_id) + + if preset_id == PRESET_REPLICAS: + draft = _replica_draft(args) + plan = build_plan(draft, hardware=hardware, model=model) + return {"args": draft, "plan": _annotate(plan, draft)} + + if preset_id == PRESET_SINGLE: + draft = _single_draft(args) + plan = build_plan(draft, hardware=hardware, model=model) + return {"args": draft, "plan": _annotate(plan, draft)} + + if preset_id == PRESET_GPU_FASTEST: + devices = list(hardware.get("gpus") or []) + usable = eligible_gpu_devices(hardware) + selector = getattr(args, "gpu", None) or "auto" + if isinstance(selector, str): + selector = selector.strip().lower() + fallback_reason = None + if selector == "none": + fallback_reason = "GPU staging was disabled by --gpu none" + elif cuda_capable is False: + fallback_reason = ( + "the selected engine does not contain the CUDA backend" + ) + elif cuda_capable is not True: + fallback_reason = ( + "CUDA engine capability could not be established" + ) + elif not usable: + fallback_reason = ( + hardware.get("gpu_discovery", {}).get("error") + or ( + "no usable NVIDIA GPU was detected" + if devices + else "no NVIDIA GPU was detected" + ) + ) + if fallback_reason: + draft = _single_draft(args, PRESET_GPU_FASTEST) + draft.ramdisk_preset_fallback = PRESET_SINGLE + draft.ramdisk_preset_reason = ( + "GPU-aware staging fell back to one ordinary shared copy: %s." + % fallback_reason + ) + plan = build_plan(draft, hardware=hardware, model=model) + plan = _annotate(plan, draft) + plan["warnings"] = list(plan.get("warnings", [])) + [ + draft.ramdisk_preset_reason + ] + return {"args": draft, "plan": plan} + draft = _gpu_local_draft( + args, + hardware, + selector, + cuda_capable, + ) + else: + draft = _set_common(_namespace(args), PRESET_MINIMAL) + draft.topology = "interleaved" + draft.memory_nodes = None + draft.cpu_list = None + draft.ramdisk_preset_reason = ( + "Profile-guided staging is sized to the largest safely admitted " + "shard closure." + ) + + full_draft = _namespace(draft) + full_draft.mode = "full" + full_draft.capacity_gb = None + full_plan = build_plan( + full_draft, + hardware=hardware, + model=model, + ) + if preset_id == PRESET_GPU_FASTEST and _memory_admitted(full_plan): + return {"args": draft, "plan": _annotate(full_plan, draft)} + + draft, plan = _partial_plan( + draft, + full_plan, + hardware, + model, + build_plan=build_plan, + load_profile=load_profile, + ) + return {"args": draft, "plan": _annotate(plan, draft)} diff --git a/c/ramdisk_support/processes.py b/c/ramdisk_support/processes.py new file mode 100644 index 000000000..76c784741 --- /dev/null +++ b/c/ramdisk_support/processes.py @@ -0,0 +1,889 @@ +"""Managed process identity, admission, readiness, and cleanup helpers.""" + +from __future__ import print_function + +import json +import os +import re +import signal +import subprocess +import threading +import time +import urllib.error +import urllib.request + +from .accelerator import _same_gpu_identity +from .common import ( + GIB, + RamdiskError, + _positive_int, + _raise_if_cancelled, + _utc_now, +) +from .platform_ops import get_platform_ops + + +def _proc_identity(pid): + return get_platform_ops().process_identity(pid) + + +def _process_group_members( + pgid, + *, + ops=None, + proc_identity=None, +): + """Return identities only from a bounded stable membership view.""" + ops = get_platform_ops() if ops is None else ops + proc_identity = ( + _proc_identity + if proc_identity is None + else proc_identity + ) + pgid = int(pgid) + + def identity_fingerprint(identity): + # Scheduler state (R/S/D) is expected to change between reads. Only + # compare the stable identity and persisted-attribution fields. + return tuple( + identity.get(key) + for key in ( + "pid", + "uid", + "inert", + "starttime", + "nonce", + "pgid", + "sid", + "state_dir", + "weights_dir", + ) + ) + + def scan(): + before = list(ops.process_group_member_pids(pgid)) + if before != sorted(set(before)): + return [], sorted(set(before) or {pgid}), None + identities = [] + unreadable = [] + for pid in before: + identity = proc_identity(pid) + if ( + isinstance(identity, dict) + and identity.get("pid") == pid + ): + identities.append(identity) + else: + unreadable.append(pid) + after = list(ops.process_group_member_pids(pgid)) + if after != before: + unreadable.extend(set(before) | set(after)) + if unreadable: + return identities, sorted(set(unreadable)), None + return identities, [], ( + before, + [identity_fingerprint(identity) for identity in identities], + ) + + first_members, first_unreadable, first_view = scan() + if first_unreadable: + return first_members, first_unreadable + second_members, second_unreadable, second_view = scan() + if second_unreadable: + return second_members, second_unreadable + if first_view != second_view: + pids = { + member.get("pid") + for member in first_members + second_members + if isinstance(member, dict) + and isinstance(member.get("pid"), int) + } + return second_members, sorted(pids or {pgid}) + + # Empty and all-inert snapshots authorize irreversible state cleanup, so + # couple them to kernel liveness and one final complete identity scan. + if not second_members or all( + _proven_inert_group_member(member) + for member in second_members + ): + alive_before = ops.process_group_alive(pgid) + final_members, final_unreadable, final_view = scan() + alive_after = ops.process_group_alive(pgid) + if final_unreadable: + return final_members, final_unreadable + if final_view != second_view: + pids = { + member.get("pid") + for member in second_members + final_members + if isinstance(member, dict) + and isinstance(member.get("pid"), int) + } + return final_members, sorted(pids or {pgid}) + if alive_before or alive_after: + return final_members, [pgid] + return final_members, [] + return second_members, [] + + +def _proven_inert_group_member(identity): + """Return whether procfs proved one stable process identity inert.""" + return ( + isinstance(identity, dict) + and identity.get("inert") is True + and _positive_int(identity.get("pid")) + and _positive_int(identity.get("starttime")) + ) + + +def _inert_group_member_matches(record, identity, expected_pgid): + """Validate one proven-dead member within a mixed live group.""" + if ( + not _proven_inert_group_member(identity) + or identity.get("uid") != record.get("uid") + or identity.get("pgid") != expected_pgid + or identity.get("sid") != expected_pgid + ): + return False + if identity["pid"] == int(record["pid"]): + return identity["starttime"] == record.get("starttime") + return True + + +def _live_group_member_matches(record, identity, expected_pgid): + """Require complete persisted attribution for every runnable member.""" + return ( + isinstance(identity, dict) + and identity.get("inert") is False + and identity.get("uid") == record.get("uid") + and identity.get("nonce") == record.get("nonce") + and identity.get("pgid") == expected_pgid + and identity.get("sid") == expected_pgid + and identity.get("state_dir") == record.get("state_dir") + and identity.get("weights_dir") == record.get("weights_dir") + ) + + +def _process_matches( + record, + *, + proc_identity=None, + process_group_members=None, +): + proc_identity = ( + _proc_identity + if proc_identity is None + else proc_identity + ) + process_group_members = ( + _process_group_members + if process_group_members is None + else process_group_members + ) + pid = int(record["pid"]) + expected_pgid = int(record.get("pgid", pid)) + + def attribution_matches(identity): + if isinstance(identity, dict) and identity.get("inert") is True: + # Stable lone-thread zombies cannot run, retain files, or mutate + # usage. Every runnable member still needs full attribution. + return _inert_group_member_matches( + record, + identity, + expected_pgid, + ) + return _live_group_member_matches(record, identity, expected_pgid) + + actual = proc_identity(pid) + if ( + _proven_inert_group_member(actual) + and actual.get("pid") == pid + ): + actual = None + if not actual: + # The Python serve wrapper can die before its engine child. A managed + # session keeps the original PGID, so validate every surviving + # member's inherited UID+nonce before treating the group as signalable. + members, unreadable = process_group_members(expected_pgid) + if not members and not unreadable: + return False, "not-running", None + if unreadable: + return ( + False, + "unverified-process-group", + {"pgid": expected_pgid, "members": unreadable}, + ) + if all(_proven_inert_group_member(member) for member in members): + if any( + not _inert_group_member_matches( + record, + member, + expected_pgid, + ) + for member in members + ): + return ( + False, + "foreign-process-group", + {"pgid": expected_pgid, "members": members}, + ) + return False, "not-running", None + if any(not attribution_matches(member) for member in members): + return ( + False, + "foreign-process-group", + {"pgid": expected_pgid, "members": members}, + ) + return ( + True, + "running-group", + {"pid": pid, "pgid": expected_pgid, "members": members}, + ) + if actual["uid"] != record.get("uid"): + return False, "foreign-uid", actual + if actual["starttime"] != record.get("starttime"): + return False, "reused-pid", actual + if actual["nonce"] != record.get("nonce"): + return False, "foreign-nonce", actual + if actual["pgid"] != expected_pgid: + return False, "foreign-process-group", actual + if actual.get("sid") != expected_pgid: + return False, "foreign-session", actual + if any( + actual.get(key) != record.get(key) + for key in ("state_dir", "weights_dir") + ): + return False, "foreign-path-attribution", actual + members, unreadable = process_group_members(expected_pgid) + if unreadable or not members: + return ( + False, + "unverified-process-group", + {"pgid": expected_pgid, "members": unreadable}, + ) + if all(_proven_inert_group_member(member) for member in members): + if any( + not _inert_group_member_matches( + record, + member, + expected_pgid, + ) + for member in members + ): + return ( + False, + "foreign-process-group", + {"pgid": expected_pgid, "members": members}, + ) + return False, "not-running", None + if any(not attribution_matches(member) for member in members): + return ( + False, + "foreign-process-group", + {"pgid": expected_pgid, "members": members}, + ) + running = dict(actual) + running["members"] = members + return True, "running", running + + +def _process_tree_alive( + record, + actual, + *, + group_alive=None, + proc_identity=None, +): + group_alive = _group_alive if group_alive is None else group_alive + proc_identity = ( + _proc_identity + if proc_identity is None + else proc_identity + ) + expected_pgid = int(record.get("pgid", record["pid"])) + if actual and actual.get("pgid") == expected_pgid: + return group_alive(expected_pgid) + return bool(proc_identity(int(record["pid"]))) + + +def _runtime_admission_requirement(plan, mount, benchmark=False): + """Return the runtime, page-table, and protected host floor.""" + reserve = plan["reserve"] + runtime_bytes = int( + reserve.get( + "benchmark_runtime_bytes" + if benchmark + else "managed_runtime_bytes" + ) + or reserve["runtime_bytes"] + ) + page_tables = int(reserve["page_table_bytes"]) + if mount.get("node") is None: + margin = int(reserve["os_margin_bytes"]) + else: + node = next( + item + for item in plan["hardware"]["nodes"] + if item["id"] == mount["node"] + ) + margin = int( + node.get( + "reserve_bytes", + max(node["memory_total_bytes"] // 10, 8 * GIB), + ) + ) + return runtime_bytes + page_tables + margin + + +def _admit_runtime( + plan, + mount, + benchmark=False, + *, + available_for_mount=None, +): + """Recheck the reviewed post-staging floor immediately before launch.""" + if available_for_mount is None: + raise RamdiskError("runtime memory availability is unavailable") + required = _runtime_admission_requirement( + plan, + mount, + benchmark=benchmark, + ) + available = available_for_mount(mount, plan=plan) + if available < required: + label = ( + "global memory" + if mount.get("node") is None + else "NUMA node %d" % mount["node"] + ) + raise RamdiskError( + "%s has %d bytes available; launch would breach the " + "%d-byte runtime/OS floor" + % (label, available, required) + ) + return { + "available_bytes": available, + "required_bytes": required, + } + + +def _admit_concurrent_runtimes( + plan, + mounts, + benchmark=False, + *, + host_available_for_mount=None, + cgroup_available_memory=None, +): + """Admit replicas against one shared cgroup-headroom snapshot.""" + if host_available_for_mount is None: + raise RamdiskError("host memory availability is unavailable") + if cgroup_available_memory is None: + raise RamdiskError("cgroup memory availability is unavailable") + mounts = list(mounts) + if not mounts: + raise RamdiskError( + "concurrent runtime admission requires at least one mount" + ) + admissions = [] + for mount in mounts: + required = _runtime_admission_requirement( + plan, + mount, + benchmark=benchmark, + ) + host_available = host_available_for_mount(mount, plan=plan) + if host_available < required: + label = ( + "global memory" + if mount.get("node") is None + else "NUMA node %d" % mount["node"] + ) + raise RamdiskError( + "%s has %d bytes available; launch would breach the " + "%d-byte runtime/OS floor" + % (label, host_available, required) + ) + admissions.append( + { + "mount": mount, + "host_available_bytes": host_available, + "required_bytes": required, + } + ) + cgroup_available = cgroup_available_memory() + aggregate_required = sum( + item["required_bytes"] + for item in admissions + ) + if ( + cgroup_available is not None + and cgroup_available < aggregate_required + ): + raise RamdiskError( + "cgroup memory has %d bytes available; concurrent launch would " + "breach the %d-byte aggregate runtime/OS floor" + % (cgroup_available, aggregate_required) + ) + return { + "mounts": admissions, + "cgroup_available_bytes": cgroup_available, + "required_bytes": aggregate_required, + } + + +def _assert_effective_masks_unchanged( + plan, + *, + discover_hardware=None, +): + """Refuse launch after the reviewed cgroup/cpuset contract drifts.""" + hardware = plan.get("hardware", {}) + placement = plan.get("placement") + accelerator = plan.get("managed_accelerator") or {} + check_masks = bool( + placement + and hardware.get("effective_mask_source") + == "kernel-task-status" + ) + check_gpus = accelerator.get("mode") == "cuda" + if not check_masks and not check_gpus: + return + if discover_hardware is None: + raise RamdiskError("hardware discovery is unavailable") + current = discover_hardware() + if check_masks: + expected_nodes = list(placement.get("effective_nodes", [])) + expected_cpus = list(placement.get("effective_cpus", [])) + if ( + list(current.get("effective_nodes", [])) != expected_nodes + or list(current.get("effective_cpus", [])) != expected_cpus + ): + raise RamdiskError( + "effective CPU/NUMA mask changed since preparation; " + "destroy and review a fresh plan" + ) + if check_gpus: + observed = { + int(device["index"]): device + for device in current.get("gpus", []) + if isinstance(device, dict) + and isinstance(device.get("index"), int) + and not isinstance(device.get("index"), bool) + } + effective_nodes = set(current.get("effective_nodes") or []) + for expected in accelerator.get("devices") or []: + index = expected.get("index") + device = observed.get(index) + if ( + device is None + or not _same_gpu_identity(expected, device) + or device.get("numa_node") != expected.get("numa_node") + or device.get("numa_node") not in effective_nodes + ): + raise RamdiskError( + "managed GPU/NUMA identity changed since preparation; " + "destroy and review a fresh plan" + ) + + +def _group_alive(pgid): + return get_platform_ops().process_group_alive(pgid) + + +_managed_children_lock = threading.Lock() +_managed_children = {} + + +def _track_managed_child(process): + """Retain Popen handles so a long-lived TUI can reap engine zombies.""" + with _managed_children_lock: + _managed_children[int(process.pid)] = process + + +def _poll_managed_child(pid): + with _managed_children_lock: + process = _managed_children.get(int(pid)) + if process is None: + return None + try: + returncode = process.poll() + except (ChildProcessError, OSError): + returncode = getattr(process, "returncode", None) + if returncode is not None: + with _managed_children_lock: + if _managed_children.get(int(pid)) is process: + _managed_children.pop(int(pid), None) + return returncode + + +def _managed_child_liveness(pid): + """Return True/False for a retained child, or None without a handle.""" + with _managed_children_lock: + process = _managed_children.get(int(pid)) + if process is None: + return None + try: + returncode = process.poll() + except (ChildProcessError, OSError): + returncode = getattr(process, "returncode", None) + if returncode is None: + return True + with _managed_children_lock: + if _managed_children.get(int(pid)) is process: + _managed_children.pop(int(pid), None) + return False + + +def _forget_managed_child(pid): + with _managed_children_lock: + _managed_children.pop(int(pid), None) + + +def _terminate_direct_child( + process, + term_seconds=10.0, + kill_seconds=3.0, +): + """Terminate an unrecorded child by PID, never an unverified PGID.""" + if process.poll() is not None: + return None + try: + process.terminate() + except ProcessLookupError: + return None + try: + process.wait(timeout=term_seconds) + return None + except ChildProcessError: + return None + except subprocess.TimeoutExpired: + pass + try: + process.kill() + except ProcessLookupError: + return None + try: + process.wait(timeout=kill_seconds) + return None + except ChildProcessError: + return None + except subprocess.TimeoutExpired: + return "direct child PID %s survived SIGKILL" % process.pid + + +def _terminate_verified_group( + record, + term_seconds=10.0, + kill_seconds=3.0, + *, + managed_child_liveness=None, + process_matches=None, + ops=None, +): + """Terminate persisted members only through freshly verified pidfds.""" + managed_child_liveness = ( + _managed_child_liveness + if managed_child_liveness is None + else managed_child_liveness + ) + # ``process_matches`` remains an accepted injection for facade/API + # compatibility. Lifecycle preflight uses it before this function, but it + # cannot safely authorize a later signal because PID/PGID reuse may occur + # between those two operations. + del process_matches + ops = get_platform_ops() if ops is None else ops + expected_pgid = int(record.get("pgid", record["pid"])) + + def signal_round(signum, stage): + result = ops.signal_verified_process_group(record, signum) + if not isinstance(result, dict): + return ( + "failed", + "PID/PGID %s verified cleanup returned an invalid result %s" + % (expected_pgid, stage), + ) + status = result.get("status") + reason = result.get("reason", "unspecified") + if status == "absent": + # Polling also reaps a retained local group leader. A live Popen + # handle contradicts procfs absence and must retain authority. + if managed_child_liveness(record["pid"]) is True: + return ( + "failed", + "PID/PGID %s retained managed child is still live %s; " + "refusing to accept process-group absence" + % (expected_pgid, stage), + ) + return "stopped", None + if status == "foreign": + return ( + "failed", + "PID/PGID %s identity changed %s (%s); refusing any " + "further signal" + % (expected_pgid, stage, reason), + ) + if status == "inconclusive": + return "inconclusive", reason + if status == "signaled": + return "running", None + return ( + "failed", + "PID/PGID %s verified cleanup returned unknown status %r %s" + % (expected_pgid, status, stage), + ) + + def signal_window(signum, duration, interval, label): + deadline = time.monotonic() + max(0.0, float(duration)) + last_inconclusive = None + first = True + while first or time.monotonic() < deadline: + first = False + state, detail = signal_round(signum, "during %s" % label) + if state == "stopped": + return "stopped", None + if state == "failed": + return "failed", detail + if state == "inconclusive": + last_inconclusive = detail + else: + last_inconclusive = None + if time.monotonic() >= deadline: + break + time.sleep(interval) + return "deadline", last_inconclusive + + state, failure = signal_window( + signal.SIGTERM, + term_seconds, + 0.1, + "SIGTERM grace period", + ) + if state == "stopped": + return None + if state == "failed": + return failure + + state, failure = signal_window( + signal.SIGKILL, + kill_seconds, + 0.05, + "SIGKILL grace period", + ) + if state == "stopped": + return None + if state == "failed": + return failure + if failure: + return ( + "process group %s remained unverified after SIGKILL (%s)" + % (expected_pgid, failure) + ) + return "process group %s survived SIGKILL" % expected_pgid + + +def _wait_managed_ready( + record, + timeout, + api_key=None, + cancel_event=None, + *, + process_matches=None, + urlopen=None, +): + process_matches = ( + _process_matches + if process_matches is None + else process_matches + ) + urlopen = urllib.request.urlopen if urlopen is None else urlopen + deadline = time.monotonic() + timeout + headers = ( + {"Authorization": "Bearer " + api_key} + if api_key + else {} + ) + last_error = "listener not ready" + while time.monotonic() < deadline: + _raise_if_cancelled(cancel_event) + matches, reason, _ = process_matches(record) + if not matches: + raise RamdiskError( + "managed engine PID %s exited before readiness (%s); see %s" + % (record["pid"], reason, record["log"]) + ) + try: + request = urllib.request.Request( + "http://127.0.0.1:%d/health" % record["port"], + headers=headers, + ) + with urlopen(request, timeout=2) as response: + payload = json.loads( + response.read().decode("utf-8") + ) + if payload.get("status") == "ok": + record["ready_at"] = _utc_now() + return + last_error = "health response was not ready" + except (OSError, ValueError, urllib.error.URLError) as exc: + last_error = str(exc) + if cancel_event is None: + time.sleep(0.5) + elif cancel_event.wait(0.5): + _raise_if_cancelled(cancel_event) + raise RamdiskError( + "managed engine on port %s did not become ready within %.0fs " + "(%s); see %s" + % (record["port"], timeout, last_error, record["log"]) + ) + + +def _resolve_engine_path(cli_path, engine_path=None): + candidates = [] + if engine_path: + candidates.append(engine_path) + here = os.path.dirname(os.path.abspath(cli_path)) + suffix = ".exe" if os.name == "nt" else "" + candidates.extend( + [ + os.path.join(here, "colibri" + suffix), + os.path.join( + os.path.dirname(here), + "libexec", + "colibri", + "colibri" + suffix, + ), + os.path.join(here, "glm" + suffix), + os.path.join( + os.path.dirname(here), + "libexec", + "colibri", + "glm" + suffix, + ), + ] + ) + for candidate in candidates: + if ( + candidate + and os.path.isfile(candidate) + and os.access(candidate, os.X_OK) + ): + return os.path.realpath(candidate) + raise RamdiskError( + "cannot locate the executable Colibri engine for " + "persistent benchmarking" + ) + + +def _managed_process_metrics( + record, + *, + process_matches=None, + process_group_members=None, + process_status=None, +): + process_matches = ( + _process_matches + if process_matches is None + else process_matches + ) + process_group_members = ( + _process_group_members + if process_group_members is None + else process_group_members + ) + if process_status is None: + process_status = get_platform_ops().process_status + rss_bytes = None + rss_processes = 0 + matches, _, _ = process_matches(record) + if matches: + expected_pgid = int(record.get("pgid", record["pid"])) + members, unreadable = process_group_members( + expected_pgid + ) + inert_members = [ + member + for member in members + if isinstance(member, dict) + and member.get("inert") is True + ] + live_members = [ + member + for member in members + if not isinstance(member, dict) + or member.get("inert") is not True + ] + verified_live = [ + member + for member in live_members + if _live_group_member_matches(record, member, expected_pgid) + ] + if ( + not unreadable + and all( + _inert_group_member_matches( + record, + member, + expected_pgid, + ) + for member in inert_members + ) + and len(verified_live) == len(live_members) + and verified_live + ): + rss_bytes = 0 + for member in verified_live: + for line in process_status( + member["pid"] + ).splitlines(): + if line.startswith("VmRSS:"): + try: + rss_bytes += int(line.split()[1]) * 1024 + rss_processes += 1 + except (ValueError, IndexError): + pass + break + tail = "" + log_path = record.get("log") + if log_path: + try: + with open(log_path, "rb") as stream: + stream.seek(0, os.SEEK_END) + stream.seek(max(0, stream.tell() - 65536)) + tail = stream.read().decode("utf-8", "replace") + except OSError: + pass + ram_experts = None + ram_bytes = None + ssd_bytes = None + matches = re.findall( + r"RAM map:\s*(\d+) experts / ([0-9.]+) GB", + tail, + ) + if matches: + ram_experts = int(matches[-1][0]) + ram_bytes = float(matches[-1][1]) * 1e9 + else: + matches = re.findall( + r"\[RAMMAP\]\s*(\d+) direct tmpfs experts,\s*" + r"([0-9.]+) GB mapped", + tail, + ) + if matches: + ram_experts = int(matches[-1][0]) + ram_bytes = float(matches[-1][1]) * 1e9 + matches = re.findall( + r"physical SSD reads:\s*([0-9.]+) GB", + tail, + re.I, + ) + if matches: + ssd_bytes = float(matches[-1]) * 1e9 + return { + "rss_bytes": rss_bytes, + "rss_processes": rss_processes, + "rammap_experts": ram_experts, + "rammap_bytes": ram_bytes, + "latest_ssd_bytes": ssd_bytes, + } diff --git a/c/ramdisk_support/state.py b/c/ramdisk_support/state.py new file mode 100644 index 000000000..bd848e1fc --- /dev/null +++ b/c/ramdisk_support/state.py @@ -0,0 +1,2477 @@ +"""Durable RAM-disk state, manifests, locking, and usage recovery.""" + +from __future__ import print_function + +import contextlib +import datetime +import json +import os +import posixpath +import re +import secrets +import stat +import threading + +from .common import ( + DEFAULT_MOUNT_ROOT, + MANIFEST_VERSION, + PROFILE_LINE_RE, + USAGE_MERGE_RE, + RamdiskError, + _path_is_below, + _path_without_symlinks, + _positive_int, + _validated_usage_header, + _utc_now, + _usage_engine_id, + _usage_engine_name, +) +from .platform_ops import current_uid, get_platform_ops +from .accelerator import _managed_accelerator_contract + + +try: + import fcntl +except ImportError: + fcntl = None + + +_lifecycle_local = threading.local() +_fallback_usage_lock = threading.RLock() +_NATIVE_DIRFD_PRIMITIVES = ( + os.name == "posix" + and os.open in getattr(os, "supports_dir_fd", set()) + and os.stat in getattr(os, "supports_dir_fd", set()) + and os.unlink in getattr(os, "supports_dir_fd", set()) + and os.rename in getattr(os, "supports_dir_fd", set()) +) + + +def _supports_native_dirfd(): + """Return whether the current runtime can bind every required file step.""" + required_flags = ("O_DIRECTORY", "O_NOFOLLOW", "O_NONBLOCK") + return ( + _NATIVE_DIRFD_PRIMITIVES + and all( + isinstance(getattr(os, name, None), int) + and getattr(os, name) != 0 + for name in required_flags + ) + ) + + +def _valid_utc_timestamp(value): + if not isinstance(value, str) or not value: + return False + candidate = value[:-1] + "+00:00" if value.endswith("Z") else value + try: + parsed = datetime.datetime.fromisoformat(candidate) + except ValueError: + return False + return ( + parsed.tzinfo is not None + and parsed.utcoffset() == datetime.timedelta(0) + ) + + +def _valid_usage_snapshot(value): + if not isinstance(value, dict): + return False + for key, count in value.items(): + if ( + not isinstance(key, str) + or not isinstance(count, int) + or isinstance(count, bool) + ): + return False + if re.fullmatch(r"-[12]:[1-9]\d*", key): + if count <= 0: + return False + elif re.fullmatch(r"\d+:\d+", key): + if count < 0: + return False + else: + return False + return True + + +def _state_root(): + base = os.environ.get("XDG_STATE_HOME") + if not base: + base = os.path.join(os.path.expanduser("~"), ".local", "state") + base = os.path.expanduser(base) + if not os.path.isabs(base): + raise RamdiskError("XDG_STATE_HOME must be an absolute durable path") + return os.path.normpath(os.path.join(base, "colibri", "ramdisk")) + + +def _manifest_path(): + override = os.environ.get("COLI_RAMDISK_MANIFEST") + if override: + override = os.path.expanduser(override) + if not os.path.isabs(override): + raise RamdiskError("COLI_RAMDISK_MANIFEST must be an absolute durable path") + return os.path.normpath(override) + return os.path.join(_state_root(), "manifest.json") + + +def _benchmarks_path(): + return os.path.join(_state_root(), "benchmarks.json") + + +def _ensure_private_dir(path): + path = os.path.normpath(path) + if not _path_without_symlinks(path): + raise RamdiskError("private state path contains a symlink: %s" % path) + os.makedirs(path, mode=0o700, exist_ok=True) + if not _path_without_symlinks(path): + raise RamdiskError("private state path changed through a symlink: %s" % path) + info = os.lstat(path) + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RamdiskError("private state path is not a real directory: %s" % path) + os.chmod(path, 0o700) + + +def _assert_durable_state_dir( + path, + plan=None, + *, + filesystem_for_path=None, +): + """Revalidate a derived engine/benchmark state directory before use.""" + path = os.path.normpath(path) + if not _path_without_symlinks(path): + raise RamdiskError("managed state path contains a symlink: %s" % path) + if filesystem_for_path is None: + raise RamdiskError("managed state filesystem validation is unavailable") + if filesystem_for_path(path) in ("tmpfs", "ramfs"): + raise RamdiskError("managed state path is on a volatile filesystem: %s" % path) + if plan is not None: + for mount in plan.get("mounts", []): + weight_path = mount.get("path") + if isinstance(weight_path, str) and _path_is_below( + os.path.realpath(path), + os.path.realpath(weight_path), + allow_equal=True, + ): + raise RamdiskError( + "managed state path overlaps volatile weights: %s" % path + ) + return path + + +def _ensure_atomic_parent(path): + """Create a missing atomic-write parent without mutating an existing one.""" + if os.path.lexists(path): + info = os.lstat(path) + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RamdiskError( + "atomic-state parent is not a real directory: %s" % path + ) + return + try: + os.makedirs(path, mode=0o700) + except FileExistsError: + info = os.lstat(path) + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RamdiskError( + "atomic-state parent is not a real directory: %s" % path + ) + return + os.chmod(path, 0o700) + + +@contextlib.contextmanager +def _lifecycle_lock(): + """Serialize all manifest-changing operations for this invoking user.""" + depth = getattr(_lifecycle_local, "depth", 0) + if depth: + _lifecycle_local.depth = depth + 1 + try: + yield + finally: + _lifecycle_local.depth -= 1 + return + if fcntl is None or not get_platform_ops().is_linux: + raise RamdiskError("RAM-disk lifecycle locking is supported only on Linux") + root = _state_root() + _ensure_private_dir(root) + lock_path = os.path.join(root, "lifecycle.lock") + with open(lock_path, "a+", encoding="utf-8") as lock: + try: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise RamdiskError("another `coli ramdisk` lifecycle operation is active") + _lifecycle_local.depth = 1 + try: + yield + finally: + _lifecycle_local.depth = 0 + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + +@contextlib.contextmanager +def _close_preserving_primary(close): + try: + yield + except BaseException: + try: + close() + except BaseException: + pass + raise + else: + close() + + +@contextlib.contextmanager +def _fdopen_preserving_primary(descriptor, *args, **kwargs): + try: + stream = os.fdopen(descriptor, *args, **kwargs) + except BaseException: + try: + os.close(descriptor) + except BaseException: + pass + raise + with _close_preserving_primary(stream.close): + yield stream + + +def _stat_identity(info): + return (info.st_dev, info.st_ino) + + +def _real_directory_info(path, source): + try: + info = os.lstat(path) + except OSError as exc: + raise RamdiskError("%s directory is unavailable: %s" % (source, exc)) from exc + if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RamdiskError("%s parent is not a real directory: %s" % (source, path)) + return info + + +def _revalidate_bound_parent(bound): + final = _real_directory_info(bound["parent"], bound["source"]) + if _stat_identity(final) != bound["identity"]: + raise RamdiskError( + "%s parent identity changed during access" % bound["source"] + ) + validator = bound.get("validator") + if validator is not None: + validator() + final = _real_directory_info(bound["parent"], bound["source"]) + if _stat_identity(final) != bound["identity"]: + raise RamdiskError( + "%s parent identity changed during validation" % bound["source"] + ) + + +@contextlib.contextmanager +def _bound_parent_descriptor( + parent, + *, + source, + validator=None, + expected_identity=None, + require_native=False, +): + parent = os.path.normpath(parent) + before = _real_directory_info(parent, source) + before_identity = _stat_identity(before) + if expected_identity is not None and before_identity != expected_identity: + raise RamdiskError("%s parent identity changed before open" % source) + if validator is not None: + validator() + if not _supports_native_dirfd(): + if require_native: + raise RamdiskError( + "%s requires descriptor-relative filesystem operations" % source + ) + bound = { + "descriptor": None, + "identity": before_identity, + "parent": parent, + "native": False, + "source": source, + "validator": validator, + } + try: + yield bound + except BaseException: + try: + _revalidate_bound_parent(bound) + except BaseException: + pass + raise + else: + _revalidate_bound_parent(bound) + return + + flags = ( + os.O_RDONLY + | int(getattr(os, "O_DIRECTORY")) + | int(getattr(os, "O_NOFOLLOW")) + | getattr(os, "O_CLOEXEC", 0) + ) + try: + descriptor = os.open(parent, flags) + except OSError as exc: + raise RamdiskError("cannot open %s parent safely: %s" % (source, exc)) from exc + with _close_preserving_primary(lambda: os.close(descriptor)): + opened = os.fstat(descriptor) + after_open = _real_directory_info(parent, source) + if ( + not stat.S_ISDIR(opened.st_mode) + or _stat_identity(opened) != before_identity + or _stat_identity(after_open) != before_identity + ): + raise RamdiskError("%s parent identity changed during verified open" % source) + if validator is not None: + validator() + after_validation = _real_directory_info(parent, source) + if _stat_identity(after_validation) != before_identity: + raise RamdiskError("%s parent identity changed during validation" % source) + bound = { + "descriptor": descriptor, + "identity": before_identity, + "parent": parent, + "native": True, + "source": source, + "validator": validator, + } + try: + yield bound + except BaseException: + try: + _revalidate_bound_parent(bound) + except BaseException: + pass + raise + else: + _revalidate_bound_parent(bound) + + +def _target_info(bound, name): + try: + if bound["native"]: + return os.stat( + name, + dir_fd=bound["descriptor"], + follow_symlinks=False, + ) + return os.lstat(os.path.join(bound["parent"], name)) + except FileNotFoundError: + return None + + +def _require_regular_target(info, path, source, *, allow_missing): + if info is None: + if allow_missing: + return + raise RamdiskError( + "%s must be an existing regular non-symlink file: %s" + % (source, path) + ) + if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode): + raise RamdiskError( + "%s must be a regular non-symlink file: %s" % (source, path) + ) + + +def _read_regular_text_from_bound_impl( + bound, + name, + *, + source, + allow_missing, + consumer=None, +): + path = os.path.join(bound["parent"], name) + before = _target_info(bound, name) + _require_regular_target(before, path, source, allow_missing=allow_missing) + flags = ( + os.O_RDONLY + | getattr(os, "O_CLOEXEC", 0) + | int(getattr(os, "O_NONBLOCK", 0) or 0) + ) + if isinstance(getattr(os, "O_NOFOLLOW", None), int): + flags |= int(getattr(os, "O_NOFOLLOW")) + try: + if bound["native"]: + descriptor = os.open( + name, + flags, + dir_fd=bound["descriptor"], + ) + else: + descriptor = os.open(path, flags) + except FileNotFoundError: + if allow_missing and before is None: + return { + "text": "", + "exists": False, + "parent_identity": bound["identity"], + "target_identity": None, + } + raise RamdiskError("%s changed before open: %s" % (source, path)) + except OSError: + # The public reader wrapper normalizes ordinary I/O failures while + # preserving RamdiskError and BaseException control flow. + raise + + with _close_preserving_primary(lambda: os.close(descriptor)): + opened = os.fstat(descriptor) + _require_regular_target(opened, path, source, allow_missing=False) + opened_identity = _stat_identity(opened) + after_open = _target_info(bound, name) + if ( + before is None + or after_open is None + or _stat_identity(before) != opened_identity + or _stat_identity(after_open) != opened_identity + ): + raise RamdiskError("%s changed during verified open: %s" % (source, path)) + with _fdopen_preserving_primary( + descriptor, + "r", + encoding="utf-8", + errors="strict", + closefd=False, + ) as stream: + try: + text = stream.read() + consumed = consumer(text) if consumer is not None else None + except RamdiskError: + raise + except (OSError, UnicodeError) as exc: + raise RamdiskError( + "cannot read %s: %s" % (source, exc) + ) from exc + final = _target_info(bound, name) + if final is None or _stat_identity(final) != opened_identity: + raise RamdiskError("%s changed during read: %s" % (source, path)) + snapshot = { + "text": text, + "exists": True, + "parent_identity": bound["identity"], + "target_identity": opened_identity, + } + if consumer is not None: + snapshot["value"] = consumed + return snapshot + + +def _read_regular_text_from_bound( + bound, + name, + *, + source, + allow_missing, + consumer=None, +): + try: + return _read_regular_text_from_bound_impl( + bound, + name, + source=source, + allow_missing=allow_missing, + consumer=consumer, + ) + except RamdiskError: + raise + except (OSError, UnicodeError) as exc: + raise RamdiskError("cannot read %s: %s" % (source, exc)) from exc + + +def _read_bound_regular_text( + path, + *, + source, + allow_missing, + validator=None, + expected_parent_identity=None, + require_native=False, + consumer=None, +): + parent = os.path.dirname(path) + name = os.path.basename(path) + if not name or os.path.join(parent, name) != path: + raise RamdiskError("%s path is not normalized: %s" % (source, path)) + with _bound_parent_descriptor( + parent, + source=source, + validator=validator, + expected_identity=expected_parent_identity, + require_native=require_native, + ) as bound: + return _read_regular_text_from_bound( + bound, + name, + source=source, + allow_missing=allow_missing, + consumer=consumer, + ) + + +def _open_bound_temporary(bound, prefix, mode): + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0) + if bound["native"]: + flags |= int(getattr(os, "O_NOFOLLOW")) + for _ in range(128): + name = prefix + secrets.token_hex(16) + try: + if bound["native"]: + descriptor = os.open( + name, + flags, + mode, + dir_fd=bound["descriptor"], + ) + else: + descriptor = os.open( + os.path.join(bound["parent"], name), + flags, + mode, + ) + except FileExistsError: + continue + return descriptor, name + raise RamdiskError("could not allocate an atomic temporary file") + + +def _fsync_bound_directory(descriptor): + os.fsync(descriptor) + + +def _atomic_replace_stream_from_bound( + bound, + name, + *, + source, + prefix, + mode, + writer, + expected_snapshot=None, +): + path = os.path.join(bound["parent"], name) + if ( + not name + or os.path.basename(name) != name + or os.path.normpath(path) != path + ): + raise RamdiskError("%s path is not normalized: %s" % (source, path)) + if isinstance(expected_snapshot, dict) and ( + expected_snapshot.get("parent_identity") != bound["identity"] + ): + raise RamdiskError("%s parent identity changed before replacement" % source) + + _revalidate_bound_parent(bound) + initial = _target_info(bound, name) + _require_regular_target(initial, path, source, allow_missing=True) + initial_identity = _stat_identity(initial) if initial is not None else None + if isinstance(expected_snapshot, dict) and ( + initial_identity != expected_snapshot.get("target_identity") + ): + raise RamdiskError("%s changed before atomic replacement" % source) + + descriptor, tmp_name = _open_bound_temporary(bound, prefix, mode) + tmp_path = os.path.join(bound["parent"], tmp_name) + try: + opened_temp = os.fstat(descriptor) + _require_regular_target( + opened_temp, + tmp_path, + "atomic temporary file", + allow_missing=False, + ) + temp_identity = _stat_identity(opened_temp) + visible_temp = _target_info(bound, tmp_name) + if ( + visible_temp is None + or _stat_identity(visible_temp) != temp_identity + ): + raise RamdiskError("atomic temporary file escaped its bound parent") + except BaseException: + try: + os.close(descriptor) + except BaseException: + pass + try: + if bound["native"]: + os.unlink(tmp_name, dir_fd=bound["descriptor"]) + else: + os.unlink(tmp_path) + except BaseException: + pass + raise + + replaced = False + try: + with _fdopen_preserving_primary( + descriptor, + "w", + encoding="utf-8", + ) as stream: + writer(stream) + stream.flush() + if hasattr(os, "fchmod"): + os.fchmod(stream.fileno(), mode) + else: + os.chmod(tmp_path, mode) + os.fsync(stream.fileno()) + + _revalidate_bound_parent(bound) + visible_temp = _target_info(bound, tmp_name) + if ( + visible_temp is None + or _stat_identity(visible_temp) != temp_identity + ): + raise RamdiskError("atomic temporary file changed before replacement") + current = _target_info(bound, name) + _require_regular_target(current, path, source, allow_missing=True) + current_identity = _stat_identity(current) if current is not None else None + if current_identity != initial_identity: + raise RamdiskError("%s changed before atomic replacement" % source) + + if bound["native"]: + os.replace( + tmp_name, + name, + src_dir_fd=bound["descriptor"], + dst_dir_fd=bound["descriptor"], + ) + else: + os.replace(tmp_path, path) + replaced = True + + committed = _target_info(bound, name) + _require_regular_target(committed, path, source, allow_missing=False) + if _stat_identity(committed) != temp_identity: + raise RamdiskError("%s changed during atomic replacement" % source) + _revalidate_bound_parent(bound) + if bound["native"]: + _fsync_bound_directory(bound["descriptor"]) + else: + _fsync_directory(bound["parent"]) + return { + "text": None, + "exists": True, + "parent_identity": bound["identity"], + "target_identity": temp_identity, + } + except BaseException: + if not replaced: + try: + if bound["native"]: + os.unlink(tmp_name, dir_fd=bound["descriptor"]) + else: + os.unlink(tmp_path) + except BaseException: + pass + raise + + +def _atomic_replace_stream( + path, + *, + prefix, + mode, + writer, + expected_snapshot=None, + validator=None, + require_native=False, + source=None, +): + parent = os.path.dirname(path) + name = os.path.basename(path) + source = source or "atomic target %s" % path + expected_parent = ( + expected_snapshot.get("parent_identity") + if isinstance(expected_snapshot, dict) + else None + ) + with _bound_parent_descriptor( + parent, + source=source, + validator=validator, + expected_identity=expected_parent, + require_native=require_native, + ) as bound: + return _atomic_replace_stream_from_bound( + bound, + name, + source=source, + prefix=prefix, + mode=mode, + writer=writer, + expected_snapshot=expected_snapshot, + ) + + +def _atomic_json_from_bound( + bound, + name, + value, + *, + mode=0o600, + source="atomic JSON target", + expected_snapshot=None, +): + def write(stream): + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + + return _atomic_replace_stream_from_bound( + bound, + name, + source=source, + prefix=".tmp-", + mode=mode, + writer=write, + expected_snapshot=expected_snapshot, + ) + + +def _atomic_json( + path, + value, + mode=0o600, + *, + expected_snapshot=None, + validator=None, + require_native=False, +): + parent = os.path.dirname(path) + _ensure_atomic_parent(parent) + + def write(stream): + json.dump(value, stream, indent=2, sort_keys=True) + stream.write("\n") + + _atomic_replace_stream( + path, + prefix=".tmp-", + mode=mode, + writer=write, + expected_snapshot=expected_snapshot, + validator=validator, + require_native=require_native, + ) + + +def _read_json(path, required=False): + try: + with open(path, "r", encoding="utf-8") as stream: + return json.load(stream) + except FileNotFoundError: + if required: + raise RamdiskError("RAM-disk manifest not found; run `coli ramdisk prepare`") + return None + except (OSError, ValueError) as exc: + raise RamdiskError("cannot read %s: %s" % (path, exc)) + + +def _manifest_mount_layout(plan): + """Validate and return the exact v1 mount layout encoded by a plan.""" + topology = plan.get("topology") + root = plan.get("mount_root") + planned = plan.get("mounts") + model = plan.get("model", {}).get("path") + if topology not in ("interleaved", "per-node") or not isinstance(root, str): + raise RamdiskError("RAM-disk manifest has an invalid topology") + root = posixpath.normpath(root) + if ( + not posixpath.isabs(root) + or posixpath.commonpath([root, "/mnt"]) != "/mnt" + or root == "/mnt" + or not _path_without_symlinks(root) + or root in ("/mnt", DEFAULT_MOUNT_ROOT + "/..") + ): + raise RamdiskError("RAM-disk manifest has an unsafe mount root") + try: + normalized_model = str(model).replace("\\", "/") + if re.match(r"^[A-Za-z]:/", normalized_model): + common = None + else: + normalized_model = posixpath.normpath(normalized_model) + common = posixpath.commonpath([root, normalized_model]) + if common in (root, normalized_model): + raise RamdiskError( + "RAM-disk manifest mount root overlaps its canonical model" + ) + except (TypeError, ValueError): + raise RamdiskError( + "RAM-disk manifest has incompatible model and mount paths" + ) + if not isinstance(planned, list) or not planned: + raise RamdiskError("RAM-disk manifest has no planned mounts") + planned_nodes = plan.get("placement", {}).get( + "memory_nodes", + plan.get("hardware", {}).get("online_nodes"), + ) + if topology == "interleaved": + expected = [(None, root)] + else: + if ( + not isinstance(planned_nodes, list) + or not planned_nodes + or any( + not isinstance(node, int) + or isinstance(node, bool) + or node < 0 + for node in planned_nodes + ) + or len(set(planned_nodes)) != len(planned_nodes) + ): + raise RamdiskError("RAM-disk manifest has an invalid NUMA node set") + expected = [ + (node, posixpath.join(root, "node%d" % node)) + for node in planned_nodes + ] + observed = [] + for record in planned: + if not isinstance(record, dict): + raise RamdiskError( + "RAM-disk manifest has invalid planned mount paths" + ) + path = record.get("path") + node = record.get("node") + if ( + not isinstance(path, str) + or not posixpath.isabs(path) + or posixpath.normpath(path) != path + or not _path_without_symlinks(path) + ): + raise RamdiskError( + "RAM-disk manifest has invalid planned mount paths" + ) + observed.append((node, path)) + if observed != expected: + raise RamdiskError( + "RAM-disk manifest mounts do not match its topology" + ) + return root, expected + + +def _load_manifest( + required=False, + *, + filesystem_for_path=None, + read_json=None, + manifest_path=None, + state_root=None, + benchmarks_path=None, + assert_durable_state_dir=None, + uid_provider=None, +): + """Read and minimally validate lifecycle state before it can drive actions.""" + read_json = _read_json if read_json is None else read_json + manifest_path = _manifest_path if manifest_path is None else manifest_path + state_root = _state_root if state_root is None else state_root + benchmarks_path = _benchmarks_path if benchmarks_path is None else benchmarks_path + uid_provider = current_uid if uid_provider is None else uid_provider + if assert_durable_state_dir is None: + def assert_durable_state_dir(path, plan=None): + return _assert_durable_state_dir( + path, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) + + manifest = read_json(manifest_path(), required=required) + if manifest is None: + return None + if ( + not isinstance(manifest, dict) + or manifest.get("version") != MANIFEST_VERSION + ): + raise RamdiskError("unsupported or malformed RAM-disk manifest") + base_port = manifest.get("base_port") + if base_port is not None and ( + isinstance(base_port, bool) + or not isinstance(base_port, int) + or not 1 <= base_port <= 65535 + ): + raise RamdiskError("RAM-disk manifest has an invalid managed base port") + deployment_id = manifest.get("deployment_id") + if deployment_id is not None and ( + not isinstance(deployment_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", deployment_id) + ): + raise RamdiskError("RAM-disk manifest has an invalid deployment identity") + plan = manifest.get("plan") + mounts = manifest.get("mounts") + processes = manifest.get("processes", []) + if ( + not isinstance(plan, dict) + or not isinstance(mounts, list) + or not isinstance(processes, list) + ): + raise RamdiskError("RAM-disk manifest is missing lifecycle records") + if ( + not isinstance(plan.get("model"), dict) + or not isinstance(plan.get("mounts"), list) + ): + raise RamdiskError("RAM-disk manifest has an invalid plan") + fingerprint = manifest.get("model_fingerprint") + if ( + not isinstance(fingerprint, str) + or not re.fullmatch(r"sha256:[0-9a-f]{64}", fingerprint) + or plan["model"].get("fingerprint") != fingerprint + or not isinstance(plan["model"].get("path"), str) + or not os.path.isabs(plan["model"]["path"]) + or not plan["mounts"] + ): + raise RamdiskError("RAM-disk manifest has an invalid model identity") + _managed_accelerator_contract(plan) + mount_root, _ = _manifest_mount_layout(plan) + planned_paths = {record["path"] for record in plan["mounts"]} + if len(planned_paths) != len(plan["mounts"]): + raise RamdiskError("RAM-disk manifest has invalid planned mount paths") + mount_by_path = {record["path"]: record for record in plan["mounts"]} + mount_ownership = {} + for record in mounts: + if ( + not isinstance(record, dict) + or not isinstance(record.get("path"), str) + or not os.path.isabs(record["path"]) + ): + raise RamdiskError( + "RAM-disk manifest contains an unsafe mount record" + ) + if record["path"] not in planned_paths: + raise RamdiskError( + "RAM-disk manifest mount does not belong to its plan" + ) + if record.get("node") != mount_by_path[record["path"]].get("node"): + raise RamdiskError( + "RAM-disk manifest mount has the wrong NUMA node" + ) + identity = record.get("identity") + identity_is_exact = ( + isinstance(identity, dict) + and _positive_int(identity.get("mount_id")) + and isinstance(identity.get("device"), str) + and bool(identity["device"]) + ) + ownership = record.get("ownership") + if ownership is None and identity_is_exact: + # Manifests written before ownership transitions were explicit + # contain only exact identities and are managed by definition. + ownership = "managed" + if ownership not in ("pending", "identified", "managed"): + raise RamdiskError( + "RAM-disk manifest mount has an invalid ownership state" + ) + if ownership == "pending" and identity is not None: + raise RamdiskError( + "RAM-disk pending mount unexpectedly has an identity" + ) + if ownership != "pending" and not identity_is_exact: + raise RamdiskError( + "RAM-disk manifest mount is missing its identity" + ) + if record["path"] in mount_ownership: + raise RamdiskError( + "RAM-disk manifest contains duplicate mount records" + ) + mount_ownership[record["path"]] = ownership + state = manifest.get("state") + if state not in ( + "preparing", + "ready", + "starting", + "running", + "stopped", + "error", + ): + raise RamdiskError("RAM-disk manifest has an invalid lifecycle state") + recovery = manifest.get("recovery") + if recovery is not None and not isinstance(recovery, dict): + raise RamdiskError("RAM-disk manifest has invalid recovery metadata") + retained_processes = ( + recovery.get("retained_processes", []) + if isinstance(recovery, dict) + else [] + ) + if not isinstance(retained_processes, list): + raise RamdiskError( + "RAM-disk manifest has invalid retained process recovery" + ) + if retained_processes and state != "error": + raise RamdiskError( + "RAM-disk retained process recovery requires the error state" + ) + pending_launches = manifest.get("pending_launches", []) + if not isinstance(pending_launches, list): + raise RamdiskError("RAM-disk manifest has invalid pending launches") + if pending_launches and state not in ("starting", "error"): + raise RamdiskError( + "RAM-disk pending launches require starting or error state" + ) + invoking_uid = uid_provider() + pending_operation_ids = set() + pending_nonces = set() + usage_merge_ids = set() + for pending in pending_launches: + operation_id = ( + pending.get("operation_id") if isinstance(pending, dict) else None + ) + nonce = pending.get("nonce") if isinstance(pending, dict) else None + uid = pending.get("uid") if isinstance(pending, dict) else None + port = pending.get("port") if isinstance(pending, dict) else None + node = pending.get("node") if isinstance(pending, dict) else None + state_dir = ( + pending.get("state_dir") if isinstance(pending, dict) else None + ) + weights_dir = ( + pending.get("weights_dir") if isinstance(pending, dict) else None + ) + launch_not_before = ( + pending.get("launch_not_before") + if isinstance(pending, dict) + else None + ) + launcher_cmdline = ( + pending.get("launcher_cmdline") + if isinstance(pending, dict) + else None + ) + launcher_pid = ( + pending.get("launcher_pid") + if isinstance(pending, dict) + else None + ) + launcher_starttime = ( + pending.get("launcher_starttime") + if isinstance(pending, dict) + else None + ) + expected_command = ( + pending.get("expected_command") + if isinstance(pending, dict) + else None + ) + baseline = ( + pending.get("usage_baseline") + if isinstance(pending, dict) + else None + ) + merge_id = ( + pending.get("usage_merge_id") + if isinstance(pending, dict) + else None + ) + observed_group = ( + pending.get("observed_group") + if isinstance(pending, dict) + else None + ) + usage_merged_at = ( + pending.get("usage_merged_at") + if isinstance(pending, dict) + else None + ) + recovery_error = ( + pending.get("recovery_error") + if isinstance(pending, dict) + else None + ) + observed_pgid = ( + observed_group.get("pgid") + if isinstance(observed_group, dict) + else None + ) + observed_uid = ( + observed_group.get("uid") + if isinstance(observed_group, dict) + else None + ) + leader_starttime = ( + observed_group.get("leader_starttime") + if isinstance(observed_group, dict) + else None + ) + planned_mount = next( + ( + record + for record in plan["mounts"] + if record.get("node") == node + ), + None, + ) + if ( + not isinstance(operation_id, str) + or not re.fullmatch(r"start:[0-9a-f]{32}", operation_id) + or not isinstance(nonce, str) + or not re.fullmatch(r"[0-9a-f]{48}", nonce) + or uid != invoking_uid + or not isinstance(port, int) + or isinstance(port, bool) + or not 1 <= port <= 65535 + or node not in {record.get("node") for record in plan["mounts"]} + or not isinstance(state_dir, str) + or not os.path.isabs(state_dir) + or planned_mount is None + or weights_dir != planned_mount.get("path") + or not isinstance(launch_not_before, int) + or isinstance(launch_not_before, bool) + or launch_not_before < 0 + or not _positive_int(launcher_pid) + or not _positive_int(launcher_starttime) + or not isinstance(launcher_cmdline, list) + or not launcher_cmdline + or any( + not isinstance(item, str) or not item + for item in launcher_cmdline + ) + or not isinstance(expected_command, list) + or not expected_command + or any( + not isinstance(item, str) or not item + for item in expected_command + ) + or not _valid_usage_snapshot(baseline) + or not isinstance(merge_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", merge_id) + or ( + usage_merged_at is not None + and not _valid_utc_timestamp(usage_merged_at) + ) + or ( + recovery_error is not None + and ( + not isinstance(recovery_error, str) + or not recovery_error + ) + ) + or ( + observed_group is not None + and ( + not isinstance(observed_group, dict) + or not _positive_int(observed_pgid) + or observed_uid != uid + or ( + leader_starttime is not None + and not _positive_int(leader_starttime) + ) + ) + ) + ): + raise RamdiskError( + "RAM-disk manifest has unsafe pending launch recovery" + ) + if operation_id != "start:" + merge_id: + raise RamdiskError( + "RAM-disk manifest has unsafe pending launch recovery" + ) + if operation_id in pending_operation_ids or nonce in pending_nonces: + raise RamdiskError( + "RAM-disk manifest has duplicate pending launch recovery" + ) + if merge_id in usage_merge_ids: + raise RamdiskError( + "RAM-disk manifest has duplicate usage transaction authority" + ) + pending_operation_ids.add(operation_id) + pending_nonces.add(nonce) + usage_merge_ids.add(merge_id) + for retained in retained_processes: + pid = retained.get("pid") if isinstance(retained, dict) else None + pgid = retained.get("pgid") if isinstance(retained, dict) else None + node = retained.get("node") if isinstance(retained, dict) else None + state_dir = ( + retained.get("state_dir") if isinstance(retained, dict) else None + ) + baseline = ( + retained.get("usage_baseline") + if isinstance(retained, dict) + else None + ) + merge_id = ( + retained.get("usage_merge_id") + if isinstance(retained, dict) + else None + ) + if ( + not _positive_int(pid) + or pgid != pid + or node not in {record.get("node") for record in plan["mounts"]} + or not isinstance(state_dir, str) + or not os.path.isabs(state_dir) + or not _valid_usage_snapshot(baseline) + or not isinstance(merge_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", merge_id) + ): + raise RamdiskError( + "RAM-disk manifest has unsafe retained process recovery" + ) + if merge_id in usage_merge_ids: + raise RamdiskError( + "RAM-disk manifest has duplicate usage transaction authority" + ) + usage_merge_ids.add(merge_id) + recorded_paths = {record["path"] for record in mounts} + non_managed = sorted( + path + for path, ownership in mount_ownership.items() + if ownership != "managed" + ) + if state not in ("preparing", "error") and non_managed: + raise RamdiskError( + "RAM-disk manifest contains pending mount ownership outside " + "preparation/recovery: %s" % ", ".join(non_managed) + ) + if ( + state in ("ready", "starting", "running", "stopped") + and recorded_paths != planned_paths + ): + raise RamdiskError( + "ready RAM-disk manifest does not contain every planned mount" + ) + if state == "running" and len(processes) != len(mounts): + raise RamdiskError( + "running RAM-disk manifest has an incomplete process set" + ) + durable = plan.get("durable_state") + expected_durable = { + "root": state_root(), + "manifest": manifest_path(), + "benchmarks": benchmarks_path(), + } + if durable != expected_durable or any( + not _path_without_symlinks(path) + for path in expected_durable.values() + ): + raise RamdiskError( + "RAM-disk manifest has an invalid durable-state identity" + ) + if filesystem_for_path is None: + raise RamdiskError("durable-state filesystem validation is unavailable") + if any( + filesystem_for_path(path) in ("tmpfs", "ramfs") + for path in expected_durable.values() + ): + raise RamdiskError( + "RAM-disk durable state is on a volatile filesystem" + ) + if any( + _path_is_below(path, mount_root, allow_equal=True) + for path in expected_durable.values() + ): + raise RamdiskError( + "RAM-disk durable state overlaps the volatile mount" + ) + source_shards = plan.get("source_shards") + if not isinstance(source_shards, list) or not source_shards: + raise RamdiskError( + "RAM-disk manifest is missing canonical shard identities" + ) + fingerprint_dir = fingerprint.split(":", 1)[1] + expected_state_root = os.path.join( + state_root(), + "engines", + fingerprint_dir, + ) + recovery_state_dirs = set() + for recovery_record in pending_launches + retained_processes: + node = recovery_record.get("node") + label = "interleaved" if node is None else "node-%d" % node + expected_recovery_state_dir = os.path.join( + expected_state_root, + label, + ) + state_dir = recovery_record["state_dir"] + if ( + state_dir != expected_recovery_state_dir + or state_dir in recovery_state_dirs + ): + raise RamdiskError( + "RAM-disk recovery has an unsafe or duplicate state directory" + ) + assert_durable_state_dir(state_dir, plan=plan) + recovery_state_dirs.add(state_dir) + process_keys = { + "pid": set(), + "pgid": set(), + "port": set(), + "node": set(), + "state_dir": set(recovery_state_dirs), + "weights_dir": set(), + } + for record in processes: + if not isinstance(record, dict): + raise RamdiskError( + "RAM-disk manifest contains an invalid process record" + ) + pid, pgid = record.get("pid"), record.get("pgid") + uid, starttime = record.get("uid"), record.get("starttime") + nonce, port = record.get("nonce"), record.get("port") + node, weights_dir = record.get("node"), record.get("weights_dir") + state_dir, command = record.get("state_dir"), record.get("command") + usage_baseline = record.get("usage_baseline") + usage_merge_id = record.get("usage_merge_id") + usage_merged_at = record.get("usage_merged_at") + mount = next( + ( + item + for item in plan["mounts"] + if item.get("node") == node + ), + None, + ) + label = ( + "interleaved" + if node is None + else "node-%d" % node + if isinstance(node, int) + else "" + ) + expected_state_dir = os.path.join(expected_state_root, label) + valid_command = ( + isinstance(command, list) + and command + and all(isinstance(item, str) and item for item in command) + ) + try: + model_at = command.index("--model") if valid_command else -1 + port_at = command.index("--port") if valid_command else -1 + serves = "serve" in command + command_matches = ( + serves + and command[model_at + 1] == plan["model"]["path"] + and int(command[port_at + 1]) == port + ) + except (ValueError, IndexError, TypeError): + command_matches = False + if ( + not _positive_int(pid) + or not _positive_int(pgid) + or pid != pgid + or uid != invoking_uid + or not _positive_int(starttime) + or not isinstance(nonce, str) + or not re.fullmatch(r"[0-9a-f]{48}", nonce) + or not isinstance(port, int) + or isinstance(port, bool) + or not 1 <= port <= 65535 + or mount is None + or weights_dir != mount["path"] + or state_dir != expected_state_dir + or not command_matches + or not _valid_usage_snapshot(usage_baseline) + or ( + usage_merge_id is not None + and ( + not isinstance(usage_merge_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", usage_merge_id) + ) + ) + or ( + usage_merged_at is not None + and ( + not _valid_utc_timestamp(usage_merged_at) + or usage_merge_id is None + ) + ) + ): + raise RamdiskError( + "RAM-disk manifest contains an unsafe managed process record" + ) + if usage_merge_id is not None: + if usage_merge_id in usage_merge_ids: + raise RamdiskError( + "RAM-disk manifest has duplicate usage transaction authority" + ) + usage_merge_ids.add(usage_merge_id) + assert_durable_state_dir(state_dir, plan=plan) + for key, value in ( + ("pid", pid), + ("pgid", pgid), + ("port", port), + ("node", node), + ("state_dir", state_dir), + ("weights_dir", weights_dir), + ): + if value in process_keys[key]: + raise RamdiskError( + "RAM-disk manifest contains duplicate managed process records" + ) + process_keys[key].add(value) + return manifest + + +def _save_manifest( + manifest, + *, + atomic_json=None, + manifest_path=None, +): + atomic_json = _atomic_json if atomic_json is None else atomic_json + manifest_path = _manifest_path if manifest_path is None else manifest_path + manifest["updated_at"] = _utc_now() + atomic_json(manifest_path(), manifest) + + +def _read_optional_text(path): + try: + with open(path, "r", encoding="utf-8", errors="strict") as stream: + return stream.read() + except FileNotFoundError: + return "" + except (OSError, UnicodeError) as exc: + raise RamdiskError("cannot read usage state %s: %s" % (path, exc)) + + +def _usage_header(counts, source="usage history"): + if not isinstance(counts, dict): + raise RamdiskError("%s counts must contain an object" % source) + records = [] + for key, value in counts.items(): + if not isinstance(key, str): + continue + match = re.fullmatch(r"(-?\d+):(\d+)", key) + if not match: + continue + layer, second = (int(item) for item in match.groups()) + if layer in (-1, -2): + try: + records.append((layer, second, int(value))) + except (TypeError, ValueError): + raise RamdiskError("%s has a malformed usage header" % source) + return _validated_usage_header(records, source=source) + + +def _validate_usage_for_plan(counts, plan, source="usage history"): + """Validate identified usage metadata against the managed model.""" + header = _usage_header(counts, source=source) + if header is None: + return None + try: + model_path = plan["model"]["path"] + with open( + os.path.join(model_path, "config.json"), + "r", + encoding="utf-8", + ) as stream: + config = json.load(stream) + if not isinstance(config, dict): + raise ValueError("config root is not an object") + dimensions = ( + int(config["num_hidden_layers"]), + int(config["n_routed_experts"]), + ) + engine_id = _usage_engine_id( + _usage_engine_name(config.get("model_type")) + ) + except (KeyError, OSError, TypeError, ValueError) as exc: + raise RamdiskError( + "cannot validate %s against the managed model config: %s" + % (source, exc) + ) + return _validated_usage_header( + [ + (-1, header["n_layers"], header["n_experts"]), + ( + -2, + header["format_version"], + header["engine_id"], + ), + ], + source=source, + expected_dimensions=dimensions, + expected_engine_id=engine_id, + ) + + +def _usage_header_counts(header): + if not header: + return {} + return { + "-1:%d" % header["n_layers"]: header["n_experts"], + "-2:%d" % header["format_version"]: header["engine_id"], + } + + +def _usage_data_counts(counts): + result = {} + for key, value in counts.items(): + if not isinstance(key, str): + continue + match = re.fullmatch(r"(\d+):(\d+)", key) + if match: + result[key] = int(value) + return result + + +def _compatible_usage_header(*histories): + reference = None + reference_source = None + for source, counts in histories: + header = _usage_header(counts, source=source) + if header is None: + continue + if reference is None: + reference = header + reference_source = source + continue + if ( + header["n_layers"], + header["n_experts"], + ) != ( + reference["n_layers"], + reference["n_experts"], + ): + raise RamdiskError( + "%s history dimensions do not match %s" + % (source, reference_source) + ) + if header["format_version"] != reference["format_version"]: + raise RamdiskError( + "%s usage format version does not match %s" + % (source, reference_source) + ) + if header["engine_id"] != reference["engine_id"]: + raise RamdiskError( + "%s engine identity does not match %s" + % (source, reference_source) + ) + return reference + + +def _usage_parse(text, source): + counts = {} + header_records = [] + for line in text.splitlines(): + match = PROFILE_LINE_RE.match(line) + if not match: + if re.match(r"^\s*-(?:1|2)(?:\s|$)", line): + raise RamdiskError("%s has a malformed usage header" % source) + continue + layer, expert, count = (int(value) for value in match.groups()) + if layer in (-1, -2): + header_records.append((layer, expert, count)) + elif layer >= 0: + counts["%d:%d" % (layer, expert)] = count + header = _validated_usage_header(header_records, source=source) + counts.update(_usage_header_counts(header)) + return counts + + +def _usage_merge_ids_parse(text): + result = set() + for line in text.splitlines(): + match = USAGE_MERGE_RE.match(line.strip()) + if match: + result.add(match.group(1)) + return result + + +def _usage_snapshot_from_bound( + bound, + name, + path, + *, + source, + allow_missing, +): + def consume(text): + return { + "counts": _usage_parse(text, path), + "merge_ids": _usage_merge_ids_parse(text), + } + + snapshot = _read_regular_text_from_bound( + bound, + name, + source=source, + allow_missing=allow_missing, + consumer=consume, + ) + if snapshot["exists"]: + snapshot.update(snapshot.pop("value")) + else: + snapshot["counts"] = {} + snapshot["merge_ids"] = set() + return snapshot + + +def _usage_snapshot( + path, + *, + source=None, + allow_missing=True, + validator=None, + expected_parent_identity=None, + require_native=False, +): + source = source or "usage state %s" % path + def consume(text): + return { + "counts": _usage_parse(text, path), + "merge_ids": _usage_merge_ids_parse(text), + } + + snapshot = _read_bound_regular_text( + path, + source=source, + allow_missing=allow_missing, + validator=validator, + expected_parent_identity=expected_parent_identity, + require_native=require_native, + consumer=consume, + ) + if snapshot["exists"]: + snapshot.update(snapshot.pop("value")) + else: + snapshot["counts"] = {} + snapshot["merge_ids"] = set() + return snapshot + + +def _usage_read(path): + return _usage_snapshot(path)["counts"] + + +def _managed_usage_read( + path, + plan=None, + *, + filesystem_for_path=None, +): + if os.path.basename(path) != ".coli_usage" or os.path.normpath(path) != path: + raise RamdiskError("managed usage history path is not canonical: %s" % path) + state_dir = os.path.dirname(path) + validator = None + if filesystem_for_path is not None: + validator = lambda: _assert_durable_state_dir( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) + return _usage_snapshot( + path, + source="managed usage history", + allow_missing=False, + validator=validator, + require_native=True, + )["counts"] + + +def _usage_merge_ids(path): + return _usage_snapshot(path)["merge_ids"] + + +def _fsync_directory(path): + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), + ) + try: + os.fsync(descriptor) + except BaseException: + try: + os.close(descriptor) + except BaseException: + # The fsync failure is the authoritative durability error. + pass + raise + else: + # When fsync succeeded, a close failure remains observable. + os.close(descriptor) + + +def _durable_unlink(path): + try: + os.unlink(path) + except FileNotFoundError: + pass + _fsync_directory(os.path.dirname(path)) + + +def _durable_unlink_from_bound( + bound, + name, + *, + source, + expected_snapshot=None, +): + path = os.path.join(bound["parent"], name) + _revalidate_bound_parent(bound) + current = _target_info(bound, name) + current_identity = _stat_identity(current) if current is not None else None + if isinstance(expected_snapshot, dict) and ( + expected_snapshot.get("parent_identity") != bound["identity"] + or expected_snapshot.get("target_identity") != current_identity + ): + raise RamdiskError("%s changed before durable unlink" % source) + if current is not None: + _require_regular_target( + current, + path, + source, + allow_missing=False, + ) + if bound["native"]: + os.unlink(name, dir_fd=bound["descriptor"]) + else: + os.unlink(path) + if bound["native"]: + _fsync_bound_directory(bound["descriptor"]) + else: + _fsync_directory(bound["parent"]) + _revalidate_bound_parent(bound) + + +def _usage_stream_writer(path, counts, merge_id=None, merge_ids=None): + markers = set(merge_ids or ()) + if merge_id: + markers.add(merge_id) + header = _usage_header(counts, source=path) + data_counts = _usage_data_counts(counts) + + def write(stream): + # route_trace.h deliberately keeps an all-zero history zero-byte, + # so emit identifying records only when there are data records. + if header and data_counts: + stream.write( + "-1 %d %d\n" + % (header["n_layers"], header["n_experts"]) + ) + stream.write( + "-2 %d %d\n" + % (header["format_version"], header["engine_id"]) + ) + for key in sorted( + data_counts, + key=lambda item: tuple( + int(value) + for value in item.split(":") + ), + ): + layer, expert = key.split(":") + stream.write( + "%s %s %d\n" % (layer, expert, data_counts[key]) + ) + for marker in sorted(markers): + stream.write("# coli-ramdisk-merge %s\n" % marker) + + return write + + +def _usage_write_from_bound( + bound, + name, + path, + counts, + merge_id=None, + merge_ids=None, + *, + expected_snapshot=None, + source="usage state", +): + return _atomic_replace_stream_from_bound( + bound, + name, + source=source, + prefix=".usage-", + mode=0o600, + writer=_usage_stream_writer(path, counts, merge_id, merge_ids), + expected_snapshot=expected_snapshot, + ) + + +def _usage_write( + path, + counts, + merge_id=None, + merge_ids=None, + *, + expected_snapshot=None, + validator=None, + require_native=False, +): + parent = os.path.dirname(path) + if not os.path.isdir(parent): + raise RamdiskError( + "usage-state parent directory is absent: %s" % parent + ) + + _atomic_replace_stream( + path, + prefix=".usage-", + mode=0o600, + writer=_usage_stream_writer(path, counts, merge_id, merge_ids), + expected_snapshot=expected_snapshot, + validator=validator, + require_native=require_native, + ) + + +def _assert_canonical_usage_target( + canonical_path, + plan=None, + *, + source_still_matches=None, +): + normalized = os.path.normpath(canonical_path) + if normalized != canonical_path or os.path.basename(canonical_path) != ".coli_usage": + raise RamdiskError( + "canonical usage target is not an exact normalized .coli_usage path" + ) + parent = os.path.dirname(normalized) + if not os.path.isdir(parent): + raise RamdiskError( + "canonical model directory is absent; usage delta remains journaled" + ) + if plan is not None: + expected = os.path.normpath( + os.path.join(plan["model"]["path"], ".coli_usage") + ) + if normalized != expected: + raise RamdiskError( + "canonical usage target is not the managed model history" + ) + if os.path.realpath(parent) != os.path.realpath(plan["model"]["path"]): + raise RamdiskError( + "canonical usage target no longer matches the managed model" + ) + if source_still_matches is None: + raise RamdiskError( + "canonical source identity validation is unavailable" + ) + source_still_matches(plan) + + +@contextlib.contextmanager +def _canonical_usage_descriptor( + canonical_path, + plan=None, + *, + source_still_matches=None, +): + validator = lambda: _assert_canonical_usage_target( + canonical_path, + plan=plan, + source_still_matches=source_still_matches, + ) + with _bound_parent_descriptor( + os.path.dirname(canonical_path), + source="canonical usage target", + validator=validator, + require_native=True, + ) as bound: + yield bound + + +def _canonical_usage_snapshot( + canonical_path, + plan=None, + *, + source_still_matches=None, +): + with _canonical_usage_descriptor( + canonical_path, + plan=plan, + source_still_matches=source_still_matches, + ) as bound: + return _usage_snapshot_from_bound( + bound, + ".coli_usage", + canonical_path, + source="canonical usage target", + allow_missing=True, + ) + + +def _canonical_usage_read( + canonical_path, + plan=None, + *, + source_still_matches=None, +): + return _canonical_usage_snapshot( + canonical_path, + plan=plan, + source_still_matches=source_still_matches, + )["counts"] + + +def _managed_state_validator( + state_dir, + plan, + filesystem_for_path, +): + return lambda: _assert_durable_state_dir( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) + + +@contextlib.contextmanager +def _managed_state_descriptor( + state_dir, + plan=None, + *, + filesystem_for_path=None, +): + validator = _managed_state_validator( + state_dir, + plan, + filesystem_for_path, + ) + with _bound_parent_descriptor( + state_dir, + source="managed state", + validator=validator, + require_native=True, + ) as bound: + yield bound + + +def _managed_usage_write( + path, + counts, + plan=None, + *, + filesystem_for_path=None, +): + if os.path.basename(path) != ".coli_usage" or os.path.normpath(path) != path: + raise RamdiskError("managed usage history path is not canonical: %s" % path) + state_dir = os.path.dirname(path) + with _managed_state_descriptor( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) as bound: + snapshot = _usage_snapshot_from_bound( + bound, + ".coli_usage", + path, + source="managed usage history", + allow_missing=True, + ) + return _usage_write_from_bound( + bound, + ".coli_usage", + path, + counts, + expected_snapshot=snapshot, + source="managed usage history", + ) + + +def _journal_snapshot_from_bound(bound, state_dir): + path = os.path.join(state_dir, ".coli_usage.delta.json") + + def consume(text): + try: + payload = json.loads(text) + except (TypeError, ValueError) as exc: + raise RamdiskError("cannot read %s: %s" % (path, exc)) from exc + if not isinstance(payload, dict): + raise RamdiskError("usage delta journal must contain a JSON object") + return payload + + return _read_regular_text_from_bound( + bound, + ".coli_usage.delta.json", + source="usage delta journal", + allow_missing=True, + consumer=consume, + ) + + +def _managed_state_snapshots( + state_dir, + plan=None, + *, + filesystem_for_path=None, + include_usage=False, + include_journal=False, +): + with _managed_state_descriptor( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) as bound: + result = {} + if include_usage: + usage_path = os.path.join(state_dir, ".coli_usage") + result["usage"] = _usage_snapshot_from_bound( + bound, + ".coli_usage", + usage_path, + source="managed usage history", + allow_missing=False, + ) + if include_journal: + result["journal"] = _journal_snapshot_from_bound(bound, state_dir) + return result + + +def _journal_payload(snapshot, path): + if not snapshot["exists"]: + return None + payload = snapshot.get("value") + if not isinstance(payload, dict): + raise RamdiskError("usage delta journal must contain a JSON object") + return payload + + +def _read_usage_journal( + state_dir, + plan=None, + *, + filesystem_for_path=None, +): + snapshots = _managed_state_snapshots( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + include_journal=True, + ) + path = os.path.join(state_dir, ".coli_usage.delta.json") + return _journal_payload(snapshots["journal"], path) + + +def _journal_merge_id(payload): + if not isinstance(payload, dict): + raise RamdiskError("usage delta journal must contain a JSON object") + merge_id = payload.get("id") + if not merge_id: + raise RamdiskError( + "usage delta journal is missing its transaction id" + ) + if ( + not isinstance(merge_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", merge_id) + ): + raise RamdiskError( + "usage delta journal has an invalid transaction id" + ) + return merge_id + + +def _bind_usage_transaction( + record, + plan=None, + *, + filesystem_for_path=None, + reserved_ids=None, +): + reserved_ids = set(reserved_ids or ()) + persisted = record.get("usage_merge_id") + if persisted is not None and ( + not isinstance(persisted, str) + or not re.fullmatch(r"[0-9a-f]{32}", persisted) + ): + raise RamdiskError( + "managed usage recovery has an invalid transaction id" + ) + payload = _read_usage_journal( + record["state_dir"], + plan=plan, + filesystem_for_path=filesystem_for_path, + ) + journal_id = _journal_merge_id(payload) if payload is not None else None + if persisted is not None and journal_id is not None and persisted != journal_id: + raise RamdiskError( + "usage delta journal transaction does not match managed record" + ) + authoritative = persisted or journal_id + if authoritative is not None and authoritative in reserved_ids: + raise RamdiskError( + "duplicate usage transaction authority: %s" % authoritative + ) + if authoritative is None: + for _ in range(128): + candidate = secrets.token_hex(16) + if candidate not in reserved_ids: + authoritative = candidate + break + else: + raise RamdiskError("could not allocate a unique usage transaction id") + merge_id = authoritative + operation_id = record.get("operation_id") + if operation_id is not None and operation_id != "start:" + merge_id: + raise RamdiskError( + "managed usage transaction does not match its operation authority" + ) + record["usage_merge_id"] = merge_id + return merge_id + + +@contextlib.contextmanager +def _usage_lock(lock): + if fcntl is None: + with _fallback_usage_lock: + yield + return + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock.fileno(), fcntl.LOCK_UN) + + +def _validated_usage_delta(payload, expected_merge_id=None): + merge_id = _journal_merge_id(payload) + delta = payload.get("delta", {}) + if not isinstance(delta, dict): + raise RamdiskError("usage delta journal has invalid counts") + if any( + not isinstance(key, str) + or re.fullmatch(r"\d+:\d+", key) is None + or not isinstance(value, int) + or isinstance(value, bool) + or value <= 0 + for key, value in delta.items() + ): + raise RamdiskError("usage delta journal has invalid counts") + headers = payload.get("headers", {}) + if not isinstance(headers, dict): + raise RamdiskError("usage delta journal has invalid headers") + _compatible_usage_header( + ("usage delta journal", headers), + ) + if expected_merge_id is not None: + if ( + not isinstance(expected_merge_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", expected_merge_id) + ): + raise RamdiskError( + "managed usage recovery has an invalid expected transaction id" + ) + if merge_id != expected_merge_id: + raise RamdiskError( + "usage delta journal transaction does not match managed record" + ) + return merge_id, delta, headers + + +def _usage_journal_transaction_id( + state_dir, + plan=None, + *, + filesystem_for_path=None, +): + """Return one validated live journal ID without applying its delta.""" + payload = _read_usage_journal( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) + if payload is None: + return None + merge_id, _, _ = _validated_usage_delta(payload) + return merge_id + + +def _apply_usage_delta( + canonical_path, + merge_id, + delta, + headers, + plan=None, + *, + source_still_matches=None, +): + lock_path = os.path.join(_state_root(), "usage.lock") + _ensure_private_dir(os.path.dirname(lock_path)) + with open(lock_path, "a+", encoding="utf-8") as lock: + with _usage_lock(lock): + with _canonical_usage_descriptor( + canonical_path, + plan=plan, + source_still_matches=source_still_matches, + ) as canonical_bound: + canonical_snapshot = _usage_snapshot_from_bound( + canonical_bound, + ".coli_usage", + canonical_path, + source="canonical usage target", + allow_missing=True, + ) + applied = set(canonical_snapshot["merge_ids"]) + if merge_id in applied: + # A prior replace may have published this marker while its + # parent-directory fsync reported an uncertain outcome. + # Re-prove the canonical namespace before the caller is + # allowed to delete the last durable delta journal. + _revalidate_bound_parent(canonical_bound) + _fsync_bound_directory(canonical_bound["descriptor"]) + _revalidate_bound_parent(canonical_bound) + return + canonical = dict(canonical_snapshot["counts"]) + merged_header = _compatible_usage_header( + ("usage delta journal", headers), + ("canonical usage history", canonical), + ) + for key, value in delta.items(): + canonical[key] = canonical.get(key, 0) + value + canonical.update(_usage_header_counts(merged_header)) + applied.add(merge_id) + _usage_write_from_bound( + canonical_bound, + ".coli_usage", + canonical_path, + canonical, + merge_ids=applied, + expected_snapshot=canonical_snapshot, + source="canonical usage target", + ) + + +def _recover_delta_from_bound( + state_bound, + journal_snapshot, + state_dir, + canonical_path, + plan=None, + expected_merge_id=None, + *, + source_still_matches=None, + keep_journal=False, +): + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + payload = _journal_payload(journal_snapshot, delta_path) + if payload is None: + if expected_merge_id is not None: + if ( + not isinstance(expected_merge_id, str) + or re.fullmatch(r"[0-9a-f]{32}", expected_merge_id) is None + ): + raise RamdiskError( + "managed usage recovery has an invalid expected transaction id" + ) + raise RamdiskError( + "expected usage delta journal is absent: %s" % delta_path + ) + _durable_unlink_from_bound( + state_bound, + ".coli_usage.delta.json", + source="usage delta journal", + expected_snapshot=journal_snapshot, + ) + return None + merge_id, delta, headers = _validated_usage_delta( + payload, + expected_merge_id=expected_merge_id, + ) + _revalidate_bound_parent(state_bound) + _apply_usage_delta( + canonical_path, + merge_id, + delta, + headers, + plan=plan, + source_still_matches=source_still_matches, + ) + _revalidate_bound_parent(state_bound) + if not keep_journal: + _durable_unlink_from_bound( + state_bound, + ".coli_usage.delta.json", + source="usage delta journal", + expected_snapshot=journal_snapshot, + ) + return merge_id + + +def _recover_delta( + state_dir, + canonical_path, + plan=None, + expected_merge_id=None, + *, + filesystem_for_path=None, + source_still_matches=None, +): + with _managed_state_descriptor( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) as state_bound: + journal_snapshot = _journal_snapshot_from_bound(state_bound, state_dir) + return _recover_delta_from_bound( + state_bound, + journal_snapshot, + state_dir, + canonical_path, + plan=plan, + expected_merge_id=expected_merge_id, + source_still_matches=source_still_matches, + ) + + +def _merge_usage( + record, + canonical_path, + plan=None, + keep_journal=False, + *, + filesystem_for_path=None, + source_still_matches=None, +): + baseline = record.get("usage_baseline") + if not _valid_usage_snapshot(baseline): + raise RamdiskError( + "managed usage recovery is missing a valid exact baseline" + ) + persisted_merge_id = record.get("usage_merge_id") + if persisted_merge_id is not None and ( + not isinstance(persisted_merge_id, str) + or not re.fullmatch(r"[0-9a-f]{32}", persisted_merge_id) + ): + raise RamdiskError( + "managed usage recovery has an invalid transaction id" + ) + state_dir = record["state_dir"] + state_usage = os.path.join(state_dir, ".coli_usage") + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + with _managed_state_descriptor( + state_dir, + plan=plan, + filesystem_for_path=filesystem_for_path, + ) as state_bound: + usage_snapshot = _usage_snapshot_from_bound( + state_bound, + ".coli_usage", + state_usage, + source="managed usage history", + allow_missing=False, + ) + journal_snapshot = _journal_snapshot_from_bound(state_bound, state_dir) + current = usage_snapshot["counts"] + source_header = _compatible_usage_header( + ("managed usage history", current), + ("usage baseline", baseline), + ) + current_counts = _usage_data_counts(current) + baseline_counts = _usage_data_counts(baseline) + for key, baseline_count in baseline_counts.items(): + if baseline_count <= 0: + continue + if key not in current_counts: + raise RamdiskError( + "managed usage history is missing positive baseline counter %s" + % key + ) + if current_counts[key] < baseline_count: + raise RamdiskError( + "managed usage history counter %s regressed below baseline" + % key + ) + delta = { + key: value - baseline_counts.get(key, 0) + for key, value in current_counts.items() + if value > baseline_counts.get(key, 0) + } + + payload = _journal_payload(journal_snapshot, delta_path) + if payload is not None: + journal_merge_id = _journal_merge_id(payload) + if ( + persisted_merge_id is not None + and persisted_merge_id != journal_merge_id + ): + raise RamdiskError( + "usage delta journal transaction does not match managed record" + ) + record["usage_merge_id"] = journal_merge_id + _recover_delta_from_bound( + state_bound, + journal_snapshot, + state_dir, + canonical_path, + plan=plan, + expected_merge_id=journal_merge_id, + source_still_matches=source_still_matches, + keep_journal=keep_journal, + ) + return + + # Prove the absent journal in the same durable directory authority that + # will create and later unlink this transaction's journal. + _durable_unlink_from_bound( + state_bound, + ".coli_usage.delta.json", + source="usage delta journal", + expected_snapshot=journal_snapshot, + ) + merge_id = persisted_merge_id or secrets.token_hex(16) + record["usage_merge_id"] = merge_id + + lock_path = os.path.join(_state_root(), "usage.lock") + _ensure_private_dir(os.path.dirname(lock_path)) + with open(lock_path, "a+", encoding="utf-8") as lock: + with _usage_lock(lock): + with _canonical_usage_descriptor( + canonical_path, + plan=plan, + source_still_matches=source_still_matches, + ) as canonical_bound: + canonical_snapshot = _usage_snapshot_from_bound( + canonical_bound, + ".coli_usage", + canonical_path, + source="canonical usage target", + allow_missing=True, + ) + if merge_id in canonical_snapshot["merge_ids"]: + return + if not delta: + canonical = dict(canonical_snapshot["counts"]) + canonical_header = _compatible_usage_header( + ("managed usage history", current), + ("usage baseline", baseline), + ("canonical usage history", canonical), + ) + if ( + source_header is not None + and _usage_header(canonical) is None + ): + canonical.update( + _usage_header_counts(canonical_header) + ) + _usage_write_from_bound( + canonical_bound, + ".coli_usage", + canonical_path, + canonical, + merge_ids=canonical_snapshot["merge_ids"], + expected_snapshot=canonical_snapshot, + source="canonical usage target", + ) + return + + journal_payload = { + "version": 1, + "id": merge_id, + "delta": delta, + "headers": _usage_header_counts(source_header), + "created_at": _utc_now(), + } + _revalidate_bound_parent(state_bound) + journal_snapshot = _atomic_json_from_bound( + state_bound, + ".coli_usage.delta.json", + journal_payload, + source="usage delta journal", + expected_snapshot=journal_snapshot, + ) + journal_snapshot["value"] = journal_payload + _recover_delta_from_bound( + state_bound, + journal_snapshot, + state_dir, + canonical_path, + plan=plan, + expected_merge_id=merge_id, + source_still_matches=source_still_matches, + keep_journal=keep_journal, + ) diff --git a/c/tests/platform_test_support.py b/c/tests/platform_test_support.py new file mode 100644 index 000000000..ee8807698 --- /dev/null +++ b/c/tests/platform_test_support.py @@ -0,0 +1,350 @@ +"""Central, checked capability markers for cross-platform unittest discovery.""" + +import ast +import os +import signal +import sys +import unittest +from pathlib import Path + + +_LINUX_OPERATIONAL_TESTS = frozenset( + { + "test_ramdisk_benchmark.BenchmarkTest.test_acceptance_is_false_when_applicable_paths_fail", + "test_ramdisk_benchmark.BenchmarkTest.test_best_runtime_knobs_are_saved_only_for_current_topology", + "test_ramdisk_benchmark.BenchmarkTest.test_cuda_benchmark_generates_only_mmap_staged_variants", + "test_ramdisk_benchmark.BenchmarkTest.test_full_per_node_benchmark_sizes_thread_sweep_to_target_node", + "test_ramdisk_integration.RealTmpfsLifecycleTest.test_prepare_status_destroy_on_real_tmpfs", + "test_ramdisk_model_planning.ScanAndPlanTest.test_protected_or_model_overlapping_mount_roots_are_blocked", + "test_ramdisk_mounts.MountAndCopyTest.test_interrupted_mount_helper_retains_pending_recovery_without_unmount", + "test_ramdisk_mounts.MountAndCopyTest.test_completed_mount_failure_retains_observed_mount_without_unmount", + "test_ramdisk_mounts.MountAndCopyTest.test_failed_mount_helper_retains_pending_when_observation_is_inconclusive", + "test_ramdisk_mounts.MountAndCopyTest.test_failed_pending_removal_save_retains_last_durable_pending_mount", + "test_ramdisk_mounts.MountAndCopyTest.test_multi_mount_failure_preflights_all_before_any_unmount", + "test_ramdisk_mounts.MountAndCopyTest.test_prepare_cleanup_runs_even_when_error_manifest_cannot_be_saved", + "test_ramdisk_mounts.MountAndCopyTest.test_prepare_never_unmounts_identityless_successful_mount_by_path", + "test_ramdisk_mounts.MountAndCopyTest.test_prepare_persists_pending_ownership_before_mount_helper", + "test_ramdisk_mounts.MountAndCopyTest.test_prepare_promotes_only_the_exact_recorded_mount_identity", + "test_ramdisk_mounts.MountAndCopyTest.test_prepare_rollback_refuses_exact_mount_with_nested_child", + "test_ramdisk_mounts.MountAndCopyTest.test_runner_oserror_retains_pending_without_absence_reconciliation", + "test_ramdisk_presentation.TuiPlacementContractTest.test_cancelled_prepare_does_not_hide_rollback_failure", + "test_ramdisk_presentation.TuiPlacementContractTest.test_clean_prepare_cancellation_removes_recovery_manifest", + "test_ramdisk_processes.ManagedLaunchTest.test_exact_popen_attempt_is_reaped_across_registration_boundaries", + "test_ramdisk_processes.ManagedLaunchTest.test_postfork_popen_exception_retains_and_reaps_exact_child", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_busy_mount_scan_includes_the_manager_process", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_dashboard_rss_sums_verified_wrapper_and_engine_group", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_destroy_persists_recovery_state_when_kernel_unmount_fails", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_destroy_preflights_every_busy_mount_before_unmounting", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_destroy_refuses_replaced_mount_identity", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_destroy_rejects_nested_child_mounts_before_any_unmount", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_destroy_retains_manifest_for_unrecorded_surviving_mount", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_manifest_rejects_missing_nonce_before_process_signaling", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_manifest_rejects_volatile_durable_state", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_stop_does_not_merge_when_retained_child_is_live", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_stop_persists_error_when_usage_merge_fails", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_stop_preserves_recoverable_error_for_incomplete_mount_layout", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_stop_preserves_termination_failure_until_group_absence_is_proven", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_stop_revalidates_identity_before_escalating_to_sigkill", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_stop_validates_every_pid_before_signaling_any", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_unpublished_recovery_real_setsid_descendant_refuses", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_verified_stop_reaps_a_locally_owned_zombie_before_escalation", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_verified_termination_treats_retained_live_child_as_independent_evidence", + } +) + +_SIGTERM_HANDLER_TESTS = frozenset( + { + "test_ramdisk_cli_module.CliModuleTest.test_cli_termination_guard_restores_the_previous_handler", + "test_ramdisk_cli_smoke.CliJsonSmokeTest.test_cli_sigterm_defers_until_stop_transaction_finishes", + "test_ramdisk_cli_smoke.CliJsonSmokeTest.test_cli_sigterm_requests_cooperative_prepare_rollback", + "test_ramdisk_cli_smoke.CliJsonSmokeTest.test_curses_repeated_sigterm_is_deferred_until_cleanup_guard_exits", + "test_ramdisk_cli_smoke.CliJsonSmokeTest.test_curses_sigterm_uses_cleanup_exception_and_restores_handler", + "test_ramdisk_curses_ui_module.CursesUiModuleTest.test_termination_guard_restores_the_previous_handler", + } +) + +_SIGINT_HANDLER_TESTS = frozenset( + { + "test_ramdisk_cli_module.CliModuleTest.test_prepare_and_destroy_confirmations_keep_ctrl_c_interruptible", + "test_ramdisk_cli_module.CliModuleTest.test_prepare_restores_cooperative_ctrl_c_after_confirmation", + "test_ramdisk_cli_module.CliModuleTest.test_real_tty_ctrl_c_interrupts_confirmation_without_input", + "test_ramdisk_cli_smoke.CliJsonSmokeTest.test_cli_repeated_sigint_stays_cooperative_through_start_rollback", + } +) + +_POSIX_PTY_TESTS = frozenset( + { + "test_ramdisk_cli_module.CliModuleTest.test_real_tty_ctrl_c_interrupts_confirmation_without_input", + } +) + +_POSIX_FIFO_TESTS = frozenset( + { + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_managed_usage_swap_to_fifo_uses_nonblocking_open", + } +) + +_NATIVE_DIRFD_TESTS = frozenset( + { + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_atomic_temp_creation_stays_inside_bound_parent", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_existing_marker_reproves_canonical_parent_before_journal_unlink", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_managed_usage_merge_rejects_symlink_swap_during_verified_open", + "test_ramdisk_state_lifecycle.StateAndSafetyTest.test_managed_usage_seed_write_binds_parent_identity", + } +) + +_LINUX_PIDFD_TESTS = frozenset( + { + "test_ramdisk_platform.LinuxOperationalReadContractTest.test_real_pidfd_group_signal_targets_each_exact_member", + } +) + +_LINUX_STDLIB_PIDFD_TESTS = frozenset( + { + "test_coli_stop.ColiStopIdentityTest.test_pidfd_is_used_instead_of_numeric_pid_when_available", + } +) + +_TARGET_PLATFORM = os.environ.get( + "COLIBRI_TEST_TARGET_PLATFORM", + sys.platform, +) + + +def _linux_pidfd_supported(): + """Match the managed runtime's stdlib-or-libc kernel capability probe.""" + if not _TARGET_PLATFORM.startswith("linux"): + return False + try: + from ramdisk_support import linux_ops + except ImportError: + return False + return linux_ops._pidfd_process_control_supported() + + +def _native_dirfd_supported(): + """Match the descriptor primitives required by durable managed state.""" + required_flags = ("O_DIRECTORY", "O_NOFOLLOW", "O_NONBLOCK") + return ( + not _TARGET_PLATFORM.startswith("win") + and os.name == "posix" + and os.open in getattr(os, "supports_dir_fd", set()) + and os.stat in getattr(os, "supports_dir_fd", set()) + and os.unlink in getattr(os, "supports_dir_fd", set()) + and os.rename in getattr(os, "supports_dir_fd", set()) + and all( + isinstance(getattr(os, name, None), int) + and getattr(os, name) != 0 + for name in required_flags + ) + ) + + +PLATFORM_SKIP_INVENTORY = { + "linux_operational": { + "supported": _TARGET_PLATFORM.startswith("linux"), + "reason": "requires Linux mount, procfs, process-group, or benchmark operations", + "tests": _LINUX_OPERATIONAL_TESTS, + }, + "sigterm_handler": { + "supported": hasattr(signal, "SIGTERM"), + "reason": "SIGTERM is unavailable", + "tests": _SIGTERM_HANDLER_TESTS, + }, + "sigint_handler": { + "supported": hasattr(signal, "SIGINT"), + "reason": "SIGINT is unavailable", + "tests": _SIGINT_HANDLER_TESTS, + }, + "posix_pty": { + "supported": os.name == "posix" and hasattr(os, "openpty"), + "reason": "a POSIX pseudo-terminal is required", + "tests": _POSIX_PTY_TESTS, + }, + "posix_fifo": { + "supported": ( + not _TARGET_PLATFORM.startswith("win") + and os.name == "posix" + and hasattr(os, "mkfifo") + ), + "reason": "a POSIX FIFO is required", + "tests": _POSIX_FIFO_TESTS, + }, + "native_dirfd": { + "supported": _native_dirfd_supported(), + "reason": "native descriptor-relative filesystem operations are unavailable", + "tests": _NATIVE_DIRFD_TESTS, + }, + "linux_pidfd": { + "supported": _linux_pidfd_supported(), + "reason": "Linux pidfd signaling is unavailable", + "tests": _LINUX_PIDFD_TESTS, + }, + "linux_stdlib_pidfd": { + "supported": ( + _TARGET_PLATFORM.startswith("linux") + and callable(getattr(os, "pidfd_open", None)) + and callable(getattr(signal, "pidfd_send_signal", None)) + ), + "reason": "Python stdlib pidfd signaling is unavailable", + "tests": _LINUX_STDLIB_PIDFD_TESTS, + }, +} + + +def _test_id(function): + module = function.__module__.rsplit(".", 1)[-1] + return "%s.%s" % (module, function.__qualname__) + + +def _requires(marker): + entry = PLATFORM_SKIP_INVENTORY[marker] + + def decorate(function): + identifier = _test_id(function) + if identifier not in entry["tests"]: + raise AssertionError( + "unregistered %s platform marker: %s" % (marker, identifier) + ) + return unittest.skipUnless( + entry["supported"], + entry["reason"], + )(function) + + return decorate + + +requires_linux_operational = _requires("linux_operational") +requires_sigterm_handler = _requires("sigterm_handler") +requires_sigint_handler = _requires("sigint_handler") +requires_posix_pty = _requires("posix_pty") +requires_posix_fifo = _requires("posix_fifo") +requires_native_dirfd = _requires("native_dirfd") +requires_linux_pidfd = _requires("linux_pidfd") +requires_linux_stdlib_pidfd = _requires("linux_stdlib_pidfd") + + +def assert_platform_skip_inventory(): + """Fail when a checked marker is added, removed, or renamed silently.""" + decorator_markers = { + "requires_linux_operational": "linux_operational", + "requires_sigterm_handler": "sigterm_handler", + "requires_sigint_handler": "sigint_handler", + "requires_posix_pty": "posix_pty", + "requires_posix_fifo": "posix_fifo", + "requires_native_dirfd": "native_dirfd", + "requires_linux_pidfd": "linux_pidfd", + "requires_linux_stdlib_pidfd": "linux_stdlib_pidfd", + } + discovered = {marker: set() for marker in PLATFORM_SKIP_INVENTORY} + tests_dir = Path(__file__).resolve().parent + + def raw_platform_skip(decorator): + if not isinstance(decorator, ast.Call): + return False + target = decorator.func + if not ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id == "unittest" + and target.attr in ("skipIf", "skipUnless") + and decorator.args + ): + return False + condition_nodes = tuple(ast.walk(decorator.args[0])) + names = { + node.id for node in condition_nodes if isinstance(node, ast.Name) + } + attributes = { + node.attr + for node in condition_nodes + if isinstance(node, ast.Attribute) + } + return ( + ("sys" in names and "platform" in attributes) + or ( + "os" in names + and attributes.intersection({"name", "openpty", "mkfifo"}) + ) + or "signal" in names + ) + + for source in tests_dir.glob("test_*.py"): + tree = ast.parse(source.read_text(encoding="utf-8"), filename=str(source)) + for class_node in ( + node for node in tree.body if isinstance(node, ast.ClassDef) + ): + if source.name.startswith("test_ramdisk_") and any( + raw_platform_skip(decorator) + for decorator in class_node.decorator_list + ): + raise AssertionError( + "raw class platform skip bypasses inventory: %s.%s" + % (source.stem, class_node.name) + ) + for function in ( + node + for node in class_node.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test_") + ): + for decorator in function.decorator_list: + name = getattr(decorator, "id", None) + marker = decorator_markers.get(name) + if marker is not None: + discovered[marker].add( + "%s.%s.%s" + % (source.stem, class_node.name, function.name) + ) + + if source.name.startswith("test_ramdisk_"): + for decorator in function.decorator_list: + if raw_platform_skip(decorator): + raise AssertionError( + "raw platform skip bypasses inventory: %s.%s.%s" + % (source.stem, class_node.name, function.name) + ) + + for call in ( + node + for node in ast.walk(function) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "skipTest" + ): + reason = " ".join( + node.value + for node in call.args + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + ).upper() + if any( + token in reason + for token in ( + "SIGINT", + "SIGTERM", + "LINUX", + "MACOS", + "POSIX", + "WINDOWS", + ) + ): + raise AssertionError( + "dynamic platform skip bypasses inventory: %s.%s.%s" + % (source.stem, class_node.name, function.name) + ) + + mismatches = [] + for marker, entry in PLATFORM_SKIP_INVENTORY.items(): + expected = set(entry["tests"]) + if discovered[marker] != expected: + mismatches.append( + "%s missing=%r stale=%r" + % ( + marker, + sorted(expected - discovered[marker]), + sorted(discovered[marker] - expected), + ) + ) + if mismatches: + raise AssertionError("platform skip inventory drift: " + "; ".join(mismatches)) diff --git a/c/tests/ramdisk_test_support.py b/c/tests/ramdisk_test_support.py new file mode 100644 index 000000000..c452e2ec6 --- /dev/null +++ b/c/tests/ramdisk_test_support.py @@ -0,0 +1,251 @@ +"""Shared dependency-free fixtures for the split RAM-disk test modules.""" + +import argparse +import contextlib +import io +import json +import os +import signal +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + + +C_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(C_DIR)) +import ramdisk # noqa: E402 + +try: + from .platform_test_support import ( + PLATFORM_SKIP_INVENTORY, + assert_platform_skip_inventory, + requires_linux_operational, + requires_linux_pidfd, + requires_linux_stdlib_pidfd, + requires_native_dirfd, + requires_posix_fifo, + requires_posix_pty, + requires_sigint_handler, + requires_sigterm_handler, + ) +except ImportError: + from platform_test_support import ( + PLATFORM_SKIP_INVENTORY, + assert_platform_skip_inventory, + requires_linux_operational, + requires_linux_pidfd, + requires_linux_stdlib_pidfd, + requires_native_dirfd, + requires_posix_fifo, + requires_posix_pty, + requires_sigint_handler, + requires_sigterm_handler, + ) + + +@contextlib.contextmanager +def canonical_temporary_directory(*args, **kwargs): + """Yield a temp root after resolving host-provided aliases such as macOS /var.""" + with tempfile.TemporaryDirectory(*args, **kwargs) as directory: + yield str(Path(directory).resolve()) + + +def host_uid(): + """Return a stable test UID on hosts where Python has no POSIX getuid().""" + getuid = getattr(os, "getuid", None) + return getuid() if getuid is not None else 1000 + + +def write_safetensors(path, tensors): + """Write a tiny valid safetensors file without third-party packages.""" + offset = 0 + header = {} + payload = bytearray() + for tensor in tensors: + name, dtype, size = tensor[:3] + shape = tensor[3] if len(tensor) > 3 else [size if dtype in ("U8", "I8") else size // 4] + header[name] = {"dtype": dtype, "shape": shape, "data_offsets": [offset, offset + size]} + payload.extend(bytes(((offset + index) % 251 for index in range(size)))) + offset += size + raw = json.dumps(header, separators=(",", ":")).encode("utf-8") + while (8 + len(raw)) % 4: + raw += b" " + with open(path, "wb") as stream: + stream.write(len(raw).to_bytes(8, "little")) + stream.write(raw) + stream.write(payload) + + +def expert_tensors(layer, expert, projections=("gate_proj", "up_proj", "down_proj")): + rows = [] + for projection in projections: + name = "model.layers.%d.mlp.experts.%d.%s.weight" % (layer, expert, projection) + rows.append((name, "U8", 16, [4, 4])) + rows.append((name + ".qs", "F32", 16, [4])) + return rows + + +def hardware_fixture(available=128 * ramdisk.GIB, nodes=1, noswap=True, numactl="/usr/bin/numactl"): + per_node = available // nodes + node_rows = [] + for node in range(nodes): + node_rows.append( + { + "id": node, + "cpus": [node * 2, node * 2 + 1], + "cpu_list": "%d-%d" % (node * 2, node * 2 + 1), + "physical_cores": 2, + "memory_total_bytes": per_node * 2, + "memory_available_bytes": per_node, + "distance": [10 if other == node else 20 for other in range(nodes)], + } + ) + effective_cpus = [ + cpu for node in node_rows for cpu in node["cpus"] + ] + return { + "linux": True, + "kernel_release": "6.8.0-test", + "online_nodes": list(range(nodes)), + "effective_nodes": list(range(nodes)), + "effective_cpus": effective_cpus, + "effective_cpu_list": ramdisk._format_range_list(effective_cpus), + "core_groups": [[cpu] for cpu in effective_cpus], + "nodes": node_rows, + "physical_cores": nodes * 2, + "effective_physical_cores": nodes * 2, + "memory": {"total_bytes": available * 2, "available_bytes": available}, + "swap": {"configured": [], "used_bytes": 0}, + "tmpfs": {"supported": True, "noswap_supported": noswap}, + "thp": { + "shmem_enabled": "always within_size [advise] never", + "modes": ["always", "within_size", "advise", "never"], + "within_size_supported": True, + "advise_supported": True, + }, + "numactl": numactl, + "mount": "/bin/mount", + "umount": "/bin/umount", + "sudo": "/usr/bin/sudo", + "hugetlb": {"total_pages": 0, "free_pages": 0, "page_size_bytes": 0}, + } + + +def set_asymmetric_node_cores(hardware, counts=(3, 5)): + """Give fixture nodes realistic, disjoint single-thread core masks.""" + cpu = 0 + groups = [] + for node, count in zip(hardware["nodes"], counts): + cpus = list(range(cpu, cpu + count)) + node["cpus"] = cpus + node["cpu_list"] = ramdisk._format_range_list(cpus) + node["physical_cores"] = count + groups.extend([[item] for item in cpus]) + cpu += count + effective_cpus = [item for group in groups for item in group] + hardware["effective_cpus"] = effective_cpus + hardware["effective_cpu_list"] = ramdisk._format_range_list(effective_cpus) + hardware["core_groups"] = groups + hardware["physical_cores"] = sum(counts) + hardware["effective_physical_cores"] = sum(counts) + + +class ModelFixture: + def __enter__(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name).resolve() + # Expert 0 spans both shards. Expert 1 is a complete one-shard closure. + write_safetensors( + self.root / "model-00001-of-00002.safetensors", + expert_tensors(0, 0, ("gate_proj",)) + [("model.embed_tokens.weight", "U8", 64)], + ) + write_safetensors( + self.root / "model-00002-of-00002.safetensors", + expert_tensors(0, 0, ("up_proj", "down_proj")) + expert_tensors(0, 1), + ) + (self.root / "config.json").write_text( + json.dumps( + { + "hidden_size": 4, + "num_hidden_layers": 1, + "num_attention_heads": 1, + "n_routed_experts": 2, + "num_experts_per_tok": 1, + "moe_intermediate_size": 4, + "intermediate_size": 4, + "kv_lora_rank": 4, + "qk_nope_head_dim": 4, + "qk_rope_head_dim": 4, + "v_head_dim": 4, + "n_shared_experts": 1, + "vocab_size": 32, + "index_head_dim": 0, + } + ), + encoding="utf-8", + ) + (self.root / "tokenizer.json").write_text("{}", encoding="utf-8") + return self + + def __exit__(self, *exc): + self.temp.cleanup() + + +def plan_args(model, **overrides): + values = { + "model": str(model), + "mode": "full", + "topology": "interleaved", + "capacity_gb": None, + "profile": None, + "mount_root": "/mnt/colibri-ram", + "allow_swappable": False, + "prefault": None, + "parallel": 2, + "ctx": 4096, + } + values.update(overrides) + return argparse.Namespace(**values) + + +__all__ = [ + "C_DIR", + "ModelFixture", + "Path", + "PLATFORM_SKIP_INVENTORY", + "argparse", + "canonical_temporary_directory", + "contextlib", + "expert_tensors", + "hardware_fixture", + "host_uid", + "io", + "json", + "mock", + "os", + "plan_args", + "ramdisk", + "assert_platform_skip_inventory", + "requires_linux_operational", + "requires_linux_pidfd", + "requires_linux_stdlib_pidfd", + "requires_native_dirfd", + "requires_posix_fifo", + "requires_posix_pty", + "requires_sigint_handler", + "requires_sigterm_handler", + "set_asymmetric_node_cores", + "shutil", + "signal", + "subprocess", + "sys", + "tempfile", + "threading", + "unittest", + "write_safetensors", +] diff --git a/c/tests/test_ramdisk.py b/c/tests/test_ramdisk.py new file mode 100644 index 000000000..227d34819 --- /dev/null +++ b/c/tests/test_ramdisk.py @@ -0,0 +1,11 @@ +"""Compatibility exports for RAM-disk test fixtures. + +The test cases were split into responsibility-focused modules. Keep the +shared fixtures importable here for integration tests and external test +invocations that historically imported ``test_ramdisk``. +""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 diff --git a/c/tests/test_ramdisk_accelerator.py b/c/tests/test_ramdisk_accelerator.py new file mode 100644 index 000000000..90c1522ae --- /dev/null +++ b/c/tests/test_ramdisk_accelerator.py @@ -0,0 +1,411 @@ +"""Managed accelerator plan and environment tests.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +import copy + +from ramdisk_support import accelerator, presets, processes + + +def _gpu_plan(root): + hardware = hardware_fixture(nodes=4) + hardware["gpus"] = [ + { + "index": 0, + "name": "GPU 0", + "uuid": "GPU-test-0", + "pci_bus_id": "0000:41:00.0", + "numa_node": 1, + "locality": "resolved", + "total_bytes": 32 * ramdisk.GIB, + "free_bytes": 28 * ramdisk.GIB, + }, + { + "index": 1, + "name": "GPU 1", + "uuid": "GPU-test-1", + "pci_bus_id": "0000:c1:00.0", + "numa_node": 3, + "locality": "resolved", + "total_bytes": 32 * ramdisk.GIB, + "free_bytes": 28 * ramdisk.GIB, + }, + ] + hardware["gpu_discovery"] = { + "status": "available", + "error": None, + } + model = ramdisk.scan_model(str(root)) + return presets.resolve_preset( + presets.PRESET_GPU_FASTEST, + plan_args(root), + hardware=hardware, + model=model, + build_plan=ramdisk.build_plan, + load_profile=ramdisk._load_profile, + cuda_capable=True, + )["plan"] + + +class ManagedAcceleratorTest(unittest.TestCase): + def test_cpu_contract_sanitizes_hostile_ambient_gpu_values(self): + environment = { + "CUDA_DEVICE_ORDER": "FASTEST_FIRST", + "CUDA_VISIBLE_DEVICES": "7", + "COLI_CUDA": "1", + "COLI_CUDA_ATTN_PREFIX": "64", + "COLI_CUDA_MTP": "1", + "COLI_ANS_SIDECAR": "/tmp/hostile-ans-sidecar", + "CUDA_RAW_EXPERTS": "1", + "COLI_GPUS": "9", + "COLI_METAL": "1", + "COLI_METAL_SPIN": "1", + "COLI_VULKAN": "1", + "COLI_VK_EXPERTS": "999", + "COLI_VK_SHADERS": "/tmp/hostile-shaders", + "CUDA_EXPERT_GB": "999", + "CUDA_RESERVE_GB": "0", + "COLI_MMAP": "1", + "PIN": "/tmp/hostile", + "REPIN": "0", + "KEEP": "yes", + } + + applied = accelerator._apply_managed_accelerator_environment( + environment, + {"managed_accelerator": {"mode": "cpu", "devices": []}}, + ) + + self.assertEqual( + applied, + { + "COLI_CUDA": "0", + "CUDA_DENSE": "0", + "COLI_CUDA_ATTN": "0", + "COLI_CUDA_ATTN_SHARD": "0", + "DRAFT": "0", + "COLI_MMAP": "0", + "COLI_RAMMAP": "1", + }, + ) + self.assertEqual(environment["KEEP"], "yes") + self.assertNotIn("COLI_GPUS", environment) + self.assertNotIn("CUDA_DEVICE_ORDER", environment) + self.assertNotIn("CUDA_VISIBLE_DEVICES", environment) + self.assertNotIn("COLI_CUDA_ATTN_PREFIX", environment) + self.assertNotIn("COLI_CUDA_MTP", environment) + self.assertNotIn("COLI_ANS_SIDECAR", environment) + self.assertNotIn("CUDA_RAW_EXPERTS", environment) + self.assertNotIn("COLI_METAL", environment) + self.assertNotIn("COLI_METAL_SPIN", environment) + self.assertNotIn("COLI_VULKAN", environment) + self.assertNotIn("COLI_VK_EXPERTS", environment) + self.assertNotIn("COLI_VK_SHADERS", environment) + self.assertNotIn("CUDA_EXPERT_GB", environment) + self.assertNotIn("CUDA_RESERVE_GB", environment) + self.assertNotIn("PIN", environment) + self.assertNotIn("REPIN", environment) + + def test_gpu_contract_applies_only_reviewed_devices_and_mmap_upload(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + environment = { + "COLI_GPU": "7", + "COLI_GPUS": "7,8", + "COLI_CUDA_PIPE": "1", + "CUDA_EXPERT_GB": "4", + "CUDA_RESERVE_GB": "0", + "REPIN": "0", + "COLI_RAMMAP": "1", + } + + applied = accelerator._apply_managed_accelerator_environment( + environment, + plan, + ) + + self.assertEqual(applied["COLI_CUDA"], "1") + self.assertEqual(applied["COLI_GPUS"], "0,1") + self.assertEqual( + applied["CUDA_VISIBLE_DEVICES"], + "GPU-test-0,GPU-test-1", + ) + self.assertNotIn("COLI_GPU", applied) + self.assertNotIn("COLI_CUDA_PIPE", applied) + self.assertEqual(applied["CUDA_EXPERT_GB"], "auto") + self.assertEqual(applied["CUDA_RESERVE_GB"], "2.147483648") + self.assertEqual(applied["COLI_CUDA_ASYNC"], "1") + self.assertEqual(applied["COLI_MMAP"], "1") + self.assertEqual(applied["COLI_RAMMAP"], "0") + self.assertEqual(applied["PIN"], "auto") + self.assertEqual(applied["PIN_GB"], "all") + self.assertEqual(applied["REPIN"], "16") + self.assertEqual(applied["CUDA_DENSE"], "0") + self.assertEqual(applied["COLI_CUDA_ATTN"], "0") + self.assertEqual(applied["COLI_CUDA_ATTN_SHARD"], "0") + + def test_gpu_contract_fails_closed_on_ambient_visibility_mask(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + environment = { + "CUDA_VISIBLE_DEVICES": "GPU-allocation-boundary", + "KEEP": "yes", + } + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "CUDA_VISIBLE_DEVICES", + ): + accelerator._apply_managed_accelerator_environment( + environment, + plan, + ) + + self.assertEqual( + environment, + { + "CUDA_VISIBLE_DEVICES": "GPU-allocation-boundary", + "KEEP": "yes", + }, + ) + + def test_nonzero_physical_devices_launch_as_reviewed_logical_ordinals(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + devices = plan["managed_accelerator"]["devices"] + devices[0].update(index=2, uuid="GPU-physical-2", cuda_ordinal=0) + devices[1].update(index=5, uuid="GPU-physical-5", cuda_ordinal=1) + + applied = accelerator._managed_accelerator_environment(plan) + + self.assertEqual( + applied["CUDA_VISIBLE_DEVICES"], + "GPU-physical-2,GPU-physical-5", + ) + self.assertEqual(applied["COLI_GPUS"], "0,1") + + def test_legacy_gpu_plan_without_ordinal_mapping_fails_closed(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + devices = plan["managed_accelerator"]["devices"] + devices[0]["index"] = 2 + devices[1]["index"] = 5 + for device in devices: + device.pop("cuda_ordinal") + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "no safe ordinal mapping", + ): + accelerator._managed_accelerator_environment(plan) + + def test_logical_ordinal_records_must_stay_in_launch_order(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["managed_accelerator"]["devices"].reverse() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "ordinal mapping", + ): + accelerator._managed_accelerator_environment(plan) + + def test_logical_ordinal_mapping_requires_stable_uuid(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["managed_accelerator"]["devices"][0]["uuid"] = "" + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "ordinal mapping", + ): + accelerator._managed_accelerator_environment(plan) + + def test_dense_layout_applies_exact_sanitized_environment(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["managed_accelerator"]["layout"] = ( + accelerator.GPU_LAYOUT_DENSE_ATTENTION_SHARDED + ) + environment = { + "CUDA_DENSE": "0", + "COLI_CUDA_ATTN": "0", + "COLI_CUDA_ATTN_SHARD": "0", + "COLI_GROUP_ASYNC": "hostile", + "COLI_DSA_GATHER": "hostile", + "DRAFT": "1", + } + + applied = accelerator._apply_managed_accelerator_environment( + environment, + plan, + ) + + self.assertEqual(applied["CUDA_DENSE"], "1") + self.assertEqual(applied["COLI_CUDA_ATTN"], "1") + self.assertEqual(applied["COLI_CUDA_ATTN_SHARD"], "1") + self.assertEqual(environment, applied) + self.assertNotIn("COLI_GROUP_ASYNC", environment) + self.assertNotIn("COLI_DSA_GATHER", environment) + self.assertEqual(environment["DRAFT"], "0") + + def test_gpu_identity_drift_fails_closed(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["hardware"]["effective_mask_source"] = "test-fixture" + current = copy.deepcopy(plan["hardware"]) + current["gpus"][1]["uuid"] = "GPU-replacement" + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "GPU/NUMA identity changed", + ): + processes._assert_effective_masks_unchanged( + plan, + discover_hardware=lambda: current, + ) + + def test_legacy_gpu_identity_falls_back_to_pci(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["hardware"]["effective_mask_source"] = "test-fixture" + current = copy.deepcopy(plan["hardware"]) + plan["managed_accelerator"]["devices"][1].pop("uuid") + current["gpus"][1].pop("uuid") + current["gpus"][1]["pci_bus_id"] = "0000:d1:00.0" + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "GPU/NUMA identity changed", + ): + processes._assert_effective_masks_unchanged( + plan, + discover_hardware=lambda: current, + ) + + def test_uuid_is_primary_when_a_gpu_pci_address_changes(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["hardware"]["effective_mask_source"] = "test-fixture" + current = copy.deepcopy(plan["hardware"]) + current["gpus"][1]["pci_bus_id"] = "0000:d1:00.0" + + processes._assert_effective_masks_unchanged( + plan, + discover_hardware=lambda: current, + ) + + def test_legacy_plan_retains_cpu_direct_rammap_contract(self): + contract = accelerator._managed_accelerator_contract({}) + environment = accelerator._managed_accelerator_environment({}) + + self.assertEqual(contract["mode"], "cpu") + self.assertEqual(contract["layout"], "experts-only") + self.assertEqual(environment["COLI_RAMMAP"], "1") + self.assertEqual(environment["COLI_MMAP"], "0") + self.assertEqual(environment["COLI_CUDA"], "0") + + def test_legacy_gpu_plan_defaults_to_experts_only_layout(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + plan["managed_accelerator"].pop("layout") + + contract = accelerator._managed_accelerator_contract(plan) + environment = accelerator._managed_accelerator_environment(plan) + + self.assertEqual(contract["layout"], "experts-only") + self.assertEqual(environment["CUDA_DENSE"], "0") + self.assertEqual(environment["COLI_CUDA_ATTN"], "0") + self.assertEqual(environment["COLI_CUDA_ATTN_SHARD"], "0") + + def test_confirmation_binds_gpu_identity_and_layout_not_free_vram(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + token = ramdisk._plan_confirmation_token(plan) + changed_projection = copy.deepcopy(plan) + changed_projection["accelerator_projection"][ + "selected_free_bytes" + ] -= ramdisk.GIB + changed_layout = copy.deepcopy(plan) + changed_layout["managed_accelerator"]["layout"] = ( + accelerator.GPU_LAYOUT_DENSE_ATTENTION + ) + changed_identity = copy.deepcopy(plan) + changed_identity["managed_accelerator"]["devices"][0]["uuid"] = ( + "GPU-replacement" + ) + + self.assertEqual( + ramdisk._plan_confirmation_token(changed_projection), + token, + ) + self.assertNotEqual( + ramdisk._plan_confirmation_token(changed_layout), + token, + ) + self.assertNotEqual( + ramdisk._plan_confirmation_token(changed_identity), + token, + ) + + def test_tui_selection_change_preserves_proven_cuda_capability(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + args = argparse.Namespace( + gpu="auto", + gpu_layout="experts-only", + managed_accelerator=copy.deepcopy( + plan["managed_accelerator"] + ), + memory_nodes="1,3", + cpu_list="2-3,6-7", + topology="interleaved", + ) + + accelerator.apply_gpu_selection( + args, + plan["hardware"], + selector="0", + reset_placement=True, + ) + + self.assertEqual( + args.managed_accelerator["capability"], + "available", + ) + + def test_rejected_selection_does_not_partially_mutate_tui_draft(self): + with ModelFixture() as fixture: + plan = _gpu_plan(fixture.root) + args = argparse.Namespace( + gpu="auto", + gpu_layout="experts-only", + managed_accelerator=copy.deepcopy( + plan["managed_accelerator"] + ), + memory_nodes="1,3", + cpu_list="2-3,6-7", + topology="interleaved", + ) + before = copy.deepcopy(vars(args)) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "requires at least two selected GPUs", + ): + accelerator.apply_gpu_selection( + args, + plan["hardware"], + selector="0", + layout="dense-attention-sharded", + reset_placement=True, + ) + + self.assertEqual(vars(args), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_cli.py b/c/tests/test_ramdisk_cli.py new file mode 100644 index 000000000..73376c549 --- /dev/null +++ b/c/tests/test_ramdisk_cli.py @@ -0,0 +1,204 @@ +import contextlib +import importlib.machinery +import importlib.util +import io +import os +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +C_DIR = Path(__file__).resolve().parents[1] +CLI = C_DIR / "coli" +sys.path.insert(0, str(C_DIR)) + +_loader = importlib.machinery.SourceFileLoader("coli_ramdisk_cli", str(CLI)) +_spec = importlib.util.spec_from_loader("coli_ramdisk_cli", _loader) +coli = importlib.util.module_from_spec(_spec) +_loader.exec_module(coli) + + +class RamdiskCliTest(unittest.TestCase): + def run_cli(self, *args, env=None): + merged_env = os.environ.copy() + if env: + merged_env.update(env) + return subprocess.run( + [sys.executable, str(CLI), *args], + cwd=C_DIR, + env=merged_env, + text=True, + capture_output=True, + check=False, + timeout=10, + ) + + def parsed_dispatch(self, *argv): + import ramdisk + + captured = {} + + def fake_dispatch(args, **kwargs): + captured["args"] = args + captured["kwargs"] = kwargs + return 0 + + with mock.patch.object(ramdisk, "dispatch", side_effect=fake_dispatch): + with self.assertRaises(SystemExit) as stopped: + coli.main(list(argv)) + self.assertEqual(stopped.exception.code, 0) + self.assertIn("args", captured) + return captured + + def test_nested_actions_are_scriptable(self): + action_options = { + "plan": ("--json",), + "prepare": (), + "status": ("--json",), + "benchmark": ("--json",), + "start": ("--base-port", "8123"), + "stop": (), + "destroy": ("--yes",), + } + for action, options in action_options.items(): + with self.subTest(action=action): + captured = self.parsed_dispatch("ramdisk", action, *options) + self.assertEqual(captured["args"].ramdisk_action, action) + self.assertEqual(Path(captured["kwargs"]["cli_path"]), CLI) + self.assertTrue(captured["kwargs"]["engine_path"]) + + def test_start_without_port_preserves_the_prepared_deployment_value(self): + captured = self.parsed_dispatch("ramdisk", "start") + self.assertIsNone(captured["args"].base_port) + + def test_shared_model_option_works_before_and_after_action(self): + before = self.parsed_dispatch( + "ramdisk", "--model", "/tmp/model-before", "plan", "--json" + ) + after = self.parsed_dispatch( + "ramdisk", "plan", "--model", "/tmp/model-after", "--json" + ) + self.assertEqual(before["args"].model, "/tmp/model-before") + self.assertEqual(after["args"].model, "/tmp/model-after") + + def test_gpu_options_work_before_and_after_action(self): + before = self.parsed_dispatch( + "ramdisk", + "--gpu", + "0,2", + "--gpu-layout", + "dense-attention", + "plan", + ) + after = self.parsed_dispatch( + "ramdisk", + "plan", + "--gpu", + "1,3", + "--gpu-layout", + "dense-attention-sharded", + ) + + self.assertEqual(before["args"].gpu, "0,2") + self.assertEqual(before["args"].gpu_layout, "dense-attention") + self.assertEqual(after["args"].gpu, "1,3") + self.assertEqual( + after["args"].gpu_layout, + "dense-attention-sharded", + ) + + def test_planning_knobs_reach_dispatch(self): + captured = self.parsed_dispatch( + "ramdisk", + "plan", + "--model", + "/tmp/model", + "--mode", + "partial", + "--topology", + "per-node", + "--memory-nodes", + "0,2", + "--cpu-list", + "0-15,32-47", + "--capacity-gb", + "12.5", + "--profile", + "/tmp/profile", + "--mount-root", + "/tmp/ram-root", + "--allow-swappable", + "--prefault", + "0", + "--parallel", + "3", + "--json", + ) + args = captured["args"] + self.assertEqual(args.mode, "partial") + self.assertEqual(args.topology, "per-node") + self.assertEqual(args.memory_nodes, "0,2") + self.assertEqual(args.cpu_list, "0-15,32-47") + self.assertEqual(args.capacity_gb, 12.5) + self.assertEqual(args.profile, "/tmp/profile") + self.assertEqual(args.mount_root, "/tmp/ram-root") + self.assertTrue(args.allow_swappable) + self.assertEqual(args.prefault, 0) + self.assertEqual(args.parallel, 3) + self.assertTrue(args.json) + + def test_bare_command_rejects_non_tty_without_curses(self): + result = self.run_cli("ramdisk") + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, "") + self.assertIn("interactive TUI requires a terminal", result.stderr) + self.assertIn("ramdisk plan --json", result.stderr) + + def test_help_lists_scriptable_actions(self): + result = self.run_cli("ramdisk", "--help") + self.assertEqual(result.returncode, 0, result.stderr) + for action in ( + "plan", + "prepare", + "status", + "benchmark", + "start", + "stop", + "destroy", + ): + self.assertIn(action, result.stdout) + help_text = " ".join(result.stdout.split()) + self.assertIn("one shared model copy and one engine", help_text) + self.assertIn("complete copy and independent engine per NUMA node", help_text) + + def test_kv_resume_notice_uses_durable_state_directory(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + model = root / "model" + state = root / "state" + model.mkdir() + state.mkdir() + self.write_kv_header(model / ".coli_kv", 3) + self.write_kv_header(state / ".coli_kv", 7) + + output = io.StringIO() + with mock.patch.dict(os.environ, {"COLI_STATE_DIR": str(state)}), \ + contextlib.redirect_stdout(output): + coli.kv_resume_notice(str(model)) + + self.assertIn("7 tokens", output.getvalue()) + self.assertNotIn("3 tokens", output.getvalue()) + + @staticmethod + def write_kv_header(path, tokens): + fields = [0] * 8 + fields[6] = tokens + path.write_bytes(b"COLIKV1\0" + struct.pack("<8i", *fields)) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_cli_module.py b/c/tests/test_ramdisk_cli_module.py new file mode 100644 index 000000000..eaa973422 --- /dev/null +++ b/c/tests/test_ramdisk_cli_module.py @@ -0,0 +1,423 @@ +"""Direct contracts for the callback-injected CLI support module.""" + +from types import SimpleNamespace + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import cli + + +class CliModuleTest(unittest.TestCase): + def test_parser_accepts_lifecycle_options_before_and_after_an_action(self): + parser = argparse.ArgumentParser(prog="coli ramdisk") + cli.configure_parser(parser) + + before = parser.parse_args( + ["--model", "/model-before", "plan", "--json"] + ) + after = parser.parse_args( + [ + "plan", + "--model", + "/model-after", + "--topology", + "per-node", + "--memory-nodes", + "0,2", + "--cpu-list", + "0-7,16-23", + "--json", + ] + ) + + self.assertEqual(before.model, "/model-before") + self.assertEqual(after.model, "/model-after") + self.assertEqual(after.topology, "per-node") + self.assertEqual(after.memory_nodes, "0,2") + self.assertEqual(after.cpu_list, "0-7,16-23") + + def test_gpu_selection_and_layout_work_before_and_after_action(self): + parser = argparse.ArgumentParser(prog="coli ramdisk") + cli.configure_parser(parser) + + before = parser.parse_args( + [ + "--gpu", + "0,2", + "--gpu-layout", + "dense-attention", + "plan", + ] + ) + after = parser.parse_args( + [ + "plan", + "--gpu", + "1", + "--gpu-layout", + "dense-attention-sharded", + ] + ) + + self.assertEqual(before.gpu, "0,2") + self.assertEqual(before.gpu_layout, "dense-attention") + self.assertEqual(after.gpu, "1") + self.assertEqual( + after.gpu_layout, + "dense-attention-sharded", + ) + + def test_dispatch_injects_cancellation_and_rendering_dependencies(self): + args = argparse.Namespace( + ramdisk_action="benchmark", + json=True, + ) + cancel_event = object() + payload = {"schema": ramdisk.BENCHMARK_SCHEMA, "variants": []} + benchmark = mock.Mock(return_value=payload) + emit_json = mock.Mock() + + @contextlib.contextmanager + def termination_guard(cancelable): + self.assertTrue(cancelable) + yield {"cancel_event": cancel_event, "signum": None} + + result = cli.dispatch( + args, + cli_path="/coli", + engine_path="/engine", + build_plan=mock.Mock(), + prepare=mock.Mock(), + status=mock.Mock(), + benchmark=benchmark, + start=mock.Mock(), + stop=mock.Mock(), + destroy=mock.Mock(), + human_plan=mock.Mock(), + human_status=mock.Mock(), + human_benchmark=mock.Mock(), + json_print=emit_json, + termination_guard=termination_guard, + ) + + self.assertEqual(result, 0) + benchmark.assert_called_once_with( + args, + cli_path="/coli", + engine_path="/engine", + cancel_event=cancel_event, + ) + emit_json.assert_called_once_with(payload) + + def test_dispatch_keeps_the_versioned_json_error_contract(self): + args = argparse.Namespace(ramdisk_action="plan", json=True) + emit_json = mock.Mock() + + result = cli.dispatch( + args, + build_plan=mock.Mock( + side_effect=ramdisk.RamdiskError("invalid plan") + ), + prepare=mock.Mock(), + status=mock.Mock(), + benchmark=mock.Mock(), + start=mock.Mock(), + stop=mock.Mock(), + destroy=mock.Mock(), + human_plan=mock.Mock(), + human_status=mock.Mock(), + human_benchmark=mock.Mock(), + json_print=emit_json, + ) + + self.assertEqual(result, 2) + emit_json.assert_called_once_with( + { + "schema": "colibri.ramdisk.error.v1", + "version": 1, + "error": "invalid plan", + } + ) + + def test_launch_tui_routes_textual_and_curses_lazily(self): + args = argparse.Namespace() + lifecycle = object() + textual = SimpleNamespace(launch_tui=mock.Mock(return_value=19)) + run_frontend = mock.Mock(side_effect=lambda callback: callback()) + finish_frontend = mock.Mock() + + textual_result = cli.launch_tui( + args, + cli_path="/coli", + engine_path="/engine", + lifecycle=lifecycle, + run_tui_frontend=run_frontend, + load_textual_frontend=lambda: textual, + target_platform="linux", + environment={"COLI_RAMDISK_UI": "textual"}, + finish_frontend=finish_frontend, + ) + + self.assertEqual(textual_result, 19) + textual.launch_tui.assert_called_once_with( + args, + cli_path="/coli", + engine_path="/engine", + lifecycle=lifecycle, + ) + finish_frontend.assert_called_once_with() + + events = [] + + @contextlib.contextmanager + def curses_guard(): + events.append("guard-enter") + try: + yield + finally: + events.append("guard-exit") + + def missing_textual(): + raise ModuleNotFoundError( + "No module named 'textual'", + name="textual", + ) + + curses_result = cli.launch_tui( + args, + lifecycle=lifecycle, + run_tui_frontend=lambda callback: callback(), + legacy_tui=lambda *_args: None, + curses_termination_guard=curses_guard, + load_textual_frontend=missing_textual, + curses_wrapper=lambda *_args: events.append("curses") or 23, + target_platform="linux", + environment={"COLI_RAMDISK_UI": "auto"}, + ) + + self.assertEqual(curses_result, 23) + self.assertEqual(events, ["guard-enter", "curses", "guard-exit"]) + + def test_non_linux_tui_rejection_precedes_frontend_loading(self): + stderr = io.StringIO() + loader = mock.Mock( + side_effect=AssertionError("frontend must stay unloaded") + ) + + with contextlib.redirect_stderr(stderr): + result = cli.launch_tui( + argparse.Namespace(), + lifecycle=object(), + run_tui_frontend=mock.Mock(), + load_textual_frontend=loader, + target_platform="darwin", + environment={"COLI_RAMDISK_UI": "textual"}, + ) + + self.assertEqual(result, 2) + self.assertIn("supported only on Linux", stderr.getvalue()) + loader.assert_not_called() + + @requires_sigterm_handler + def test_cli_termination_guard_restores_the_previous_handler(self): + previous = signal.getsignal(signal.SIGTERM) + + with cli._cli_termination_guard(True) as termination: + handler = signal.getsignal(signal.SIGTERM) + self.assertTrue(callable(handler)) + handler(signal.SIGTERM, None) + self.assertTrue(termination["cancel_event"].is_set()) + self.assertEqual( + termination["signum"], + int(signal.SIGTERM), + ) + + self.assertIs(signal.getsignal(signal.SIGTERM), previous) + + @requires_sigint_handler + def test_prepare_and_destroy_confirmations_keep_ctrl_c_interruptible(self): + previous = signal.getsignal(signal.SIGINT) + + def interrupt_prompt(_message): + handler = signal.getsignal(signal.SIGINT) + self.assertIs(handler, signal.default_int_handler) + handler(signal.SIGINT, None) + + for action in ("prepare", "destroy"): + args = argparse.Namespace(ramdisk_action=action, json=False) + + def prepare(_args, cancel_event=None): + cli._confirm("prepare?") + self.fail("prepare continued after Ctrl-C") + + def destroy(_args): + cli._confirm("destroy?") + self.fail("destroy continued after Ctrl-C") + + with self.subTest(action=action), mock.patch.object( + cli.sys, + "stdin", + mock.Mock(isatty=mock.Mock(return_value=True)), + ), mock.patch.object( + cli.sys, + "stdout", + mock.Mock(isatty=mock.Mock(return_value=True)), + ), mock.patch( + "builtins.input", + side_effect=interrupt_prompt, + ), self.assertRaises(KeyboardInterrupt): + cli.dispatch( + args, + build_plan=mock.Mock(), + prepare=prepare, + status=mock.Mock(), + benchmark=mock.Mock(), + start=mock.Mock(), + stop=mock.Mock(), + destroy=destroy, + human_plan=mock.Mock(), + human_status=mock.Mock(), + human_benchmark=mock.Mock(), + ) + + self.assertIs(signal.getsignal(signal.SIGINT), previous) + + @requires_sigint_handler + def test_prepare_restores_cooperative_ctrl_c_after_confirmation(self): + args = argparse.Namespace(ramdisk_action="prepare", json=False) + + def prepare(_args, cancel_event=None): + cli._confirm("prepare?") + handler = signal.getsignal(signal.SIGINT) + self.assertTrue(callable(handler)) + self.assertIsNot(handler, signal.default_int_handler) + handler(signal.SIGINT, None) + self.assertTrue(cancel_event.is_set()) + raise ramdisk._OperationCancelled("termination requested") + + with mock.patch.object( + cli.sys, + "stdin", + mock.Mock(isatty=mock.Mock(return_value=True)), + ), mock.patch.object( + cli.sys, + "stdout", + mock.Mock(isatty=mock.Mock(return_value=True)), + ), mock.patch( + "builtins.input", + return_value="yes", + ), mock.patch("sys.stderr", new_callable=io.StringIO): + result = cli.dispatch( + args, + build_plan=mock.Mock(), + prepare=prepare, + status=mock.Mock(), + benchmark=mock.Mock(), + start=mock.Mock(), + stop=mock.Mock(), + destroy=mock.Mock(), + human_plan=mock.Mock(), + human_status=mock.Mock(), + human_benchmark=mock.Mock(), + ) + + self.assertEqual(result, 128 + int(signal.SIGINT)) + + @requires_posix_pty + @requires_sigint_handler + def test_real_tty_ctrl_c_interrupts_confirmation_without_input(self): + import select + import time + + script = r""" +import sys +sys.path.insert(0, sys.argv[1]) +from ramdisk_support import cli + +with cli._cli_termination_guard(True): + cli._confirm("confirm?") +raise SystemExit(99) +""" + master_fd, slave_fd = os.openpty() + process = subprocess.Popen( + [sys.executable, "-c", script, str(C_DIR)], + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + close_fds=True, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + os.close(slave_fd) + output = bytearray() + try: + deadline = time.monotonic() + 5.0 + while b"[y/N]" not in output and time.monotonic() < deadline: + readable, _, _ = select.select([master_fd], [], [], 0.1) + if readable: + output.extend(os.read(master_fd, 4096)) + self.assertIn(b"[y/N]", output, output.decode(errors="replace")) + + process.send_signal(signal.SIGINT) + try: + process.wait(timeout=2.0) + except subprocess.TimeoutExpired: + self.fail( + "Ctrl-C was swallowed at the confirmation prompt:\n%s" + % output.decode(errors="replace") + ) + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=2.0) + os.close(master_fd) + + self.assertIn( + process.returncode, + (-int(signal.SIGINT), 128 + int(signal.SIGINT)), + ) + + def test_import_does_not_load_terminal_or_linux_backends(self): + script = r""" +import importlib.abc +import sys + +blocked = { + "curses", + "ramdisk_textual", + "ramdisk_support.linux_ops", + "ramdisk_support.mounts", +} + +class RejectOptional(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if ( + fullname in blocked + or fullname == "textual" + or fullname.startswith("textual.") + ): + raise AssertionError("eager optional import: " + fullname) + return None + +sys.meta_path.insert(0, RejectOptional()) +sys.path.insert(0, sys.argv[1]) +from ramdisk_support import cli + +assert callable(cli.configure_parser) +assert not (blocked & set(sys.modules)), sorted(blocked & set(sys.modules)) +""" + result = subprocess.run( + [sys.executable, "-c", script, str(C_DIR)], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_cli_smoke.py b/c/tests/test_ramdisk_cli_smoke.py new file mode 100644 index 000000000..f8dcac9e1 --- /dev/null +++ b/c/tests/test_ramdisk_cli_smoke.py @@ -0,0 +1,188 @@ +"""RAM-disk command-line JSON and subprocess smoke tests.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + + +class CliJsonSmokeTest(unittest.TestCase): + @requires_sigterm_handler + def test_curses_sigterm_uses_cleanup_exception_and_restores_handler(self): + previous = signal.getsignal(signal.SIGTERM) + with self.assertRaises(ramdisk._TuiTerminationSignal) as raised: + with ramdisk._curses_termination_guard(): + handler = signal.getsignal(signal.SIGTERM) + self.assertTrue(callable(handler)) + handler(signal.SIGTERM, None) + + self.assertEqual(raised.exception.signum, int(signal.SIGTERM)) + self.assertIs(signal.getsignal(signal.SIGTERM), previous) + + @requires_sigterm_handler + def test_curses_repeated_sigterm_is_deferred_until_cleanup_guard_exits(self): + previous = signal.getsignal(signal.SIGTERM) + with ramdisk._curses_termination_guard(): + handler = signal.getsignal(signal.SIGTERM) + with self.assertRaises(ramdisk._TuiTerminationSignal): + handler(signal.SIGTERM, None) + handler(signal.SIGTERM, None) + + self.assertIs(signal.getsignal(signal.SIGTERM), previous) + + @requires_sigterm_handler + def test_cli_sigterm_requests_cooperative_prepare_rollback(self): + args = argparse.Namespace(ramdisk_action="prepare", json=False) + previous = signal.getsignal(signal.SIGTERM) + + def interrupted_prepare(_args, cancel_event=None): + handler = signal.getsignal(signal.SIGTERM) + self.assertTrue(callable(handler)) + handler(signal.SIGTERM, None) + self.assertTrue(cancel_event.is_set()) + raise ramdisk._OperationCancelled("termination requested") + + with mock.patch.object( + ramdisk, + "prepare", + side_effect=interrupted_prepare, + ), mock.patch("sys.stderr", new_callable=io.StringIO): + self.assertEqual( + ramdisk.dispatch(args), + 128 + int(signal.SIGTERM), + ) + + self.assertIs(signal.getsignal(signal.SIGTERM), previous) + + @requires_sigint_handler + def test_cli_repeated_sigint_stays_cooperative_through_start_rollback(self): + args = argparse.Namespace(ramdisk_action="start", json=False) + previous = signal.getsignal(signal.SIGINT) + + def interrupted_start( + _args, + cli_path=None, + engine_path=None, + cancel_event=None, + ): + handler = signal.getsignal(signal.SIGINT) + self.assertTrue(callable(handler)) + handler(signal.SIGINT, None) + handler(signal.SIGINT, None) + self.assertTrue(cancel_event.is_set()) + raise ramdisk._OperationCancelled("termination requested") + + with mock.patch.object( + ramdisk, + "start", + side_effect=interrupted_start, + ), mock.patch("sys.stderr", new_callable=io.StringIO): + self.assertEqual( + ramdisk.dispatch(args), + 128 + int(signal.SIGINT), + ) + + self.assertIs(signal.getsignal(signal.SIGINT), previous) + + @requires_sigterm_handler + def test_cli_sigterm_defers_until_stop_transaction_finishes(self): + args = argparse.Namespace(ramdisk_action="stop", json=False) + completed = [] + + def interrupted_stop(_args): + handler = signal.getsignal(signal.SIGTERM) + self.assertTrue(callable(handler)) + handler(signal.SIGTERM, None) + completed.append(True) + return {"state": "stopped"} + + with mock.patch.object(ramdisk, "stop", side_effect=interrupted_stop): + self.assertEqual( + ramdisk.dispatch(args), + 128 + int(signal.SIGTERM), + ) + + self.assertEqual(completed, [True]) + + def test_stop_dispatch_surfaces_an_incomplete_recovery_workspace(self): + args = argparse.Namespace(ramdisk_action="stop", json=False) + with mock.patch.object( + ramdisk, "stop", return_value={"state": "error"} + ), mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: + self.assertEqual(ramdisk.dispatch(args), 2) + + self.assertIn("workspace is incomplete", stderr.getvalue()) + self.assertIn("ramdisk status", stderr.getvalue()) + + def test_benchmark_dispatch_preserves_versioned_json_schema(self): + payload = { + "schema": ramdisk.BENCHMARK_SCHEMA, + "version": ramdisk.MANIFEST_VERSION, + "variants": [], + } + args = argparse.Namespace(ramdisk_action="benchmark", json=True) + with mock.patch.object(ramdisk, "benchmark", return_value=payload), mock.patch.object( + ramdisk, "_json_print" + ) as emit: + self.assertEqual(ramdisk.dispatch(args), 0) + emit.assert_called_once_with(payload) + + def test_plan_json_is_parseable_even_when_host_has_blockers(self): + with ModelFixture() as fixture: + result = subprocess.run( + [sys.executable, str(C_DIR / "coli"), "ramdisk", "plan", "--model", str(fixture.root), "--json"], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + timeout=20, + ) + payload = json.loads(result.stdout) + self.assertEqual(payload["schema"], ramdisk.PLAN_SCHEMA) + self.assertIn(result.returncode, (0, 2)) + self.assertEqual(result.stderr, "") + + def test_invalid_plan_and_absent_status_keep_json_contract(self): + with ModelFixture() as fixture, canonical_temporary_directory() as state: + environment = dict( + os.environ, + XDG_STATE_HOME=state, + COLI_RAMDISK_MANIFEST=os.path.join(state, "manifest.json"), + ) + invalid = subprocess.run( + [ + sys.executable, + str(C_DIR / "coli"), + "ramdisk", + "plan", + "--model", + str(fixture.root), + "--capacity-gb", + "nan", + "--json", + ], + cwd=C_DIR, + env=environment, + text=True, + capture_output=True, + check=False, + timeout=20, + ) + status = subprocess.run( + [sys.executable, str(C_DIR / "coli"), "ramdisk", "status", "--json"], + cwd=C_DIR, + env=environment, + text=True, + capture_output=True, + check=False, + timeout=20, + ) + self.assertEqual(invalid.returncode, 2) + self.assertEqual(json.loads(invalid.stdout)["schema"], "colibri.ramdisk.error.v1") + self.assertEqual(invalid.stderr, "") + self.assertEqual(status.returncode, 0) + self.assertEqual(json.loads(status.stdout)["schema"], ramdisk.STATUS_SCHEMA) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_facade.py b/c/tests/test_ramdisk_facade.py new file mode 100644 index 000000000..73f8bd5c3 --- /dev/null +++ b/c/tests/test_ramdisk_facade.py @@ -0,0 +1,227 @@ +import argparse +import contextlib +import inspect +import io +import subprocess +import sys +import unittest +from pathlib import Path +from unittest import mock + + +C_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(C_DIR)) +import ramdisk # noqa: E402 + + +PUBLIC_SIGNATURES = { + "discover_hardware": "()", + "scan_model": "(model_dir)", + "build_plan": "(args, hardware=None, model=None)", + "configure_parser": "(parser, common_parent=None)", + "prepare": ( + "(args, progress=None, display_plan=True, expected_plan_token=None, " + "cancel_event=None)" + ), + "start": "(args, cli_path=None, engine_path=None, cancel_event=None)", + "stop": "(args=None)", + "destroy": "(args, expected_manifest_token=None)", + "status": "(deep=True)", + "benchmark": "(args, cli_path=None, engine_path=None, cancel_event=None)", + "dispatch": "(args, cli_path=None, engine_path=None, system=None)", + "launch_tui": "(args, cli_path=None, engine_path=None, system=None)", +} + +SCHEMA_CONSTANTS = { + "MANIFEST_VERSION": 1, + "PLAN_SCHEMA": "colibri.ramdisk.plan.v1", + "STATUS_SCHEMA": "colibri.ramdisk.status.v1", + "BENCHMARK_SCHEMA": "colibri.ramdisk.benchmark.v1", +} + + +class RamdiskFacadeContractTest(unittest.TestCase): + def test_public_exports_and_signatures_remain_stable(self): + self.assertTrue(issubclass(ramdisk.RamdiskError, RuntimeError)) + for name, expected_signature in PUBLIC_SIGNATURES.items(): + with self.subTest(name=name): + exported = getattr(ramdisk, name) + self.assertTrue(callable(exported)) + self.assertEqual(str(inspect.signature(exported)), expected_signature) + + def test_schema_names_and_versions_remain_stable(self): + for name, expected in SCHEMA_CONSTANTS.items(): + with self.subTest(name=name): + self.assertEqual(getattr(ramdisk, name), expected) + + def test_import_does_not_eagerly_load_optional_textual_frontend(self): + script = """ +import importlib.abc +import sys + +class RejectTextual(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "ramdisk_textual" or fullname == "textual" or fullname.startswith("textual."): + raise AssertionError("optional Textual frontend loaded eagerly: " + fullname) + return None + +sys.meta_path.insert(0, RejectTextual()) +sys.path.insert(0, sys.argv[1]) +import ramdisk + +loaded = sorted( + name + for name in sys.modules + if name == "ramdisk_textual" or name == "textual" or name.startswith("textual.") +) +assert loaded == [], loaded +""" + result = subprocess.run( + [sys.executable, "-c", script, str(C_DIR)], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_import_keeps_optional_facade_layers_lazy(self): + script = r""" +import importlib.abc +import sys + +blocked = { + "ramdisk_textual", + "ramdisk_ui", + "ramdisk_support.benchmark", + "ramdisk_support.curses_ui", + "ramdisk_support.presentation", + "ramdisk_support.processes", + "ramdisk_support.runtime_monitor", +} + +class RejectOptionalLayers(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname in blocked: + raise AssertionError("eager optional facade import: " + fullname) + return None + +sys.meta_path.insert(0, RejectOptionalLayers()) +sys.path.insert(0, sys.argv[1]) +import ramdisk + +assert callable(ramdisk.build_plan) +assert not (blocked & set(sys.modules)), sorted(blocked & set(sys.modules)) +""" + result = subprocess.run( + [sys.executable, "-c", script, str(C_DIR)], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_import_does_not_initialize_the_stdlib_http_stack(self): + script = r""" +import importlib.abc +import sys + +blocked = {"ssl", "urllib.request"} + +class RejectHttpStack(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname in blocked: + raise AssertionError("eager network import: " + fullname) + return None + +sys.meta_path.insert(0, RejectHttpStack()) +sys.path.insert(0, sys.argv[1]) +import ramdisk + +assert callable(ramdisk.status) +assert not (blocked & set(sys.modules)), sorted(blocked & set(sys.modules)) +""" + result = subprocess.run( + [sys.executable, "-c", script, str(C_DIR)], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_lazy_public_exports_remain_discoverable_and_star_importable(self): + script = r""" +import sys + +sys.path.insert(0, sys.argv[1]) +import ramdisk + +assert "BENCHMARK_PROMPT" in dir(ramdisk) +assert "urllib" in dir(ramdisk) +assert "ramdisk_support.benchmark" not in sys.modules +assert "urllib.request" not in sys.modules + +namespace = {} +exec("from ramdisk import *", namespace) +assert isinstance(namespace["BENCHMARK_PROMPT"], str) +assert namespace["BENCHMARK_PROMPT"] +assert namespace["urllib"] is sys.modules["urllib"] +""" + result = subprocess.run( + [sys.executable, "-c", script, str(C_DIR)], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_historical_urllib_patch_seam_remains_lazy_and_forwarded(self): + process_module = mock.Mock() + with mock.patch( + "ramdisk.urllib.request.urlopen", + new=mock.sentinel.urlopen, + ), mock.patch.object( + ramdisk, + "_processes_module", + return_value=process_module, + ): + ramdisk._wait_managed_ready(mock.sentinel.record, 3.0) + + process_module._wait_managed_ready.assert_called_once_with( + mock.sentinel.record, + 3.0, + api_key=None, + cancel_event=None, + process_matches=ramdisk._process_matches, + urlopen=mock.sentinel.urlopen, + ) + + def test_non_linux_tui_fails_before_loading_ui_or_probing_hardware(self): + stderr = io.StringIO() + with ( + mock.patch.object(ramdisk.sys, "platform", "darwin"), + mock.patch.object( + ramdisk, + "_load_textual_frontend", + side_effect=AssertionError("Textual frontend should not load"), + ) as load_textual, + mock.patch.object( + ramdisk, + "discover_hardware", + side_effect=AssertionError("Linux hardware should not be probed"), + ) as discover_hardware, + contextlib.redirect_stderr(stderr), + ): + result = ramdisk.launch_tui(argparse.Namespace()) + + self.assertEqual(result, 2) + self.assertIn("the TUI is supported only on Linux", stderr.getvalue()) + load_textual.assert_not_called() + discover_hardware.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_integration.py b/c/tests/test_ramdisk_integration.py new file mode 100644 index 000000000..cf8b7e06f --- /dev/null +++ b/c/tests/test_ramdisk_integration.py @@ -0,0 +1,114 @@ +"""Opt-in real-tmpfs lifecycle smoke test. + +Run in a privileged or isolated user/mount namespace: + + COLI_RAMDISK_INTEGRATION=1 python3 -m unittest -v tests.test_ramdisk_integration + +The ordinary dependency-free suite skips this test because mounting is a +machine-level capability, not a unit-test prerequisite. +""" + +import argparse +import os +import tempfile +import unittest +from unittest import mock +from pathlib import Path + +try: + from . import test_ramdisk + from .platform_test_support import requires_linux_operational +except ImportError: # unittest discovery imports tests as top-level modules + import test_ramdisk + from platform_test_support import requires_linux_operational + +import ramdisk + + +@unittest.skipUnless( + os.environ.get("COLI_RAMDISK_INTEGRATION") == "1", + "set COLI_RAMDISK_INTEGRATION=1 inside a private mount namespace", +) +class RealTmpfsLifecycleTest(unittest.TestCase): + @requires_linux_operational + def test_prepare_status_destroy_on_real_tmpfs(self): + with test_ramdisk.ModelFixture() as fixture, tempfile.TemporaryDirectory( + prefix="colibri-ramdisk-state-", dir="/var/tmp" + ) as durable: + mount_root = "/mnt/colibri-ramdisk-it-%d" % os.getpid() + args = test_ramdisk.plan_args( + fixture.root, + mount_root=mount_root, + allow_swappable=True, + thp="auto", + yes=True, + ) + before_swap = ramdisk.discover_hardware()["swap"]["used_bytes"] + with mock.patch.dict( + os.environ, + { + "XDG_STATE_HOME": durable, + "COLI_RAMDISK_MANIFEST": os.path.join( + durable, "colibri", "ramdisk", "manifest.json" + ), + }, + ): + plan = ramdisk.build_plan(args) + # This test validates the prepare/status/destroy lifecycle, not the + # production 16 GiB OS reserve. Fail on every non-memory prerequisite, + # then bypass only that reserve in the reviewed plan returned to + # prepare; production planning and admission remain untouched. + blockers = plan["blockers"] + memory_blockers = [ + b for b in blockers if "memory" in b.lower() or "reserve" in b.lower() + ] + self.assertEqual( + [b for b in blockers if b not in memory_blockers], + [], + "non-memory plan blockers: %s" % blockers, + ) + plan["blockers"] = [] + plan["reserve"]["os_margin_bytes"] = 0 + plan["reserve"]["total_os_margin_bytes"] = 0 + plan["reserve"]["required_global_bytes"] = ( + plan["staging"]["staged_bytes"] + + plan["reserve"]["runtime_bytes"] + + plan["reserve"]["page_table_bytes"] + ) + plan["reserve"]["total_required_bytes"] = plan["reserve"][ + "required_global_bytes" + ] + with mock.patch.object(ramdisk, "build_plan", return_value=plan): + prepared = ramdisk.prepare(args, display_plan=False) + cleanup_needed = True + try: + self.assertEqual(prepared["state"], "ready") + report = ramdisk.status() + self.assertTrue(report["source_fingerprint_verified"]) + self.assertTrue(all(item["verified"] for item in report["mounts"])) + self.assertEqual( + {item["filesystem"] for item in report["mounts"]}, {"tmpfs"} + ) + durable_kv = ( + Path(ramdisk._state_root()) + / "engines" + / "preserved" + / ".coli_kv" + ) + durable_kv.parent.mkdir(parents=True, mode=0o700) + durable_kv.write_bytes(b"durable-test-state") + destroyed = ramdisk.destroy(argparse.Namespace(yes=True)) + self.assertTrue(destroyed["destroyed"]) + cleanup_needed = False + self.assertFalse(os.path.ismount(mount_root)) + self.assertEqual(ramdisk.status()["state"], "absent") + self.assertEqual(durable_kv.read_bytes(), b"durable-test-state") + finally: + if cleanup_needed: + ramdisk.destroy(argparse.Namespace(yes=True)) + after_swap = ramdisk.discover_hardware()["swap"]["used_bytes"] + self.assertLessEqual(after_swap, before_swap) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_model_planning.py b/c/tests/test_ramdisk_model_planning.py new file mode 100644 index 000000000..bd993014d --- /dev/null +++ b/c/tests/test_ramdisk_model_planning.py @@ -0,0 +1,704 @@ +"""RAM-disk model scanning, hardware discovery, and placement planning tests.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import discovery as discovery_support + + +def quantized_expert_tensors(fmt, hidden=384, intermediate=256): + tensors = [] + for projection, rows, columns in ( + ("gate_proj", intermediate, hidden), + ("up_proj", intermediate, hidden), + ("down_proj", hidden, intermediate), + ): + name = "model.layers.0.mlp.experts.0.%s.weight" % projection + if fmt == 1: + weight_bytes, scale_bytes = rows * columns, rows * 4 + elif fmt == 5: + groups = (columns + 63) // 64 + weight_bytes, scale_bytes = rows * groups * 24, rows * groups * 4 + elif fmt == 6: + weight_bytes = rows * ((columns + 255) // 256) * 98 + scale_bytes = 4 + elif fmt == 8: + weight_bytes = rows * columns + scale_bytes = ((rows + 127) // 128) * ((columns + 127) // 128) * 4 + else: + raise AssertionError("unsupported test format") + tensors.append((name, "U8", weight_bytes, [weight_bytes])) + tensors.append((name + ".qs", "F32", scale_bytes, [scale_bytes // 4])) + return tensors + + +class ScanAndPlanTest(unittest.TestCase): + GLM_USAGE_HEADER = "-1 1 2\n-2 1 3815245270\n" + + def test_scan_indexes_complete_six_tensor_experts_and_sorted_shards(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + self.assertEqual(model["shard_names"], sorted(model["shard_names"])) + self.assertEqual(set(model["experts"]), {"0:0", "0:1"}) + self.assertEqual(len(model["experts"]["0:0"]["tensors"]), 6) + self.assertEqual(model["experts"]["0:0"]["shards"], model["shard_names"]) + self.assertTrue(model["experts"]["0:1"]["direct_map_eligible"]) + + def test_scan_accepts_all_direct_engine_expert_formats(self): + for fmt in (5, 6, 8): + with self.subTest(fmt=fmt), ModelFixture() as fixture: + for shard in fixture.root.glob("*.safetensors"): + shard.unlink() + tensors = quantized_expert_tensors(fmt) + write_safetensors(fixture.root / "model.safetensors", tensors) + config_path = fixture.root / "config.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config.update(hidden_size=384, moe_intermediate_size=256) + config_path.write_text(json.dumps(config), encoding="utf-8") + model = ramdisk.scan_model(str(fixture.root)) + + expert = model["experts"]["0:0"] + self.assertTrue(expert["direct_map_eligible"]) + self.assertEqual(expert["tensor_bytes"], sum(item[2] for item in tensors)) + + def test_scan_rejects_ambiguous_or_corrupt_direct_format_geometry(self): + fixtures = { + "unstamped E8/FP8 collision": (6, 98, 64, None), + "wrong FP8 scale count": (8, 384, 256, 4), + } + for label, (fmt, hidden, intermediate, extra_scale_bytes) in fixtures.items(): + with self.subTest(case=label), ModelFixture() as fixture: + for shard in fixture.root.glob("*.safetensors"): + shard.unlink() + tensors = quantized_expert_tensors(fmt, hidden, intermediate) + if extra_scale_bytes: + name, dtype, size, shape = tensors[1] + tensors[1] = (name, dtype, size + extra_scale_bytes, + [shape[0] + extra_scale_bytes // 4]) + write_safetensors(fixture.root / "model.safetensors", tensors) + config_path = fixture.root / "config.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config.update(hidden_size=hidden, moe_intermediate_size=intermediate) + config_path.write_text(json.dumps(config), encoding="utf-8") + model = ramdisk.scan_model(str(fixture.root)) + + self.assertFalse(model["experts"]["0:0"]["direct_map_eligible"]) + + def test_scan_preserves_unstamped_int8_fp8_collision_inversion(self): + with ModelFixture() as fixture: + for shard in fixture.root.glob("*.safetensors"): + shard.unlink() + # gate/up [2,256] have two per-row scales and two FP8 blocks. The + # engine's unstamped policy deliberately selects incumbent int8. + tensors = quantized_expert_tensors(1, hidden=256, intermediate=2) + write_safetensors(fixture.root / "model.safetensors", tensors) + config_path = fixture.root / "config.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config.update(hidden_size=256, moe_intermediate_size=2) + config_path.write_text(json.dumps(config), encoding="utf-8") + model = ramdisk.scan_model(str(fixture.root)) + + self.assertTrue(model["experts"]["0:0"]["direct_map_eligible"]) + + def test_fingerprint_changes_when_source_identity_changes(self): + with ModelFixture() as fixture: + before = ramdisk.scan_model(str(fixture.root))["fingerprint"] + shard = fixture.root / "model-00002-of-00002.safetensors" + stat = shard.stat() + # FILETIME and several mounted filesystems round sub-second + # changes, so use a change every supported host can observe. + os.utime( + shard, + ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000), + ) + after = ramdisk.scan_model(str(fixture.root))["fingerprint"] + self.assertNotEqual(before, after) + + def test_fingerprint_includes_configuration_and_tokenizer_content(self): + with ModelFixture() as fixture: + before = ramdisk.scan_model(str(fixture.root))["fingerprint"] + tokenizer = fixture.root / "tokenizer.json" + tokenizer.write_text('{"version":"changed"}', encoding="utf-8") + after_tokenizer = ramdisk.scan_model(str(fixture.root))["fingerprint"] + config = json.loads((fixture.root / "config.json").read_text(encoding="utf-8")) + config["vocab_size"] += 1 + (fixture.root / "config.json").write_text(json.dumps(config), encoding="utf-8") + after_config = ramdisk.scan_model(str(fixture.root))["fingerprint"] + self.assertNotEqual(before, after_tokenizer) + self.assertNotEqual(after_tokenizer, after_config) + + def test_model_and_profile_json_require_object_roots(self): + with ModelFixture() as fixture: + config_path = fixture.root / "config.json" + config_path.write_text("[]", encoding="utf-8") + with self.assertRaisesRegex(ramdisk.RamdiskError, "JSON object"): + ramdisk.scan_model(str(fixture.root)) + + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + profile = fixture.root / "profile.json" + profile.write_text("[]", encoding="utf-8") + with self.assertRaisesRegex(ramdisk.RamdiskError, "contain an object"): + ramdisk._load_profile(str(profile), model) + + def test_partial_selection_is_profile_driven_deterministic_and_budgeted(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + shard2 = next(item for item in model["shards"] if item["name"].startswith("model-00002")) + selected, experts = ramdisk._select_partial( + model, {"0:0": 1000, "0:1": 100}, shard2["size_bytes"] + ) + self.assertEqual(selected, [shard2["name"]]) + self.assertEqual(experts, ["0:1"]) + self.assertLessEqual(sum(item["size_bytes"] for item in model["shards"] if item["name"] in selected), shard2["size_bytes"]) + + def test_partial_selection_can_stage_ineligible_experts_via_tmpfs_slabs(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + model["experts"]["0:1"]["direct_map_eligible"] = False + shard2 = next( + item for item in model["shards"] if item["name"].startswith("model-00002") + ) + selected, direct_experts = ramdisk._select_partial( + model, {"0:1": 100}, shard2["size_bytes"] + ) + self.assertEqual(selected, [shard2["name"]]) + self.assertEqual(direct_experts, []) + + def test_partial_plan_compares_shard_closures_with_same_budget_pinning(self): + with ModelFixture() as fixture: + profile = fixture.root / ".coli_usage" + profile.write_text("0 0 1000\n0 1 10\n", encoding="utf-8") + plan = ramdisk.build_plan( + plan_args(fixture.root, mode="partial", capacity_gb=1), + hardware=hardware_fixture(), + ) + comparison = plan["profile"]["pin_comparison"] + self.assertEqual(comparison["budget_bytes"], ramdisk.GIB) + self.assertGreaterEqual(comparison["coverage"], plan["profile"]["coverage"]) + self.assertGreater( + plan["profile"]["predicted_expert_bytes_avoided_per_staged_byte"], 0 + ) + + def test_headered_profile_validates_model_dimensions_and_engine(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + profile = fixture.root / ".coli_usage" + profile.write_text( + self.GLM_USAGE_HEADER + "0 1 10\n", + encoding="utf-8", + ) + _, counts = ramdisk._load_profile(str(profile), model) + self.assertEqual(counts, {"0:1": 10}) + + profile.write_text( + "-1 9 2\n-2 1 3815245270\n0 1 10\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "dimensions"): + ramdisk._load_profile(str(profile), model) + + profile.write_text( + "-1 1 2\n-2 1 1\n0 1 10\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "engine"): + ramdisk._load_profile(str(profile), model) + + def test_partial_mode_requires_a_profile_and_positive_budget(self): + with ModelFixture() as fixture: + with self.assertRaisesRegex(ramdisk.RamdiskError, "positive --capacity"): + ramdisk.build_plan( + plan_args(fixture.root, mode="partial"), + hardware=hardware_fixture(), + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "requires .coli_usage"): + ramdisk.build_plan( + plan_args(fixture.root, mode="partial", capacity_gb=1), + hardware=hardware_fixture(), + ) + + def test_profile_fingerprint_mismatch_is_rejected(self): + with ModelFixture() as fixture: + profile = fixture.root / "profile.json" + profile.write_text( + json.dumps( + { + "model_fingerprint": "sha256:not-this-model", + "counts": [{"layer": 0, "expert": 1, "count": 9}], + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "fingerprint"): + ramdisk.build_plan( + plan_args(fixture.root, mode="partial", capacity_gb=1, profile=str(profile)), + hardware=hardware_fixture(), + ) + + def test_capacity_refusal_preserves_os_and_runtime_reserve(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture(available=1 * ramdisk.GIB) + ) + self.assertTrue(any("reserve" in blocker for blocker in plan["blockers"])) + self.assertGreaterEqual(plan["reserve"]["os_margin_bytes"], 16 * ramdisk.GIB) + + def test_cgroup_v2_uses_tightest_limiting_ancestor_headroom(self): + with canonical_temporary_directory() as temporary: + mountpoint = Path(temporary) / "cgroup" + parent = mountpoint / "service.slice" + leaf = parent / "colibri.scope" + leaf.mkdir(parents=True) + for directory, maximum, current, high in ( + (mountpoint, "max", "0", "max"), + (parent, "1000", "450", "700"), + (leaf, "900", "100", "650"), + ): + (directory / "memory.max").write_text(maximum, encoding="utf-8") + (directory / "memory.current").write_text(current, encoding="utf-8") + (directory / "memory.high").write_text(high, encoding="utf-8") + info = ramdisk._discover_cgroup_memory( + cgroup_text="0::/service.slice/colibri.scope\n", + mountinfo_text=( + "36 25 0:32 /host.slice/container.scope %s " + "rw,nosuid,nodev,noexec - cgroup2 cgroup rw\n" + % mountpoint + ), + ) + + self.assertEqual(info["version"], 2) + self.assertEqual(info["status"], "limited") + self.assertEqual(info["available_bytes"], 550) + self.assertEqual(info["limit_bytes"], 1000) + self.assertEqual(info["current_bytes"], 450) + self.assertEqual(info["high_available_bytes"], 250) + self.assertTrue(info["limiting_path"].endswith("service.slice")) + + def test_cgroup_discovery_fails_closed_when_proc_contract_is_unreadable(self): + class DeniedLinuxOps: + is_linux = True + + @staticmethod + def read_cgroup_contract(path): + raise ramdisk.RamdiskError( + "cannot read cgroup contract %s: denied" % path + ) + + for cgroup_text, mountinfo_text, expected_path in ( + (None, "", "/proc/self/cgroup"), + ("0::/\n", None, "/proc/self/mountinfo"), + ): + with self.subTest(path=expected_path): + info = discovery_support._discover_cgroup_memory_with_ops( + cgroup_text=cgroup_text, + mountinfo_text=mountinfo_text, + ops=DeniedLinuxOps(), + ) + + self.assertEqual(info["status"], "unavailable") + self.assertIn(expected_path, info["error"]) + + def test_hybrid_cgroup_falls_back_to_visible_v1_memory_mount(self): + with canonical_temporary_directory() as temporary: + mountpoint = Path(temporary) / "memory" + leaf = mountpoint / "legacy" + leaf.mkdir(parents=True) + (mountpoint / "memory.limit_in_bytes").write_text( + "9223372036854771712", encoding="utf-8" + ) + (mountpoint / "memory.usage_in_bytes").write_text( + "0", encoding="utf-8" + ) + (leaf / "memory.limit_in_bytes").write_text( + "4096", encoding="utf-8" + ) + (leaf / "memory.usage_in_bytes").write_text( + "1024", encoding="utf-8" + ) + info = ramdisk._discover_cgroup_memory( + cgroup_text="0::/unified\n7:memory:/legacy\n", + mountinfo_text=( + "42 25 0:38 / %s rw,nosuid,nodev,noexec " + "- cgroup cgroup rw,memory\n" % mountpoint + ), + ) + + self.assertEqual(info["version"], 1) + self.assertEqual(info["status"], "limited") + self.assertEqual(info["available_bytes"], 3072) + + def test_cgroup_v1_memory_limit_has_compatible_headroom(self): + with canonical_temporary_directory() as temporary: + mountpoint = Path(temporary) / "memory" + leaf = mountpoint / "colibri" + leaf.mkdir(parents=True) + (mountpoint / "memory.limit_in_bytes").write_text( + "9223372036854771712", encoding="utf-8" + ) + (mountpoint / "memory.usage_in_bytes").write_text("0", encoding="utf-8") + (leaf / "memory.limit_in_bytes").write_text("2048", encoding="utf-8") + (leaf / "memory.usage_in_bytes").write_text("512", encoding="utf-8") + info = ramdisk._discover_cgroup_memory( + cgroup_text="5:cpu:/other\n7:memory:/colibri\n", + mountinfo_text=( + "42 25 0:38 / %s rw,nosuid,nodev,noexec - cgroup cgroup rw,memory\n" + % mountpoint + ), + ) + + self.assertEqual(info["version"], 1) + self.assertEqual(info["available_bytes"], 1536) + self.assertEqual(info["limit_bytes"], 2048) + self.assertIsNone(info["high_available_bytes"]) + + def test_plan_caps_capacity_at_cgroup_hard_limit_and_warns_on_high(self): + with ModelFixture() as fixture: + hardware = hardware_fixture() + hardware["cgroup_memory"] = { + "version": 2, + "status": "limited", + "available_bytes": ramdisk.GIB, + "high_available_bytes": ramdisk.GIB // 2, + "error": None, + } + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware) + + self.assertEqual(plan["reserve"]["available_bytes"], ramdisk.GIB) + self.assertEqual(plan["reserve"]["host_available_bytes"], 128 * ramdisk.GIB) + self.assertTrue( + any("cgroup memory hard-limit headroom" in item for item in plan["blockers"]) + ) + self.assertTrue(any("memory.high" in item for item in plan["warnings"])) + + def test_plan_blocks_when_cgroup_memory_contract_cannot_be_read(self): + with ModelFixture() as fixture: + hardware = hardware_fixture() + hardware["cgroup_memory"] = { + "version": 2, + "status": "unavailable", + "available_bytes": None, + "high_available_bytes": None, + "error": "permission denied", + } + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware) + + self.assertTrue( + any( + "cannot validate cgroup memory headroom" in item + for item in plan["blockers"] + ) + ) + + def test_runtime_availability_is_capped_by_current_cgroup_headroom(self): + cgroup = {"available_bytes": 400, "error": None} + with mock.patch.object( + ramdisk, "_meminfo", return_value={"MemAvailable": 1000} + ), mock.patch.object( + ramdisk, "_discover_cgroup_memory", return_value=cgroup + ): + self.assertEqual(ramdisk._available_memory(), 400) + with mock.patch.object( + ramdisk, "_node_meminfo", return_value={"MemFree": 900} + ), mock.patch.object( + ramdisk, "_discover_cgroup_memory", return_value=cgroup + ): + self.assertEqual( + ramdisk._available_for_mount({"node": 0}, plan={}), 400 + ) + + def test_per_node_replication_refuses_any_under_capacity_node(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(available=18 * ramdisk.GIB, nodes=2) + plan = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), hardware=hardware + ) + self.assertTrue(any("NUMA node" in blocker for blocker in plan["blockers"])) + + def test_missing_noswap_blocks_unless_explicitly_accepted(self): + with ModelFixture() as fixture: + blocked = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture(noswap=False) + ) + accepted = ramdisk.build_plan( + plan_args(fixture.root, allow_swappable=True), hardware=hardware_fixture(noswap=False) + ) + self.assertTrue(any("noswap" in blocker for blocker in blocked["blockers"])) + self.assertFalse(any("noswap" in blocker for blocker in accepted["blockers"])) + + @requires_linux_operational + def test_protected_or_model_overlapping_mount_roots_are_blocked(self): + with ModelFixture() as fixture: + broad = ramdisk.build_plan( + plan_args(fixture.root, mount_root="/"), hardware=hardware_fixture() + ) + overlap = ramdisk.build_plan( + plan_args(fixture.root, mount_root=str(fixture.root / "ram")), hardware=hardware_fixture() + ) + self.assertTrue(any("protected broad" in blocker for blocker in broad["blockers"])) + self.assertTrue(any("canonical model" in blocker for blocker in overlap["blockers"])) + + def test_invalid_numeric_planning_inputs_are_actionable(self): + with ModelFixture() as fixture: + with self.assertRaisesRegex(ramdisk.RamdiskError, "finite positive"): + ramdisk.build_plan( + plan_args(fixture.root, capacity_gb=float("nan")), + hardware=hardware_fixture(), + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "--ctx"): + ramdisk.build_plan( + plan_args(fixture.root, ctx=-1), hardware=hardware_fixture() + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "--parallel"): + ramdisk.build_plan( + plan_args(fixture.root, parallel=0), hardware=hardware_fixture() + ) + + def test_per_node_plan_reports_exact_replica_totals(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), + hardware=hardware_fixture(nodes=2), + ) + self.assertEqual(plan["staging"]["replica_count"], 2) + self.assertEqual( + plan["staging"]["total_staged_bytes"], + plan["staging"]["staged_bytes"] * 2, + ) + self.assertEqual( + plan["reserve"]["total_runtime_bytes"], + plan["reserve"]["runtime_bytes"] * 2, + ) + + def test_sparse_effective_masks_remain_exact_in_shared_plan(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=3) + for row, node_id, cpus in zip( + hardware["nodes"], + (0, 2, 8), + ([0, 1], [4, 5], [16, 17]), + ): + row.update( + { + "id": node_id, + "cpus": list(cpus), + "cpu_list": ramdisk._format_range_list(cpus), + } + ) + hardware.update( + { + "online_nodes": [0, 2, 8], + "effective_nodes": [0, 8], + "effective_cpus": [0, 1, 16, 17], + "core_groups": [[0], [1], [16], [17]], + } + ) + plan = ramdisk.build_plan( + plan_args( + fixture.root, + memory_nodes="0,8", + cpu_list="0-1,16-17", + ), + hardware=hardware, + ) + + self.assertEqual(plan["placement"]["memory_nodes"], [0, 8]) + self.assertEqual(plan["placement"]["cpu_list"], "0-1,16-17") + self.assertEqual( + plan["mounts"][0]["policy"], r"interleave=static:0\,8" + ) + self.assertEqual(plan["staging"]["replica_count"], 1) + self.assertTrue( + any("may fall back" in warning for warning in plan["warnings"]) + ) + + def test_selected_replica_nodes_create_only_selected_full_copies(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan( + plan_args( + fixture.root, + topology="per-node", + memory_nodes="0,2", + cpu_list="0-1,4-5", + ), + hardware=hardware_fixture(nodes=4), + ) + + self.assertEqual([mount["node"] for mount in plan["mounts"]], [0, 2]) + self.assertEqual( + [mount["policy"] for mount in plan["mounts"]], + ["bind=static:0", "bind=static:2"], + ) + self.assertEqual(plan["staging"]["replica_count"], 2) + self.assertEqual( + [entry["cpu_list"] for entry in plan["placement"]["engine_cpu_sets"]], + ["0-1", "4-5"], + ) + + def test_remote_cpu_selection_is_explicit_in_shared_mode(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan( + plan_args( + fixture.root, + memory_nodes="0", + cpu_list="2-3", + ), + hardware=hardware_fixture(nodes=2), + ) + + self.assertEqual(plan["placement"]["remote_cpu_list"], "2-3") + self.assertEqual(plan["placement"]["memory_policy"], "strict-bind") + self.assertEqual(plan["mounts"][0]["policy"], "bind=static:0") + environment = ramdisk._benchmark_environment( + {"plan": plan}, + plan["mounts"][0]["path"], + "/durable/state", + True, + ) + self.assertEqual(environment["COLI_NUMA"], "1") + self.assertEqual(environment["COLI_NUMA_NODES"], "0") + self.assertTrue( + any("remote NUMA access" in warning for warning in plan["warnings"]) + ) + + def test_replica_cpu_selection_cannot_name_unselected_nodes(self): + with ModelFixture() as fixture: + with self.assertRaisesRegex( + ramdisk.RamdiskError, "outside the selected replica nodes" + ): + ramdisk.build_plan( + plan_args( + fixture.root, + topology="per-node", + memory_nodes="0", + cpu_list="2-3", + ), + hardware=hardware_fixture(nodes=2), + ) + + def test_effective_cpuset_is_a_hard_placement_boundary(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=2) + hardware["effective_nodes"] = [0] + hardware["effective_cpus"] = [0, 1] + hardware["core_groups"] = [[0], [1]] + with self.assertRaisesRegex(ramdisk.RamdiskError, "effective host mask"): + ramdisk.build_plan( + plan_args(fixture.root, memory_nodes="1"), + hardware=hardware, + ) + with self.assertRaisesRegex(ramdisk.RamdiskError, "effective host mask"): + ramdisk.build_plan( + plan_args(fixture.root, cpu_list="2-3"), + hardware=hardware, + ) + + def test_cpu_selection_must_keep_effective_sibling_groups_whole(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=1) + hardware["core_groups"] = [[0, 1]] + with self.assertRaisesRegex(ramdisk.RamdiskError, "whole effective physical cores"): + ramdisk.build_plan( + plan_args(fixture.root, cpu_list="0"), + hardware=hardware, + ) + + def test_engine_threads_are_counted_only_inside_selected_cpu_mask(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=1) + hardware["nodes"][0]["physical_cores"] = 64 + hardware["physical_cores"] = 64 + hardware["effective_physical_cores"] = 64 + hardware["effective_cpus"] = [0] + hardware["core_groups"] = [[0]] + plan = ramdisk.build_plan( + plan_args(fixture.root, cpu_list="0"), + hardware=hardware, + ) + + self.assertEqual( + plan["placement"]["engine_cpu_sets"][0]["physical_cores"], 1 + ) + self.assertEqual(ramdisk._node_core_count(plan), 1) + + def test_effective_mask_drift_changes_review_identity(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=2) + broad = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware) + hardware["effective_nodes"] = [0] + hardware["effective_cpus"] = [0, 1] + hardware["core_groups"] = [[0], [1]] + constrained = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware + ) + + self.assertNotEqual( + ramdisk._plan_confirmation_token(broad), + ramdisk._plan_confirmation_token(constrained), + ) + + def test_managed_launch_revalidation_rejects_cpuset_drift(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=2) + hardware["effective_mask_source"] = "kernel-task-status" + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware) + current = hardware_fixture(nodes=2) + current["effective_nodes"] = [0] + current["effective_cpus"] = [0, 1] + with mock.patch.object(ramdisk, "discover_hardware", return_value=current): + with self.assertRaisesRegex(ramdisk.RamdiskError, "changed since preparation"): + ramdisk._assert_effective_masks_unchanged(plan) + + def test_numa_sampling_avoids_page_order_resonance(self): + total_pages = 1 << 20 + sample_pages = 256 + for nodes in (2, 4): + indices = ramdisk._sample_page_indices( + total_pages, + sample_pages, + nodes, + ) + self.assertEqual( + indices, + ramdisk._sample_page_indices( + total_pages, + sample_pages, + nodes, + ), + ) + self.assertEqual(len(indices), sample_pages) + self.assertEqual(len(set(indices)), sample_pages) + self.assertTrue( + all(0 <= index < total_pages for index in indices) + ) + for order in range(10): + allocation_units = ( + total_pages + (1 << order) - 1 + ) // (1 << order) + if allocation_units < 7 * nodes: + continue + counts = [ + sum( + 1 + for index in indices + if (index >> order) % nodes == node + ) + for node in range(nodes) + ] + ideal = float(sample_pages) / nodes + deviation = max( + abs(count - ideal) / ideal + for count in counts + ) + self.assertLessEqual( + deviation, + 0.15, + "order %d across %d nodes: %r" + % (order, nodes, counts), + ) + self.assertEqual( + ramdisk._sample_page_indices(8, 20, 4), + list(range(8)), + ) diff --git a/c/tests/test_ramdisk_mounts.py b/c/tests/test_ramdisk_mounts.py new file mode 100644 index 000000000..dd0de6deb --- /dev/null +++ b/c/tests/test_ramdisk_mounts.py @@ -0,0 +1,1278 @@ +"""RAM-disk mount, copy, and staged-namespace tests.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import lifecycle as lifecycle_support +from ramdisk_support import mounts as mounts_support + + +class MountAndCopyTest(unittest.TestCase): + def setUp(self): + for name, value in ( + ("_ensure_busy_mount_scan_available", None), + ("_busy_mount_references", []), + ): + patcher = mock.patch.object(ramdisk, name, return_value=value) + patcher.start() + self.addCleanup(patcher.stop) + + def _stage_fixture_namespace(self, fixture, destination): + for source in fixture.root.glob("*.safetensors"): + target = Path(destination) / source.name + shutil.copy2(source, target) + target.chmod(0o400) + + def test_prepare_proves_cleanup_scan_before_confirmation_or_mutation(self): + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + + unavailable = ramdisk.RamdiskError( + "install the psmisc package and retry" + ) + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, + "_ensure_busy_mount_scan_available", + side_effect=unavailable, + ) as ensure_scan, mock.patch.object( + ramdisk, "_confirm" + ) as confirm, mock.patch.object( + ramdisk, "_save_manifest" + ) as save, mock.patch.object( + ramdisk, "_mount_tmpfs" + ) as mount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "install the psmisc package", + ): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), + display_plan=False, + ) + + ensure_scan.assert_called_once_with( + plan["mount_root"], + hardware=plan["hardware"], + ) + confirm.assert_not_called() + save.assert_not_called() + mount.assert_not_called() + + def test_cgroup_headroom_uses_the_injected_discovery_service(self): + discover = mock.Mock( + return_value={"available_bytes": 1234, "error": None} + ) + + self.assertEqual( + mounts_support._default_cgroup_available_memory( + discover_cgroup_memory=discover, + ), + 1234, + ) + discover.assert_called_once_with() + + with mock.patch.object( + ramdisk, + "_discover_cgroup_memory", + return_value={"available_bytes": 5678, "error": None}, + ): + self.assertEqual(ramdisk._cgroup_available_memory(), 5678) + + with self.assertRaisesRegex( + mounts_support.RamdiskError, + "cannot validate cgroup memory headroom: unreadable", + ): + mounts_support._default_cgroup_available_memory( + discover_cgroup_memory=lambda: { + "available_bytes": None, + "error": "unreadable", + }, + ) + + def test_mountinfo_preserves_noncontiguous_mpol_nodemask(self): + line = ( + "36 25 0:32 / /mnt/colibri-ram rw,noatime - tmpfs tmpfs " + "rw,noswap,nodev,nosuid,noexec,mode=700,huge=within_size," + "mpol=interleave:0-1,3\n" + ) + # A still-open NamedTemporaryFile cannot be reopened by native Windows. + # The parser is portable when fed a closed, byte-exact fixture. + with tempfile.TemporaryDirectory() as directory: + fixture = Path(directory) / "mountinfo" + fixture.write_bytes(line.encode("utf-8")) + parsed = ramdisk._mount_table(str(fixture)) + self.assertEqual(parsed[0]["super_options"][-1], "rw") + self.assertIn("mpol=interleave:0-1,3", parsed[0]["super_options"]) + + def test_mount_falls_back_from_within_size_to_advise_only_on_option_error(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware_fixture()) + calls = [] + + def run(command, **kwargs): + calls.append(command) + if len(calls) == 1: + return subprocess.CompletedProcess(command, 32, "", "mount: invalid argument") + return subprocess.CompletedProcess(command, 0, "", "") + + with mock.patch.object( + ramdisk, "_trusted_system_binary", return_value="/bin/mount" + ), mock.patch.object(ramdisk, "_run", side_effect=run), mock.patch.object( + ramdisk, "_privileged", side_effect=lambda command, hardware: command + ): + ramdisk._mount_tmpfs(plan, plan["mounts"][0]) + self.assertIn("huge=within_size", calls[0][4]) + self.assertIn("huge=advise", calls[1][4]) + self.assertTrue(plan["mounts"][0]["effective_noswap"]) + + def test_swappable_fallback_preserves_supported_within_size_thp(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan( + plan_args(fixture.root, allow_swappable=True), + hardware=hardware_fixture(), + ) + calls = [] + + def run(command, **kwargs): + calls.append(command) + if len(calls) <= 2: + return subprocess.CompletedProcess( + command, 32, "", "mount: invalid argument" + ) + return subprocess.CompletedProcess(command, 0, "", "") + + with mock.patch.object( + ramdisk, "_trusted_system_binary", return_value="/bin/mount" + ), mock.patch.object(ramdisk, "_run", side_effect=run), mock.patch.object( + ramdisk, "_privileged", side_effect=lambda command, hardware: command + ): + ramdisk._mount_tmpfs(plan, plan["mounts"][0]) + self.assertEqual(len(calls), 3) + self.assertIn("huge=within_size", calls[2][4]) + self.assertNotIn("noswap", calls[2][4].split(",")) + self.assertFalse(plan["mounts"][0]["effective_noswap"]) + + def test_mount_keeps_private_tmpfs_over_reusable_underlying_directory(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware_fixture()) + options = ramdisk._mount_option_list(plan, plan["mounts"][0]) + self.assertIn("mode=0700", options) + self.assertIn("X-mount.mkdir=0755", options) + + def test_non_option_mount_error_does_not_retry_weaker_options(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware_fixture()) + result = subprocess.CompletedProcess([], 1, "", "permission denied") + with mock.patch.object( + ramdisk, "_trusted_system_binary", return_value="/bin/mount" + ), mock.patch.object( + ramdisk, "_run", return_value=result + ) as run, mock.patch.object( + ramdisk, "_privileged", side_effect=lambda command, hardware: command + ): + with self.assertRaises( + mounts_support._MountHelperCompletedError + ): + ramdisk._mount_tmpfs(plan, plan["mounts"][0]) + self.assertEqual(run.call_count, 1) + + def test_mount_runner_oserror_is_not_a_completed_helper_failure(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + failure = OSError("Popen failed after helper construction began") + with mock.patch.object( + ramdisk, "_trusted_system_binary", return_value="/bin/mount" + ), mock.patch.object( + ramdisk, "_run", side_effect=failure + ), mock.patch.object( + ramdisk, "_privileged", side_effect=lambda command, hardware: command + ): + with self.assertRaises(OSError) as caught: + ramdisk._mount_tmpfs(plan, plan["mounts"][0]) + + self.assertIs(caught.exception, failure) + + def test_unmount_uses_trusted_noncanonicalizing_util_linux_command(self): + run = mock.Mock( + return_value=subprocess.CompletedProcess([], 0, "", "") + ) + hardware = hardware_fixture() + + mounts_support._umount_path( + "/mnt/colibri-test", + hardware, + trusted_system_binary=mock.Mock( + return_value="/usr/bin/umount" + ), + run=run, + privileged=lambda command, ignored: command, + ) + + run.assert_called_once_with( + [ + "/usr/bin/umount", + "--no-canonicalize", + "--", + "/mnt/colibri-test", + ] + ) + + @requires_linux_operational + def test_interrupted_mount_helper_retains_pending_recovery_without_unmount(self): + snapshots = [] + observed = {"mounted": False} + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + mount = plan["mounts"][0] + actual = { + "mount_id": 44, + "device": "0:44", + "filesystem": "tmpfs", + "source": "tmpfs", + } + interrupted = ramdisk._TuiTerminationSignal(signal.SIGTERM) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + def run(_command, **_kwargs): + # The kernel mount completed, but delivery of the helper result was + # interrupted before the lifecycle could persist its identity. + observed["mounted"] = True + raise interrupted + + def mount_at(_path): + return actual if observed["mounted"] else None + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, "_trusted_system_binary", return_value="/bin/mount" + ), mock.patch.object( + ramdisk, "_privileged", side_effect=lambda command, hardware: command + ), mock.patch.object( + ramdisk, "_run", side_effect=run + ), mock.patch.object( + ramdisk, "_mount_at", side_effect=mount_at + ), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ) as validate, mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pathname-only rollback", + ): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + self.assertTrue(observed["mounted"]) + validate.assert_not_called() + unmount.assert_not_called() + retained = snapshots[-1] + self.assertEqual(retained["state"], "error") + self.assertEqual(retained["recovery"]["state"], "attention-required") + self.assertEqual(retained["recovery"]["retained_mounts"], [mount["path"]]) + self.assertEqual(retained["mounts"][0]["ownership"], "pending") + self.assertEqual(retained["mounts"][0]["cleanup"]["state"], "retained") + self.assertNotIn("identity", retained["mounts"][0]) + + @requires_linux_operational + def test_prepare_persists_pending_ownership_before_mount_helper(self): + snapshots = [] + helper_saw_pending = [] + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + def mount_helper(_plan, mount): + pending = snapshots[-1]["mounts"] + helper_saw_pending.append( + len(pending) == 1 + and pending[0]["path"] == mount["path"] + and pending[0]["ownership"] == "pending" + and pending[0]["operation_id"].endswith(":mount:0") + and pending[0]["requested"]["filesystem"] == "tmpfs" + and pending[0]["requested"]["source"] == "tmpfs" + and "identity" not in pending[0] + ) + raise mounts_support._MountHelperCompletedError( + "mount helper failed" + ) + + with mock.patch.object(ramdisk, "_load_manifest", return_value=None), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object(ramdisk, "_save_manifest", side_effect=save), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ), mock.patch.object(ramdisk, "_mount_tmpfs", side_effect=mount_helper), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "mount helper failed"): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + self.assertEqual(helper_saw_pending, [True]) + unmount.assert_not_called() + self.assertEqual(snapshots[-1]["state"], "error") + self.assertEqual(snapshots[-1]["mounts"], []) + self.assertEqual(snapshots[-1]["recovery"]["state"], "clean") + self.assertEqual(snapshots[-1]["recovery"]["retained_mounts"], []) + + @requires_linux_operational + def test_failed_pending_removal_save_retains_last_durable_pending_mount(self): + snapshots = [] + successful = [] + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + mount = plan["mounts"][0] + + def save(manifest): + snapshot = json.loads(json.dumps(manifest)) + snapshots.append(snapshot) + if len(snapshots) == 3: + raise OSError("pending removal write failed") + successful.append(snapshot) + + mount_at = mock.Mock(side_effect=[None, None]) + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, "_mount_at", mount_at + ), mock.patch.object( + ramdisk, + "_mount_tmpfs", + side_effect=mounts_support._MountHelperCompletedError( + "mount helper failed" + ), + ), mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object(ramdisk, "_umount_path") as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pending removal write failed.*pathname-only rollback", + ) as caught: + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + self.assertIsInstance(caught.exception.__cause__, OSError) + self.assertEqual(mount_at.call_count, 2) + unmount.assert_not_called() + self.assertEqual(successful[1]["mounts"][0]["ownership"], "pending") + self.assertEqual(snapshots[2]["mounts"], []) + retained = successful[-1] + self.assertEqual(retained["recovery"]["state"], "attention-required") + self.assertEqual(retained["recovery"]["retained_mounts"], [mount["path"]]) + self.assertEqual(retained["mounts"][0]["ownership"], "pending") + self.assertNotIn("identity", retained["mounts"][0]) + + @requires_linux_operational + def test_completed_mount_failure_retains_observed_mount_without_unmount(self): + snapshots = [] + observed = {} + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + mount = plan["mounts"][0] + actual = { + "mount_id": 45, + "device": "0:45", + "filesystem": "tmpfs", + "source": "tmpfs", + } + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + def mount_at(_path): + return observed.get("identity") + + def mount_helper(_plan, _mount): + observed["identity"] = actual + raise mounts_support._MountHelperCompletedError( + "mount helper failed" + ) + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, "_mount_at", side_effect=mount_at + ), mock.patch.object( + ramdisk, + "_mount_tmpfs", + side_effect=mount_helper, + ), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ) as validate, mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pathname-only rollback", + ): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + validate.assert_not_called() + unmount.assert_not_called() + retained = snapshots[-1] + self.assertEqual(retained["recovery"]["state"], "attention-required") + self.assertEqual(retained["recovery"]["retained_mounts"], [mount["path"]]) + self.assertEqual(retained["mounts"][0]["ownership"], "pending") + self.assertNotIn("identity", retained["mounts"][0]) + + @requires_linux_operational + def test_failed_mount_helper_retains_pending_when_observation_is_inconclusive(self): + snapshots = [] + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + mount = plan["mounts"][0] + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, + "_mount_at", + side_effect=[None, OSError("mount table unavailable")], + ), mock.patch.object( + ramdisk, + "_mount_tmpfs", + side_effect=mounts_support._MountHelperCompletedError( + "mount helper failed" + ), + ), mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object(ramdisk, "_umount_path") as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pathname-only rollback", + ): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + unmount.assert_not_called() + retained = snapshots[-1] + self.assertEqual(retained["recovery"]["state"], "attention-required") + self.assertEqual(retained["recovery"]["retained_mounts"], [mount["path"]]) + self.assertEqual(retained["mounts"][0]["ownership"], "pending") + self.assertNotIn("identity", retained["mounts"][0]) + + @requires_linux_operational + def test_runner_oserror_retains_pending_without_absence_reconciliation(self): + snapshots = [] + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + mount = plan["mounts"][0] + mount_at = mock.Mock(return_value=None) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, "_mount_at", mount_at + ), mock.patch.object( + ramdisk, + "_mount_tmpfs", + side_effect=OSError("mount runner outcome unknown"), + ), mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object(ramdisk, "_umount_path") as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pathname-only rollback", + ) as caught: + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + self.assertIsInstance(caught.exception.__cause__, OSError) + self.assertEqual(mount_at.call_count, 1) + unmount.assert_not_called() + retained = snapshots[-1] + self.assertEqual(retained["recovery"]["state"], "attention-required") + self.assertEqual(retained["recovery"]["retained_mounts"], [mount["path"]]) + self.assertEqual(retained["mounts"][0]["ownership"], "pending") + self.assertNotIn("identity", retained["mounts"][0]) + + @requires_linux_operational + def test_prepare_never_unmounts_identityless_successful_mount_by_path(self): + snapshots = [] + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + with mock.patch.object(ramdisk, "_load_manifest", return_value=None), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object(ramdisk, "_save_manifest", side_effect=save), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ), mock.patch.object(ramdisk, "_mount_tmpfs"), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "could not read its mount identity", + ): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + unmount.assert_not_called() + retained = snapshots[-1] + self.assertEqual(retained["state"], "error") + self.assertEqual(retained["mounts"][0]["ownership"], "pending") + self.assertEqual( + retained["recovery"]["retained_mounts"], + [plan["mounts"][0]["path"]], + ) + + @requires_linux_operational + def test_prepare_promotes_only_the_exact_recorded_mount_identity(self): + snapshots = [] + actual = { + "mount_id": 19, + "device": "0:19", + "filesystem": "tmpfs", + "source": "tmpfs", + } + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + def mount_helper(_plan, mount): + mount["effective_thp"] = "advise" + mount["effective_noswap"] = False + + with mock.patch.object(ramdisk, "_load_manifest", return_value=None), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object(ramdisk, "_save_manifest", side_effect=save), mock.patch.object( + ramdisk, + "_mount_at", + side_effect=[ + None, + actual, + actual, + actual, + actual, + actual, + None, + ], + ), mock.patch.object(ramdisk, "_mount_tmpfs", side_effect=mount_helper), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ), mock.patch.object( + ramdisk, "_populate_mount", side_effect=ramdisk.RamdiskError("copy failed") + ), mock.patch.object(ramdisk, "_mount_table", return_value=[]), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "copy failed"): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + ownership_history = [ + snapshot["mounts"][0]["ownership"] + for snapshot in snapshots + if snapshot["mounts"] + ] + self.assertEqual(ownership_history[:3], ["pending", "identified", "managed"]) + identified = next( + snapshot["mounts"][0] + for snapshot in snapshots + if snapshot["mounts"] + and snapshot["mounts"][0]["ownership"] == "identified" + ) + self.assertEqual(identified["identity"], actual) + self.assertEqual(identified["effective_thp"], "advise") + self.assertFalse(identified["effective_noswap"]) + unmount.assert_called_once_with(plan["mounts"][0]["path"], plan["hardware"]) + + @requires_linux_operational + def test_multi_mount_failure_preflights_all_before_any_unmount(self): + snapshots = [] + observed = {} + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), + hardware=hardware_fixture(nodes=2), + ) + first, second = plan["mounts"] + first_actual = { + "mount_id": 31, + "device": "0:31", + "filesystem": "tmpfs", + "source": "tmpfs", + } + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + def mount_tmpfs(_plan, mount): + if mount["path"] == first["path"]: + observed[mount["path"]] = first_actual + return + raise mounts_support._MountHelperCompletedError( + "second mount failed" + ) + + def mount_at(path): + return observed.get(path) + + def unmount(path, hardware): + self.assertEqual(hardware, plan["hardware"]) + observed.pop(path) + + with mock.patch.object(ramdisk, "_load_manifest", return_value=None), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object(ramdisk, "_save_manifest", side_effect=save), mock.patch.object( + ramdisk, "_mount_at", side_effect=mount_at + ), mock.patch.object(ramdisk, "_mount_tmpfs", side_effect=mount_tmpfs), mock.patch.object( + ramdisk, "_validate_mount", side_effect=lambda mount, ignored: observed[mount["path"]] + ), mock.patch.object(ramdisk, "_mount_table", return_value=[]), mock.patch.object( + ramdisk, "_umount_path", side_effect=unmount + ) as unmount_mock: + with self.assertRaisesRegex(ramdisk.RamdiskError, "second mount failed"): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + unmount_mock.assert_called_once_with(first["path"], plan["hardware"]) + recovery = snapshots[-1]["recovery"] + self.assertEqual( + recovery["released_mounts"], + [first["path"]], + ) + self.assertEqual(recovery["retained_mounts"], []) + by_path = { + record["path"]: record + for record in snapshots[-1]["mounts"] + } + self.assertEqual(by_path[first["path"]]["cleanup"]["state"], "unmounted") + self.assertNotIn(second["path"], by_path) + + @requires_linux_operational + def test_prepare_rollback_refuses_exact_mount_with_nested_child(self): + snapshots = [] + actual = { + "mount_id": 41, + "device": "0:41", + "filesystem": "tmpfs", + "source": "tmpfs", + } + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + path = plan["mounts"][0]["path"] + child = {"path": os.path.join(path, "foreign-child")} + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + + with mock.patch.object(ramdisk, "_load_manifest", return_value=None), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object(ramdisk, "_save_manifest", side_effect=save), mock.patch.object( + ramdisk, "_mount_at", side_effect=[None, actual, actual] + ), mock.patch.object(ramdisk, "_mount_tmpfs"), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ), mock.patch.object( + ramdisk, "_populate_mount", side_effect=ramdisk.RamdiskError("copy failed") + ), mock.patch.object(ramdisk, "_mount_table", return_value=[child]), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "nested mount"): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + + unmount.assert_not_called() + self.assertEqual(snapshots[-1]["recovery"]["retained_mounts"], [path]) + self.assertEqual( + snapshots[-1]["mounts"][0]["cleanup"]["state"], + "retained", + ) + + @requires_linux_operational + def test_prepare_cleanup_runs_even_when_error_manifest_cannot_be_saved(self): + snapshots = [] + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + if len(snapshots) > 1: + raise OSError("state full") + + with mock.patch.object(ramdisk, "_load_manifest", return_value=None), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ), mock.patch.object(ramdisk, "_mount_tmpfs") as mount, mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "rollback/reporting errors"): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), display_plan=False + ) + mount.assert_not_called() + unmount.assert_not_called() + self.assertEqual( + snapshots[-1]["recovery"]["retained_mounts"], + [], + ) + self.assertEqual(snapshots[-1]["recovery"]["state"], "clean") + + def test_failed_identified_save_does_not_authorize_preparation_unmount(self): + snapshots = [] + actual = { + "mount_id": 51, + "device": "0:51", + "filesystem": "tmpfs", + "source": "tmpfs", + } + with ModelFixture() as fixture, mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ): + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + + def save(manifest): + snapshots.append(json.loads(json.dumps(manifest))) + if len(snapshots) >= 3: + raise OSError("durable manifest unavailable") + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=None + ), mock.patch.object( + ramdisk, "build_plan", return_value=plan + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object( + ramdisk, "_mount_at", side_effect=[None, actual, actual] + ), mock.patch.object( + ramdisk, "_mount_tmpfs" + ), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ) as validate, mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, "_busy_mount_references", return_value=[] + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "durable manifest unavailable", + ): + ramdisk.prepare.__wrapped__( + plan_args(fixture.root, yes=True), + display_plan=False, + ) + + validate.assert_not_called() + unmount.assert_not_called() + self.assertEqual(snapshots[-1]["mounts"][0]["ownership"], "identified") + self.assertEqual(snapshots[-1]["mounts"][0]["identity"], actual) + self.assertEqual( + snapshots[-1]["mounts"][0]["cleanup"]["state"], + "retained", + ) + + def test_preparation_rollback_revalidates_identity_before_unmount(self): + path = "/mnt/colibri-test" + expected = { + "mount_id": 61, + "device": "0:61", + "filesystem": "tmpfs", + "source": "tmpfs", + } + replacement = dict(expected, mount_id=62, device="0:62") + manifest = { + "state": "error", + "plan": { + "hardware": hardware_fixture(), + "mounts": [{"path": path, "node": None}], + }, + "mounts": [ + { + "path": path, + "node": None, + "ownership": "identified", + "identity": expected, + } + ], + } + unmount = mock.Mock() + + failures, retained, released = ( + lifecycle_support._rollback_preparation_mounts( + manifest, + mount_at=mock.Mock(side_effect=[expected, replacement]), + mount_table=mock.Mock(return_value=[]), + path_is_below=ramdisk._path_is_below, + busy_mount_references=mock.Mock(return_value=[]), + umount_path=unmount, + validate_mount=mock.Mock(return_value=expected), + ) + ) + + self.assertTrue(failures) + self.assertEqual(retained, {path}) + self.assertEqual(released, set()) + unmount.assert_not_called() + + def test_incomplete_busy_scan_withholds_every_preparation_unmount(self): + paths = [ + "/mnt/colibri-test/node0", + "/mnt/colibri-test/node1", + ] + identities = { + path: { + "mount_id": 71 + index, + "device": "0:%d" % (71 + index), + "filesystem": "tmpfs", + "source": "tmpfs", + } + for index, path in enumerate(paths) + } + hardware = hardware_fixture(nodes=2) + manifest = { + "state": "error", + "plan": { + "hardware": hardware, + "mounts": [ + {"path": path, "node": index} + for index, path in enumerate(paths) + ], + }, + "mounts": [ + { + "path": path, + "node": index, + "ownership": "identified", + "identity": identities[path], + } + for index, path in enumerate(paths) + ], + } + scans = [] + + def busy_references(path, *, hardware=None): + scans.append((path, hardware)) + if path == paths[0]: + raise ramdisk.RamdiskError( + "managed cleanup requires complete /proc visibility" + ) + return [] + + unmount = mock.Mock() + failures, retained, released = ( + lifecycle_support._rollback_preparation_mounts( + manifest, + mount_at=lambda path: identities[path], + mount_table=mock.Mock(return_value=[]), + path_is_below=ramdisk._path_is_below, + busy_mount_references=busy_references, + umount_path=unmount, + validate_mount=( + lambda record, ignored_plan: record["identity"] + ), + ) + ) + + self.assertTrue( + any("complete /proc visibility" in item for item in failures) + ) + self.assertEqual(retained, set(paths)) + self.assertEqual(released, set()) + self.assertTrue(all(item[1] is hardware for item in scans)) + unmount.assert_not_called() + + def test_preparation_rollback_requires_post_unmount_absence(self): + path = "/mnt/colibri-test" + expected = { + "mount_id": 63, + "device": "0:63", + "filesystem": "tmpfs", + "source": "tmpfs", + } + replacement = dict(expected, mount_id=64, device="0:64") + + for after in (expected, replacement): + with self.subTest(after_mount_id=after["mount_id"]): + record = { + "path": path, + "node": None, + "ownership": "identified", + "identity": expected, + } + manifest = { + "state": "error", + "plan": { + "hardware": hardware_fixture(), + "mounts": [{"path": path, "node": None}], + }, + "mounts": [record], + } + unmount = mock.Mock() + failures, retained, released = ( + lifecycle_support._rollback_preparation_mounts( + manifest, + mount_at=mock.Mock( + side_effect=[ + expected, + expected, + expected, + expected, + after, + ] + ), + mount_table=mock.Mock(return_value=[]), + path_is_below=ramdisk._path_is_below, + busy_mount_references=mock.Mock(return_value=[]), + umount_path=unmount, + validate_mount=mock.Mock(return_value=expected), + ) + ) + + self.assertTrue(failures) + self.assertEqual(retained, {path}) + self.assertEqual(released, set()) + unmount.assert_called_once() + self.assertIn("remains or was replaced", failures[0]) + + def test_preparation_rollback_rechecks_identity_after_busy_scan(self): + path = "/mnt/colibri-test" + expected = { + "mount_id": 65, + "device": "0:65", + "filesystem": "tmpfs", + "source": "tmpfs", + } + replacement = dict(expected, mount_id=66, device="0:66") + current = {"identity": expected} + busy_calls = {"count": 0} + + def busy(_path, hardware=None): + busy_calls["count"] += 1 + if busy_calls["count"] == 3: + current["identity"] = replacement + return [] + + record = { + "path": path, + "node": None, + "ownership": "identified", + "identity": expected, + } + unmount = mock.Mock() + failures, retained, released = ( + lifecycle_support._rollback_preparation_mounts( + { + "state": "error", + "plan": { + "hardware": hardware_fixture(), + "mounts": [{"path": path, "node": None}], + }, + "mounts": [record], + }, + mount_at=lambda ignored: current["identity"], + mount_table=lambda: [], + path_is_below=ramdisk._path_is_below, + busy_mount_references=busy, + umount_path=unmount, + validate_mount=lambda ignored, plan: current["identity"], + ) + ) + + self.assertTrue(failures) + self.assertEqual(retained, {path}) + self.assertEqual(released, set()) + self.assertIn("after busy scan", failures[0]) + unmount.assert_not_called() + + def test_copy_uses_atomic_publish_validates_header_and_removes_source_cache(self): + with ModelFixture() as fixture, canonical_temporary_directory() as destination: + source = fixture.root / "model-00001-of-00002.safetensors" + target = Path(destination) / source.name + ramdisk._copy_one( + str(source), str(target), source.stat().st_size, 0, available=lambda: ramdisk.GIB + ) + self.assertEqual(target.stat().st_size, source.stat().st_size) + self.assertEqual(target.stat().st_mode & 0o222, 0) + self.assertFalse(any(".coli-copy-" in item.name for item in Path(destination).iterdir())) + + def test_copy_stream_uses_binary_descriptors_and_preserves_ctrl_z(self): + binary_flag = 1 << 28 + payload = b"safetensors-prefix\x1asafetensors-suffix" + opened = [] + real_open = mounts_support.os.open + + def recording_open(path, flags, mode=0o777): + opened.append((path, flags)) + return real_open(path, flags & ~binary_flag, mode) + + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "source.bin" + destination = Path(directory) / "destination.bin" + source.write_bytes(payload) + with mock.patch.object( + mounts_support.os, + "O_BINARY", + binary_flag, + create=True, + ), mock.patch.object( + mounts_support.os, + "open", + side_effect=recording_open, + ): + mounts_support._copy_stream( + str(source), + str(destination), + len(payload), + ) + + self.assertEqual(destination.read_bytes(), payload) + + self.assertEqual(len(opened), 2) + self.assertTrue(opened[0][1] & binary_flag, "source descriptor must be binary") + self.assertTrue(opened[1][1] & binary_flag, "destination descriptor must be binary") + + def test_mount_validation_rejects_foreign_filesystem(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware_fixture()) + mount = plan["mounts"][0] + foreign = { + "mount_id": 3, + "device": "8:1", + "filesystem": "ext4", + "source": "/dev/sda1", + "options": ["rw", "noatime", "nodev", "nosuid", "noexec"], + "super_options": [], + } + with mock.patch.object(ramdisk, "_mount_at", return_value=foreign): + with self.assertRaisesRegex(ramdisk.RamdiskError, "foreign"): + ramdisk._validate_mount(mount, plan) + + def test_mount_validation_requires_managed_thp_numa_and_safety_options(self): + with ModelFixture() as fixture: + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware_fixture(nodes=2)) + mount = plan["mounts"][0] + actual = { + "mount_id": 9, + "device": "0:42", + "filesystem": "tmpfs", + "source": "tmpfs", + "options": ["rw", "noatime", "nodev", "nosuid", "noexec"], + "super_options": [ + "mode=700", + "noswap", + "huge=within_size", + "mpol=interleave=static:0-1", + ], + } + with mock.patch.object(ramdisk, "_mount_at", return_value=actual): + self.assertEqual(ramdisk._validate_mount(mount, plan)["mount_id"], 9) + actual["super_options"].remove("mpol=interleave=static:0-1") + with mock.patch.object(ramdisk, "_mount_at", return_value=actual): + with self.assertRaisesRegex(ramdisk.RamdiskError, "NUMA policy"): + ramdisk._validate_mount(mount, plan) + + def test_single_selected_node_rejects_sampled_pages_on_another_host_node(self): + with ModelFixture() as fixture, canonical_temporary_directory() as destination: + plan = ramdisk.build_plan( + plan_args(fixture.root, memory_nodes="0"), + hardware=hardware_fixture(nodes=2), + ) + self._stage_fixture_namespace(fixture, destination) + mount = {"path": destination, "node": None} + with mock.patch.object( + ramdisk, + "_sample_numa_allocation", + return_value={"1": 128}, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, "escaped the reviewed NUMA" + ): + ramdisk._validate_namespace(plan, mount) + + def test_interleaved_namespace_accepts_balanced_sample(self): + with ModelFixture() as fixture, canonical_temporary_directory() as destination: + plan = ramdisk.build_plan( + plan_args(fixture.root), + hardware=hardware_fixture(nodes=2), + ) + self._stage_fixture_namespace(fixture, destination) + mount = {"path": destination, "node": None} + with mock.patch.object( + ramdisk, + "_sample_numa_allocation", + return_value={"0": 50, "1": 50}, + ): + self.assertEqual( + ramdisk._validate_namespace(plan, mount), + {"0": 100, "1": 100}, + ) + + def test_interleaved_namespace_reports_imbalanced_sample(self): + with ModelFixture() as fixture, canonical_temporary_directory() as destination: + plan = ramdisk.build_plan( + plan_args(fixture.root), + hardware=hardware_fixture(nodes=2), + ) + self._stage_fixture_namespace(fixture, destination) + mount = {"path": destination, "node": None} + with mock.patch.object( + ramdisk, + "_sample_numa_allocation", + return_value={"0": 60, "1": 40}, + ): + with self.assertRaises(ramdisk.RamdiskError) as raised: + ramdisk._validate_namespace(plan, mount) + message = str(raised.exception) + self.assertIn("0=120, 1=80", message) + self.assertIn("20.0%", message) + self.assertIn("exceeds 15%", message) + self.assertNotIn("%%", message) + + def test_node_local_namespace_uses_single_percent_sign(self): + with ModelFixture() as fixture, canonical_temporary_directory() as destination: + plan = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), + hardware=hardware_fixture(nodes=2), + ) + self._stage_fixture_namespace(fixture, destination) + mount = {"path": destination, "node": 0} + with mock.patch.object( + ramdisk, + "_sample_numa_allocation", + return_value={"0": 90, "1": 10}, + ): + with self.assertRaises(ramdisk.RamdiskError) as raised: + ramdisk._validate_namespace(plan, mount) + self.assertIn("95% local allocation", str(raised.exception)) + self.assertNotIn("%%", str(raised.exception)) + + def test_mount_lookup_rejects_stacked_exact_paths(self): + mounts = [ + {"mount_id": 4, "path": "/mnt/colibri-test"}, + {"mount_id": 9, "path": "/mnt/colibri-test"}, + ] + with mock.patch.object(ramdisk, "_mount_table", return_value=mounts): + with self.assertRaisesRegex(ramdisk.RamdiskError, "stacked mounts"): + ramdisk._mount_at("/mnt/colibri-test") + + def test_filesystem_lookup_rejects_stacked_longest_mountpoint(self): + mounts = [ + { + "mount_id": 41, + "parent_id": 1, + "path": "/durable", + "filesystem": "xfs", + }, + { + "mount_id": 42, + "parent_id": 41, + "path": "/durable", + "filesystem": "tmpfs", + }, + ] + with mock.patch.object(ramdisk, "_mount_table", return_value=mounts): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "ambiguous stacked mounts", + ): + ramdisk._filesystem_for_path("/durable/manifest.json") diff --git a/c/tests/test_ramdisk_packaging.py b/c/tests/test_ramdisk_packaging.py new file mode 100644 index 000000000..21d36a72b --- /dev/null +++ b/c/tests/test_ramdisk_packaging.py @@ -0,0 +1,715 @@ +import importlib.util +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + + +C_DIR = Path(__file__).resolve().parents[1] +ROOT = C_DIR.parent +MAKE = shutil.which("make") +PIP_AVAILABLE = importlib.util.find_spec("pip") is not None +SETUPTOOLS_AVAILABLE = importlib.util.find_spec("setuptools") is not None +ISOLATED_PEP517_REQUESTED = ( + os.environ.get("COLIBRI_TEST_ISOLATED_PEP517") == "1" +) +SUPPORT_MODULES = ( + "version.py", + "resource_plan.py", + "doctor.py", + "autotune.py", + "openai_server.py", + "ramdisk.py", + "ramdisk_ui.py", + "ramdisk_textual.py", +) +SUPPORT_PACKAGE = "ramdisk_support" +REQUIRED_SUPPORT_PACKAGE_MODULES = ( + "__init__.py", + "accelerator.py", + "benchmark.py", + "cli.py", + "common.py", + "curses_ui.py", + "discovery.py", + "linux_ops.py", + "lifecycle.py", + "model.py", + "mounts.py", + "planning.py", + "platform_ops.py", + "presentation.py", + "presets.py", + "processes.py", + "runtime_monitor.py", + "state.py", +) + + +def copy_support(destination, exclude=(), package_exclude=()): + for name in SUPPORT_MODULES: + if name not in exclude: + shutil.copy2(C_DIR / name, destination / name) + if SUPPORT_PACKAGE not in exclude: + package_destination = destination / SUPPORT_PACKAGE + shutil.copytree( + C_DIR / SUPPORT_PACKAGE, + package_destination, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), + ) + for name in package_exclude: + path = package_destination / name + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + + +class RamdiskPackagingTest(unittest.TestCase): + def _run_packaging_command(self, command, *, cwd): + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + env={ + **os.environ, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INPUT": "1", + }, + ) + if result.returncode: + self.fail( + "packaging command failed (%s):\nstdout:\n%s\nstderr:\n%s" + % ( + " ".join(map(str, command)), + result.stdout, + result.stderr, + ) + ) + return result + + def test_spdx_license_declares_a_compatible_setuptools_minimum(self): + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + minimum_match = re.search( + r'setuptools>=(\d+(?:\.\d+)+)', + pyproject, + ) + self.assertIsNotNone(minimum_match, pyproject) + self.assertGreaterEqual( + tuple(map(int, minimum_match.group(1).split("."))), + (77, 0, 1), + "SPDX project metadata requires setuptools 77.0.1 or newer", + ) + + def _assert_wheel_contains_runnable_control_plane( + self, + *, + isolated_build, + ): + with tempfile.TemporaryDirectory() as stage: + stage_root = Path(stage) + source = stage_root / "source" + wheel_dir = stage_root / "wheel" + installed = stage_root / "installed" + runtime = source / "c" + source.mkdir() + wheel_dir.mkdir() + runtime.mkdir() + + for name in ("pyproject.toml", "README.md", "LICENSE"): + shutil.copy2(ROOT / name, source / name) + shutil.copytree(ROOT / "colibri", source / "colibri") + for name in ( + "__init__.py", + "coli", + "requirements-tui.txt", + "download_fp8.py", + ): + shutil.copy2(C_DIR / name, runtime / name) + copy_support(runtime) + shutil.copytree(C_DIR / "tools", runtime / "tools") + + if PIP_AVAILABLE: + command = [ + sys.executable, + "-m", + "pip", + "--isolated", + "wheel", + "--use-pep517", + ] + if not isolated_build: + command.extend( + ["--no-build-isolation", "--no-index"] + ) + command.extend( + [ + "--verbose", + "--no-deps", + "--wheel-dir", + str(wheel_dir), + str(source), + ] + ) + self._run_packaging_command(command, cwd=stage_root) + else: + if isolated_build: + self.fail( + "default isolated PEP 517 verification requires pip" + ) + self._run_packaging_command( + [ + sys.executable, + "-c", + ( + "import setuptools.build_meta as backend, sys; " + "print(backend.build_wheel(sys.argv[1]))" + ), + str(wheel_dir), + ], + cwd=source, + ) + + wheels = tuple(wheel_dir.glob("*.whl")) + self.assertEqual(len(wheels), 1) + + if PIP_AVAILABLE: + self._run_packaging_command( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-index", + "--no-deps", + "--target", + str(installed), + str(wheels[0]), + ], + cwd=stage_root, + ) + + with zipfile.ZipFile(wheels[0]) as wheel: + members = set(wheel.namelist()) + for name in ("coli", "requirements-tui.txt", *SUPPORT_MODULES): + self.assertIn(f"c/{name}", members) + for name in REQUIRED_SUPPORT_PACKAGE_MODULES: + self.assertIn(f"c/{SUPPORT_PACKAGE}/{name}", members) + if not PIP_AVAILABLE: + wheel.extractall(installed) + + smoke = self._run_packaging_command( + [ + sys.executable, + "-c", + ( + "import sys; " + "sys.path.insert(0, sys.argv[1]); " + "from colibri.cli import main; " + "sys.argv = ['coli', 'ramdisk', '--help']; " + "main()" + ), + str(installed), + ], + cwd=stage_root, + ) + + self.assertIn("interleaved = one shared model copy", smoke.stdout) + self.assertIn("prepare", smoke.stdout) + + @unittest.skipUnless( + SETUPTOOLS_AVAILABLE, + "wheel backend unavailable; source/install packaging contracts still run", + ) + def test_wheel_contains_runnable_ramdisk_control_plane(self): + """The selected ambient backend builds without package-index access.""" + self._assert_wheel_contains_runnable_control_plane( + isolated_build=False, + ) + + @unittest.skipUnless( + PIP_AVAILABLE and ISOLATED_PEP517_REQUESTED, + "set COLIBRI_TEST_ISOLATED_PEP517=1 for the networked build gate", + ) + def test_default_isolated_pep517_wheel_installs_and_runs(self): + """Default PEP 517 isolation resolves, builds, installs, and runs.""" + self._assert_wheel_contains_runnable_control_plane( + isolated_build=True, + ) + + @unittest.skipUnless(MAKE, "make is required") + def test_staged_install_dry_run_includes_support_module(self): + with tempfile.TemporaryDirectory() as stage: + result = subprocess.run( + [ + MAKE, + "--no-print-directory", + "-n", + "install", + f"DESTDIR={stage}", + "PREFIX=/usr", + ], + cwd=C_DIR, + text=True, + capture_output=True, + check=True, + ) + + self.assertIn("ramdisk.py", result.stdout) + self.assertIn("ramdisk_ui.py", result.stdout) + self.assertIn("ramdisk_textual.py", result.stdout) + self.assertIn('rm -rf "', result.stdout) + self.assertIn("/ramdisk_support", result.stdout) + self.assertIn("install -d -m 755", result.stdout) + self.assertIn("install -m 644 ramdisk_support/*.py", result.stdout) + self.assertNotIn("cp -R ramdisk_support", result.stdout) + self.assertIn("requirements-tui.txt", result.stdout) + self.assertIn("version.py", result.stdout) + self.assertIn("autotune.py", result.stdout) + self.assertIn("coli.libexec", result.stdout) + self.assertIn("/usr/libexec/colibri/", result.stdout) + + def test_staged_installed_cli_can_load_ramdisk_support_modules(self): + with tempfile.TemporaryDirectory() as stage: + prefix = Path(stage) / "usr" + bin_dir = prefix / "bin" + libexec_dir = prefix / "libexec" / "colibri" + bin_dir.mkdir(parents=True) + libexec_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", bin_dir / "coli") + copy_support(libexec_dir) + + result = subprocess.run( + [sys.executable, str(bin_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=True, + ) + + self.assertIn("interleaved = one shared model copy", result.stdout) + self.assertIn("prepare", result.stdout) + + def test_custom_install_layout_uses_its_recorded_support_directory(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + bin_dir = root / "custom-bin" + support_dir = root / "unusual" / "python-and-engine" + bin_dir.mkdir() + support_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", bin_dir / "coli") + (bin_dir / "coli.libexec").write_text( + str(support_dir) + "\n", encoding="utf-8" + ) + copy_support(support_dir) + + result = subprocess.run( + [sys.executable, str(bin_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=True, + ) + + self.assertIn("interleaved = one shared model copy", result.stdout) + + def test_recorded_install_layout_wins_over_stale_colocated_bundle(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + bin_dir = root / "bin" + support_dir = root / "support" + bin_dir.mkdir() + support_dir.mkdir() + shutil.copy2(C_DIR / "coli", bin_dir / "coli") + (bin_dir / "coli.libexec").write_text( + str(support_dir) + "\n", encoding="utf-8" + ) + copy_support(bin_dir) + copy_support(support_dir) + (bin_dir / "ramdisk_ui.py").write_text( + 'raise RuntimeError("stale colocated UI loaded")\n', encoding="utf-8" + ) + + result = subprocess.run( + [sys.executable, str(bin_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=True, + ) + + self.assertIn("interleaved = one shared model copy", result.stdout) + + def test_colocated_release_modules_win_over_stale_sibling_install(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + release_dir = root / "release" + stale_dir = root / "libexec" / "colibri" + release_dir.mkdir() + stale_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", release_dir / "coli") + copy_support(release_dir) + copy_support(stale_dir) + (stale_dir / "version.py").write_text( + '__version__ = "stale"\n', encoding="utf-8" + ) + (stale_dir / "ramdisk_ui.py").write_text( + 'raise RuntimeError("stale UI loaded")\n', encoding="utf-8" + ) + (stale_dir / SUPPORT_PACKAGE / "planning.py").write_text( + 'raise RuntimeError("stale planning loaded")\n', + encoding="utf-8", + ) + + result = subprocess.run( + [sys.executable, str(release_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=True, + ) + + self.assertIn("interleaved = one shared model copy", result.stdout) + + def test_symlinked_support_package_cannot_mix_bundle_generations(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + release_dir = root / "release" + sibling_dir = root / "libexec" / "colibri" + release_dir.mkdir() + sibling_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", release_dir / "coli") + copy_support(release_dir, exclude=(SUPPORT_PACKAGE,)) + copy_support(sibling_dir) + try: + os.symlink( + sibling_dir / SUPPORT_PACKAGE, + release_dir / SUPPORT_PACKAGE, + target_is_directory=True, + ) + except (NotImplementedError, OSError) as exc: + self.skipTest("directory symlinks are unavailable: %s" % exc) + + result = subprocess.run( + [sys.executable, str(release_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsafe colocated support bundle", result.stderr) + self.assertIn("ramdisk_support/", result.stderr) + + def test_installed_support_wins_over_stale_engine_override_bundle(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + bin_dir = root / "usr" / "bin" + support_dir = root / "usr" / "libexec" / "colibri" + stale_engine_dir = root / "stale-engine-bundle" + bin_dir.mkdir(parents=True) + support_dir.mkdir(parents=True) + stale_engine_dir.mkdir() + shutil.copy2(C_DIR / "coli", bin_dir / "coli") + copy_support(support_dir) + copy_support(stale_engine_dir) + (stale_engine_dir / "version.py").write_text( + '__version__ = "stale"\n', encoding="utf-8" + ) + (stale_engine_dir / "ramdisk_ui.py").write_text( + 'raise RuntimeError("stale engine UI loaded")\n', encoding="utf-8" + ) + environment = dict(os.environ) + environment["COLI_ENGINE"] = str(stale_engine_dir / "colibri") + + result = subprocess.run( + [sys.executable, str(bin_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=True, + env=environment, + ) + + self.assertIn("interleaved = one shared model copy", result.stdout) + + def test_partial_flat_bundle_cannot_mix_with_complete_sibling_install(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + release_dir = root / "release" + sibling_dir = root / "libexec" / "colibri" + release_dir.mkdir() + sibling_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", release_dir / "coli") + copy_support(release_dir, exclude=("ramdisk_ui.py",)) + copy_support(sibling_dir) + + result = subprocess.run( + [sys.executable, str(release_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("incomplete colocated support bundle", result.stderr) + self.assertIn("ramdisk_ui.py", result.stderr) + + def test_missing_support_package_cannot_mix_with_complete_sibling_install(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + release_dir = root / "release" + sibling_dir = root / "libexec" / "colibri" + release_dir.mkdir() + sibling_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", release_dir / "coli") + copy_support(release_dir, exclude=(SUPPORT_PACKAGE,)) + copy_support(sibling_dir) + + result = subprocess.run( + [sys.executable, str(release_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("incomplete colocated support bundle", result.stderr) + self.assertIn("ramdisk_support/", result.stderr) + + def test_partial_support_package_cannot_mix_with_complete_sibling_install(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + release_dir = root / "release" + sibling_dir = root / "libexec" / "colibri" + release_dir.mkdir() + sibling_dir.mkdir(parents=True) + shutil.copy2(C_DIR / "coli", release_dir / "coli") + copy_support(release_dir, package_exclude=("planning.py",)) + copy_support(sibling_dir) + + result = subprocess.run( + [sys.executable, str(release_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("incomplete colocated support bundle", result.stderr) + self.assertIn("ramdisk_support/planning.py", result.stderr) + + def test_partial_launcher_directory_cannot_shadow_complete_engine_bundle(self): + with tempfile.TemporaryDirectory() as stage: + root = Path(stage) + bin_dir = root / "bin" + engine_dir = root / "engine-bundle" + bin_dir.mkdir() + engine_dir.mkdir() + shutil.copy2(C_DIR / "coli", bin_dir / "coli") + shutil.copy2(C_DIR / "ramdisk_ui.py", bin_dir / "ramdisk_ui.py") + copy_support(engine_dir) + environment = dict(os.environ) + environment["COLI_ENGINE"] = str(engine_dir / "colibri") + + result = subprocess.run( + [sys.executable, str(bin_dir / "coli"), "ramdisk", "--help"], + text=True, + capture_output=True, + check=False, + env=environment, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("incomplete colocated support bundle", result.stderr) + self.assertIn("resource_plan.py", result.stderr) + + def test_complete_support_bundle_tolerates_missing_version_metadata(self): + with tempfile.TemporaryDirectory() as stage: + release_dir = Path(stage) / "release" + release_dir.mkdir() + shutil.copy2(C_DIR / "coli", release_dir / "coli") + copy_support(release_dir, exclude=("version.py",)) + + result = subprocess.run( + [sys.executable, str(release_dir / "coli"), "--version"], + text=True, + capture_output=True, + check=True, + ) + + self.assertEqual(result.stdout.strip(), "colibri unknown") + + def test_nix_package_uses_current_engine_and_runs_python_tests(self): + flake = (ROOT / "flake.nix").read_text(encoding="utf-8") + self.assertIn("c/ramdisk.py", flake) + self.assertIn("c/ramdisk_ui.py", flake) + self.assertIn("c/ramdisk_textual.py", flake) + self.assertIn("c/autotune.py", flake) + self.assertIn("install -d -m 755 $out/lib/colibri/ramdisk_support", flake) + self.assertIn( + "install -m 644 c/ramdisk_support/*.py " + "$out/lib/colibri/ramdisk_support/", + flake, + ) + self.assertNotIn("cp -R c/ramdisk_support", flake) + self.assertIn("textual", flake) + self.assertIn('make -C c colibri ARCH="$ARCH"', flake) + self.assertIn("cp c/colibri", flake) + self.assertIn('COLI_ENGINE "$out/lib/colibri/colibri"', flake) + self.assertIn('program = "${colibri}/bin/colibri";', flake) + self.assertIn("nativeCheckInputs = [pythonEnv];", flake) + self.assertIn( + "pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux", + flake, + ) + self.assertIn("pkgs.psmisc", flake) + self.assertIn( + "--prefix PATH : ${pkgs.lib.makeBinPath [pkgs.psmisc pkgs.util-linux]}", + flake, + ) + self.assertIn("export PYTHONDONTWRITEBYTECODE=1", flake) + self.assertIn("make test\n", flake) + self.assertNotIn("make test-c\n", flake) + + def test_release_builds_current_engine_and_includes_ramdisk_module(self): + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text( + encoding="utf-8" + ) + self.assertIn("for t in colibri inkling kimi_k3 olmoe; do", workflow) + self.assertIn("make $t ${{ matrix.make_args }}", workflow) + self.assertIn("cp c/colibri${{ matrix.ext }}", workflow) + self.assertIn("cp c/ramdisk.py dist/", workflow) + self.assertIn("cp c/ramdisk_ui.py dist/", workflow) + self.assertIn("cp c/ramdisk_textual.py dist/", workflow) + self.assertIn("cp c/autotune.py dist/", workflow) + self.assertIn("mkdir -p dist/ramdisk_support", workflow) + self.assertIn( + "cp c/ramdisk_support/*.py dist/ramdisk_support/", + workflow, + ) + self.assertNotIn("cp -R c/ramdisk_support", workflow) + self.assertIn("cp c/requirements-tui.txt dist/", workflow) + self.assertIn("python3 coli ramdisk --help", workflow) + self.assertIn( + "python3 -m compileall -q ramdisk.py ramdisk_ui.py " + "ramdisk_textual.py ramdisk_support", + workflow, + ) + self.assertIn( + "python3 -m pip install --disable-pip-version-check " + "-r requirements-tui.txt", + workflow, + ) + self.assertIn("import ramdisk_textual", workflow) + self.assertIn("pkgutil.walk_packages", workflow) + self.assertIn("packaged RAM-disk support contains generated artifacts", workflow) + self.assertIn('grep -Fq "interleaved = one shared model copy"', workflow) + self.assertNotIn("python3 coli info 2>&1 || true", workflow) + self.assertNotIn("make glm ${{ matrix.make_args }}", workflow) + + def test_setuptools_discovers_ramdisk_support_recursively(self): + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + self.assertIn('"c.ramdisk_support*"', pyproject) + + def test_launcher_bundle_contract_lists_every_support_module(self): + discovered = { + path.name + for path in (C_DIR / SUPPORT_PACKAGE).glob("*.py") + } + self.assertEqual(discovered, set(REQUIRED_SUPPORT_PACKAGE_MODULES)) + nested = [ + path.relative_to(C_DIR / SUPPORT_PACKAGE).as_posix() + for path in (C_DIR / SUPPORT_PACKAGE).rglob("*.py") + if path.parent != C_DIR / SUPPORT_PACKAGE + ] + self.assertEqual( + nested, + [], + "release/install packaging intentionally requires a flat support package", + ) + + def test_clean_removes_makefile_outputs_and_legacy_engine_names(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifacts = [ + root / name + for name in ( + ".build-config", + "colibri", "colibri.exe", "glm", "glm.exe", + "olmoe", "olmoe.exe", "inkling", "inkling.exe", + "kimi_k3", "kimi_k3.exe", "iobench", "iobench.exe", + "backend_cuda.o", "backend_cuda_ink.o", "backend_loader.o", + "backend_vulkan.o", "backend_cuda_test", "backend_cuda_test.exe", + "ragged_attention_test", "ragged_attention_test.exe", + "backend_cuda_bench", "backend_cuda_bench.exe", "backend_metal.o", + "backend_metal_test", "backend_metal_test.exe", + "gemm_largebatch_test", "gemm_largebatch_test.exe", "coli_cuda.dll", + "coli_cuda.lib", "coli_cuda.exp", "tools/libiq3.so", + "tools/libiq3.dylib", "tools/iq3.dll", "tools/librans_c.so", + "tools/librans_c.dylib", "tools/rans_c.dll", + "shaders/qmatmul_gate_up.spv", "shaders/attention_absorb.spv", + "shaders/rmsnorm.spv", + ) + ] + tests_dir = root / "tests" + tests_dir.mkdir() + cache_dirs = [ + root / "__pycache__", + root / "ramdisk_support" / "__pycache__", + tests_dir / "__pycache__", + ] + for cache_dir in cache_dirs: + cache_dir.mkdir(parents=True) + (cache_dir / "stale.pyc").write_bytes(b"bytecode") + makefile = (C_DIR / "Makefile").read_text(encoding="utf-8") + test_basenames = re.findall( + r"^tests/(test_[a-z0-9_]+)\$\(EXE\):", makefile, re.MULTILINE + ) + test_basenames.extend( + ( + "bench_topp", + "bench_dsa_select", + "bench_idot", + "bench_mla_simd", + "fuzz_rans", + ) + ) + artifacts.extend( + tests_dir / f"{name}{suffix}" + for name in test_basenames + for suffix in ("", ".exe") + ) + for artifact in artifacts: + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(b"binary") + + subprocess.run( + [sys.executable, str(C_DIR / "tools" / "clean.py")], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + self.assertTrue(all(not artifact.exists() for artifact in artifacts)) + self.assertTrue(all(not cache_dir.exists() for cache_dir in cache_dirs)) + + def test_environment_reference_documents_rammap_contract(self): + reference = (ROOT / "docs" / "ENVIRONMENT.md").read_text(encoding="utf-8") + for variable in ( + "COLI_WEIGHTS_DIR", + "COLI_STATE_DIR", + "COLI_RAMMAP", + "COLI_RAM_PREFAULT", + "COLI_NUMA_NODES", + "COLI_CPU_AFFINITY", + ): + self.assertIn(f"`{variable}`", reference) + self.assertIn("Incompatible with `COLI_MMAP=1`", reference) + self.assertIn("volatile weight mounts never hold runtime state", reference) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_planning_module.py b/c/tests/test_ramdisk_planning_module.py new file mode 100644 index 000000000..a291a9e1a --- /dev/null +++ b/c/tests/test_ramdisk_planning_module.py @@ -0,0 +1,324 @@ +"""Direct contracts for the extracted RAM-disk planning module.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import planning + + +class PlanningModuleTest(unittest.TestCase): + def _services(self, root): + platform = argparse.Namespace(is_linux=True) + return { + "discover_hardware": ramdisk.discover_hardware, + "scan_model": ramdisk.scan_model, + "load_profile": ramdisk._load_profile, + "select_partial": ramdisk._select_partial, + "runtime_reserve": ramdisk._runtime_reserve, + "build_placement": ramdisk._build_placement, + "reusable_empty_mountpoint": ramdisk._reusable_empty_mountpoint, + "filesystem_for_path": lambda path: "ext4", + "state_root": lambda: str(root / "state"), + "manifest_path": lambda: str(root / "state" / "manifest.json"), + "benchmarks_path": lambda: str(root / "state" / "benchmarks.json"), + "current_euid": lambda: 0, + "get_platform_ops": lambda: platform, + } + + def test_facade_resolves_every_planning_dependency_at_call_time(self): + args = mock.sentinel.args + hardware = mock.sentinel.hardware + model = mock.sentinel.model + dependencies = { + "discover_hardware": mock.sentinel.discover_hardware, + "scan_model": mock.sentinel.scan_model, + "_load_profile": mock.sentinel.load_profile, + "_select_partial": mock.sentinel.select_partial, + "_runtime_reserve": mock.sentinel.runtime_reserve, + "_build_placement": mock.sentinel.build_placement, + "_reusable_empty_mountpoint": mock.sentinel.reusable_empty_mountpoint, + "_filesystem_for_path": mock.sentinel.filesystem_for_path, + "_state_root": mock.sentinel.state_root, + "_manifest_path": mock.sentinel.manifest_path, + "_benchmarks_path": mock.sentinel.benchmarks_path, + "current_euid": mock.sentinel.current_euid, + "get_platform_ops": mock.sentinel.get_platform_ops, + } + with mock.patch.multiple(ramdisk, **dependencies), mock.patch.object( + ramdisk, + "_planning_build_plan", + return_value=mock.sentinel.plan, + ) as implementation: + result = ramdisk.build_plan( + args, + hardware=hardware, + model=model, + ) + + self.assertIs(result, mock.sentinel.plan) + implementation.assert_called_once_with( + args, + hardware=hardware, + model=model, + discover_hardware=mock.sentinel.discover_hardware, + scan_model=mock.sentinel.scan_model, + load_profile=mock.sentinel.load_profile, + select_partial=mock.sentinel.select_partial, + runtime_reserve=mock.sentinel.runtime_reserve, + build_placement=mock.sentinel.build_placement, + reusable_empty_mountpoint=mock.sentinel.reusable_empty_mountpoint, + filesystem_for_path=mock.sentinel.filesystem_for_path, + state_root=mock.sentinel.state_root, + manifest_path=mock.sentinel.manifest_path, + benchmarks_path=mock.sentinel.benchmarks_path, + current_euid=mock.sentinel.current_euid, + get_platform_ops=mock.sentinel.get_platform_ops, + ) + + def test_direct_builder_matches_facade_with_the_same_services(self): + with ModelFixture() as fixture: + hardware = hardware_fixture(nodes=2) + model = ramdisk.scan_model(str(fixture.root)) + args = plan_args(fixture.root, memory_nodes="0-1", cpu_list="0-3") + services = self._services(fixture.root) + facade_patches = { + "_load_profile": services["load_profile"], + "_select_partial": services["select_partial"], + "_runtime_reserve": services["runtime_reserve"], + "_build_placement": services["build_placement"], + "_reusable_empty_mountpoint": services[ + "reusable_empty_mountpoint" + ], + "_filesystem_for_path": services["filesystem_for_path"], + "_state_root": services["state_root"], + "_manifest_path": services["manifest_path"], + "_benchmarks_path": services["benchmarks_path"], + "current_euid": services["current_euid"], + "get_platform_ops": services["get_platform_ops"], + } + with mock.patch.multiple(ramdisk, **facade_patches): + expected = ramdisk.build_plan( + args, + hardware=hardware, + model=model, + ) + actual = planning.build_plan( + args, + hardware=hardware, + model=model, + **services, + ) + + expected.pop("created_at") + actual.pop("created_at") + self.assertEqual(actual, expected) + + def test_injected_discovery_and_model_callbacks_are_resolved_at_call_time(self): + with ModelFixture() as fixture: + hardware = hardware_fixture() + model = ramdisk.scan_model(str(fixture.root)) + calls = [] + services = self._services(fixture.root) + + def discover(): + calls.append("discover") + return hardware + + def scan(path): + calls.append(("scan", path)) + return model + + services.update( + discover_hardware=discover, + scan_model=scan, + ) + plan = planning.build_plan( + plan_args(fixture.root), + **services, + ) + + self.assertEqual(calls, ["discover", ("scan", str(fixture.root))]) + self.assertEqual(plan["model"]["fingerprint"], model["fingerprint"]) + self.assertNotIn("coli ramdisk is supported only on Linux", plan["blockers"]) + + def test_unsupported_hardware_returns_a_plan_without_platform_probing(self): + with ModelFixture() as fixture: + hardware = hardware_fixture() + hardware["linux"] = False + hardware["tmpfs"] = { + "supported": False, + "noswap_supported": False, + } + model = ramdisk.scan_model(str(fixture.root)) + services = self._services(fixture.root) + + def unexpected_platform_probe(): + raise AssertionError( + "unsupported planning must not select Linux platform operations" + ) + + def unexpected_filesystem_probe(path): + raise AssertionError( + "unsupported planning must not inspect Linux mount tables: %s" + % path + ) + + services.update( + get_platform_ops=unexpected_platform_probe, + filesystem_for_path=unexpected_filesystem_probe, + ) + plan = planning.build_plan( + plan_args(fixture.root), + hardware=hardware, + model=model, + **services, + ) + + self.assertEqual(plan["schema"], ramdisk.PLAN_SCHEMA) + self.assertEqual(plan["version"], ramdisk.MANIFEST_VERSION) + self.assertIn( + "coli ramdisk is supported only on Linux", + plan["blockers"], + ) + + def test_runtime_topology_prefers_the_recorded_placement_contract(self): + plan = { + "placement": { + "engine_cpu_sets": [ + { + "node": None, + "physical_cores": 0, + "cpu_list": "8-9", + "cpus": [1], + }, + { + "node": 2, + "physical_cores": 3, + "cpu_list": "", + "cpus": [4, 5, 7], + }, + ], + "memory_nodes": [2], + }, + "hardware": { + "physical_cores": 32, + "effective_cpus": [0, 1], + "online_nodes": [0, 1], + "nodes": [], + }, + } + + self.assertEqual(planning._node_core_count(plan), 1) + self.assertEqual(planning._engine_cpu_list(plan), "8-9") + self.assertEqual(planning._memory_node_list(plan), "2") + self.assertEqual(planning._node_core_count(plan, node=2), 3) + self.assertEqual( + planning._engine_cpu_list(plan, node=2), + "4-5,7", + ) + + def test_runtime_topology_falls_back_to_recorded_hardware(self): + plan = { + "placement": {}, + "hardware": { + "physical_cores": 6, + "effective_cpus": [0, 2, 3], + "online_nodes": [0, 2], + "nodes": [ + { + "id": 0, + "physical_cores": 1, + "cpus": [0], + }, + { + "id": 2, + "physical_cores": 2, + "cpus": [4, 5], + }, + ], + }, + } + + self.assertEqual(planning._node_core_count(plan), 6) + self.assertEqual(planning._engine_cpu_list(plan), "0,2-3") + self.assertEqual(planning._memory_node_list(plan), "0,2") + self.assertEqual(planning._node_core_count(plan, node=2), 2) + self.assertEqual(planning._engine_cpu_list(plan, node=2), "4-5") + self.assertEqual(planning._memory_node_list(plan, node=2), "2") + + plan["hardware"]["effective_cpus"] = [] + self.assertEqual(planning._engine_cpu_list(plan), "0,4-5") + + def test_runtime_topology_reports_missing_hardware_contracts(self): + plan = { + "placement": {}, + "hardware": { + "physical_cores": 4, + "effective_cpus": [], + "online_nodes": [], + "nodes": [], + }, + } + + with self.assertRaisesRegex( + planning.RamdiskError, + "NUMA node 7 is absent", + ): + planning._node_core_count(plan, node=7) + with self.assertRaisesRegex( + planning.RamdiskError, + "managed engine CPU mask is empty", + ): + planning._engine_cpu_list(plan) + with self.assertRaisesRegex( + planning.RamdiskError, + "managed memory-node mask is empty", + ): + planning._memory_node_list(plan) + + def test_shared_single_node_keeps_engine_numa_policy_enabled(self): + plan = { + "placement": {"memory_nodes": [0]}, + "hardware": {"online_nodes": [0]}, + } + + self.assertTrue(planning._managed_numa_enabled(plan)) + self.assertFalse(planning._managed_numa_enabled(plan, node=0)) + + def test_importing_planning_does_not_load_host_or_lifecycle_modules(self): + code = """ +import json +import sys +import ramdisk_support.planning + +forbidden = [ + "ramdisk_support.benchmark", + "ramdisk_support.cli", + "ramdisk_support.discovery", + "ramdisk_support.lifecycle", + "ramdisk_support.linux_ops", + "ramdisk_support.mounts", + "ramdisk_support.platform_ops", + "ramdisk_support.presentation", + "ramdisk_support.processes", + "ramdisk_support.state", +] +print(json.dumps([name for name in forbidden if name in sys.modules])) +""" + result = subprocess.run( + [sys.executable, "-c", code], + cwd=C_DIR, + env=dict(os.environ, PYTHONPATH=str(C_DIR)), + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_platform.py b/c/tests/test_ramdisk_platform.py new file mode 100644 index 000000000..b4c25d387 --- /dev/null +++ b/c/tests/test_ramdisk_platform.py @@ -0,0 +1,3822 @@ +import argparse +import errno +import io +import json +import os +import signal +import stat +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + + +C_DIR = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(C_DIR)) + +import ramdisk # noqa: E402 +from ramdisk_support import discovery, linux_ops, processes # noqa: E402 +from ramdisk_support.platform_ops import ( # noqa: E402 + UNSUPPORTED_PLATFORM_REASON, + UnsupportedPlatformOps, + get_platform_ops, +) + +if __package__: + from .platform_test_support import ( # noqa: E402 + PLATFORM_SKIP_INVENTORY, + assert_platform_skip_inventory, + requires_linux_pidfd, + ) +else: + from platform_test_support import ( # noqa: E402 + PLATFORM_SKIP_INVENTORY, + assert_platform_skip_inventory, + requires_linux_pidfd, + ) + + +def _proc_stat_record(pid, pgid, session, starttime, state="S"): + fields = [state, "1", str(pgid), str(session)] + ["0"] * 15 + fields[17] = "1" + fields.append(str(starttime)) + return "%d (managed worker) %s\n" % (pid, " ".join(fields)) + + +class RamdiskPlatformTest(unittest.TestCase): + FRESH_PROCESS_CONTRACT = r""" +import argparse +import builtins +import contextlib +import io +import json +import os +import sys +from pathlib import Path + +support_dir = sys.argv[1] +target_platform = sys.argv[2] +temporary_root = Path(sys.argv[3]).resolve() + +missing_by_platform = { + "win32": ( + "getuid", + "geteuid", + "getgid", + "getgroups", + "getpgid", + "killpg", + "statvfs", + "sched_getaffinity", + ), + "darwin": ("sched_getaffinity",), +} +removed = [] +for name in missing_by_platform[target_platform]: + if hasattr(os, name): + delattr(os, name) + removed.append(name) +unavailable = [ + name for name in missing_by_platform[target_platform] if not hasattr(os, name) +] +assert unavailable == list(missing_by_platform[target_platform]), unavailable + +real_open = builtins.open + +def guarded_open(path, *args, **kwargs): + try: + spelling = os.fsdecode(os.fspath(path)).replace("\\", "/") + except TypeError: + spelling = "" + if spelling in ("/proc", "/sys") or spelling.startswith(("/proc/", "/sys/")): + raise AssertionError("unsupported platform probed Linux path " + spelling) + return real_open(path, *args, **kwargs) + +builtins.open = guarded_open +sys.modules["fcntl"] = None +sys.path.insert(0, support_dir) + +import ramdisk +from ramdisk_support.platform_ops import ( + UNSUPPORTED_PLATFORM_REASON, + UnsupportedPlatformOps, +) + +eager_optional_modules = sorted( + name + for name in ( + "ramdisk_support.benchmark", + "ramdisk_support.curses_ui", + "ramdisk_support.runtime_monitor", + "ramdisk_ui", + "ramdisk_textual", + "ssl", + "urllib.request", + ) + if name in sys.modules +) +assert eager_optional_modules == [], eager_optional_modules + +ops = UnsupportedPlatformOps(target_platform) +ramdisk.get_platform_ops = lambda platform_name=None: ops +process_errors = {} +for name, operation in ( + ("identity", lambda: ramdisk._proc_identity(1, ops=ops)), + ("group_alive", lambda: ramdisk._group_alive(1, ops=ops)), + ( + "busy_mounts", + lambda: ramdisk._busy_mount_references("/mnt/colibri", ops=ops), + ), +): + try: + operation() + except ramdisk.RamdiskError as exc: + process_errors[name] = str(exc) + else: + raise AssertionError( + "%s unexpectedly used an unsupported process capability" % name + ) +assert process_errors == { + "busy_mounts": UNSUPPORTED_PLATFORM_REASON, + "identity": UNSUPPORTED_PLATFORM_REASON, + "group_alive": UNSUPPORTED_PLATFORM_REASON, +}, process_errors + +parser = argparse.ArgumentParser(prog="coli ramdisk") +ramdisk.configure_parser(parser) +help_text = parser.format_help() +assert "ACTION" in help_text +status_args = parser.parse_args(["status", "--json"]) +stop_args = parser.parse_args(["stop"]) + +os.environ["XDG_STATE_HOME"] = str(temporary_root / "state") +os.environ["COLI_RAMDISK_MANIFEST"] = str(temporary_root / "manifest.json") +report = ramdisk.status() +assert report["present"] is False, report + +def forbidden(label): + def fail(*args, **kwargs): + del args, kwargs + raise AssertionError("unsupported mutator reached " + label) + return fail + +forbidden_names = ( + "_lifecycle_lock", + "_load_manifest", + "_save_manifest", + "_trusted_system_binary", + "_confirm", + "_mount_tmpfs", + "_umount_path", + "_resolve_engine_path", +) +originals = {name: getattr(ramdisk, name) for name in forbidden_names} +for name in forbidden_names: + setattr(ramdisk, name, forbidden(name)) + +mutating_errors = {} +empty_args = argparse.Namespace() +for name, operation in ( + ("prepare", lambda: ramdisk.prepare(empty_args)), + ("start", lambda: ramdisk.start(empty_args)), + ("stop", lambda: ramdisk.stop()), + ("benchmark", lambda: ramdisk.benchmark(empty_args)), + ("destroy", lambda: ramdisk.destroy(empty_args)), +): + try: + operation() + except ramdisk.RamdiskError as exc: + mutating_errors[name] = str(exc) + else: + raise AssertionError(name + " unexpectedly ran on " + target_platform) +assert mutating_errors == { + "benchmark": UNSUPPORTED_PLATFORM_REASON, + "destroy": UNSUPPORTED_PLATFORM_REASON, + "prepare": UNSUPPORTED_PLATFORM_REASON, + "start": UNSUPPORTED_PLATFORM_REASON, + "stop": UNSUPPORTED_PLATFORM_REASON, +}, mutating_errors +for name, original in originals.items(): + setattr(ramdisk, name, original) +assert not (temporary_root / "state").exists() +assert not (temporary_root / "manifest.json").exists() +assert not any( + name in sys.modules + for name in ( + "ramdisk_support.benchmark", + "ramdisk_support.curses_ui", + "ramdisk_support.runtime_monitor", + "ramdisk_ui", + "ramdisk_textual", + ) +), sorted(sys.modules) + +stdout = io.StringIO() +stderr = io.StringIO() +with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = ramdisk.dispatch(status_args) +payload = json.loads(stdout.getvalue()) +assert exit_code == 0, exit_code +assert stderr.getvalue() == "", stderr.getvalue() +assert payload["schema"] == ramdisk.STATUS_SCHEMA, payload +assert payload["present"] is False, payload + +stdout = io.StringIO() +stderr = io.StringIO() +with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + stop_exit_code = ramdisk.dispatch(stop_args) +assert stop_exit_code == 2, stop_exit_code +assert stdout.getvalue() == "", stdout.getvalue() +assert stderr.getvalue() == ( + "coli ramdisk: " + UNSUPPORTED_PLATFORM_REASON + "\n" +), stderr.getvalue() + +hardware = ramdisk._discover_hardware(ops=ops) +assert ( + hardware["capabilities"]["reason"] == UNSUPPORTED_PLATFORM_REASON +), hardware["capabilities"] + +print( + json.dumps( + { + "platform": target_platform, + "eager_optional_modules": eager_optional_modules, + "removed": sorted(removed), + "unavailable": sorted(unavailable), + "plan_schema": ramdisk.PLAN_SCHEMA, + "platform_reason": hardware["capabilities"]["reason"], + "process_errors": process_errors, + "mutating_errors": mutating_errors, + "stop_error": UNSUPPORTED_PLATFORM_REASON, + "status_schema": payload["schema"], + }, + sort_keys=True, + ) +) +""" + + def _run_fresh_process_contract(self, platform_name): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + result = subprocess.run( + [ + sys.executable, + "-c", + self.FRESH_PROCESS_CONTRACT, + str(C_DIR), + platform_name, + str(root), + ], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + def test_win32_fresh_process_facade_stays_portable_without_posix_os_apis(self): + result = self._run_fresh_process_contract("win32") + + self.assertEqual(result["platform"], "win32") + self.assertEqual(result["eager_optional_modules"], []) + self.assertEqual( + set(result["unavailable"]), + { + "getuid", + "geteuid", + "getgid", + "getgroups", + "getpgid", + "killpg", + "sched_getaffinity", + "statvfs", + }, + ) + self.assertEqual(result["plan_schema"], ramdisk.PLAN_SCHEMA) + self.assertEqual(result["platform_reason"], UNSUPPORTED_PLATFORM_REASON) + self.assertEqual( + set(result["process_errors"].values()), + {UNSUPPORTED_PLATFORM_REASON}, + ) + self.assertEqual( + set(result["mutating_errors"]), + {"prepare", "start", "stop", "benchmark", "destroy"}, + ) + self.assertEqual( + set(result["mutating_errors"].values()), + {UNSUPPORTED_PLATFORM_REASON}, + ) + self.assertEqual(result["stop_error"], UNSUPPORTED_PLATFORM_REASON) + self.assertEqual(result["status_schema"], ramdisk.STATUS_SCHEMA) + + def test_darwin_fresh_process_facade_stays_portable_without_linux_os_apis(self): + result = self._run_fresh_process_contract("darwin") + + self.assertEqual(result["platform"], "darwin") + self.assertEqual(result["eager_optional_modules"], []) + self.assertEqual(result["unavailable"], ["sched_getaffinity"]) + self.assertEqual(result["plan_schema"], ramdisk.PLAN_SCHEMA) + self.assertEqual(result["platform_reason"], UNSUPPORTED_PLATFORM_REASON) + self.assertEqual( + set(result["process_errors"].values()), + {UNSUPPORTED_PLATFORM_REASON}, + ) + self.assertEqual( + set(result["mutating_errors"]), + {"prepare", "start", "stop", "benchmark", "destroy"}, + ) + self.assertEqual( + set(result["mutating_errors"].values()), + {UNSUPPORTED_PLATFORM_REASON}, + ) + self.assertEqual(result["stop_error"], UNSUPPORTED_PLATFORM_REASON) + self.assertEqual(result["status_schema"], ramdisk.STATUS_SCHEMA) + + def test_import_does_not_probe_linux_facilities(self): + script = r""" +import builtins +import os +import shutil +import sys + +real_open = builtins.open + +def guarded_open(path, *args, **kwargs): + try: + spelling = os.fsdecode(os.fspath(path)).replace("\\", "/") + except TypeError: + spelling = "" + if spelling in ("/proc", "/sys") or spelling.startswith(("/proc/", "/sys/")): + raise AssertionError("RAM-disk import probed " + spelling) + return real_open(path, *args, **kwargs) + +def reject_which(name, *args, **kwargs): + raise AssertionError("RAM-disk import searched PATH for " + name) + +builtins.open = guarded_open +shutil.which = reject_which +if hasattr(os, "sched_getaffinity"): + del os.sched_getaffinity +sys.path.insert(0, sys.argv[1]) +import ramdisk +""" + result = subprocess.run( + [sys.executable, "-c", script, str(C_DIR)], + cwd=C_DIR, + text=True, + capture_output=True, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_selector_returns_explicit_unsupported_capabilities(self): + ops = get_platform_ops("darwin") + + self.assertIsInstance(ops, UnsupportedPlatformOps) + self.assertEqual( + ops.capabilities(), + { + "platform": "darwin", + "hardware_discovery": False, + "cgroup_memory": False, + "numa": False, + "ramdisk_lifecycle": False, + "reason": UNSUPPORTED_PLATFORM_REASON, + }, + ) + + def test_process_mutators_reject_missing_control_before_lock_or_state(self): + ops = mock.Mock( + is_linux=True, + process_control_supported=False, + ) + forbidden_lock = mock.Mock( + side_effect=AssertionError("lifecycle lock was reached") + ) + empty_args = argparse.Namespace() + + with mock.patch.object( + ramdisk, "get_platform_ops", return_value=ops + ), mock.patch.object( + ramdisk, "_lifecycle_lock", forbidden_lock + ): + for name, operation in ( + ("start", lambda: ramdisk.start(empty_args)), + ("stop", lambda: ramdisk.stop()), + ("benchmark", lambda: ramdisk.benchmark(empty_args)), + ("destroy", lambda: ramdisk.destroy(empty_args)), + ): + with self.subTest(operation=name), self.assertRaisesRegex( + ramdisk.RamdiskError, + UNSUPPORTED_PLATFORM_REASON, + ): + operation() + + forbidden_lock.assert_not_called() + + def test_linux_process_mutator_reports_missing_pidfd_before_state(self): + ops = linux_ops.LinuxPlatformOps() + forbidden_lock = mock.Mock( + side_effect=AssertionError("lifecycle lock was reached") + ) + with mock.patch.object( + linux_ops, + "_pidfd_process_control_supported", + return_value=False, + ), mock.patch.object( + ramdisk, + "get_platform_ops", + return_value=ops, + ), mock.patch.object( + ramdisk, + "_lifecycle_lock", + forbidden_lock, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "requires Linux pidfd_open.*pidfd_send_signal", + ): + ramdisk.stop() + + forbidden_lock.assert_not_called() + + def test_facade_discovery_does_not_probe_linux_facilities_when_unsupported(self): + ops = UnsupportedPlatformOps("win32") + with ( + mock.patch.object(discovery, "get_platform_ops", return_value=ops), + mock.patch( + "builtins.open", + side_effect=AssertionError("unsupported discovery read a host file"), + ) as open_file, + mock.patch.object( + linux_ops.shutil, + "which", + side_effect=AssertionError("unsupported discovery searched PATH"), + ) as which, + ): + hardware = ramdisk.discover_hardware() + + self.assertFalse(hardware["linux"]) + self.assertFalse(hardware["capabilities"]["hardware_discovery"]) + self.assertEqual( + hardware["capabilities"]["reason"], + UNSUPPORTED_PLATFORM_REASON, + ) + self.assertFalse(hardware["tmpfs"]["supported"]) + self.assertIsNone(hardware["mount"]) + open_file.assert_not_called() + which.assert_not_called() + + def test_managed_launch_discovery_is_explicitly_unsupported_off_linux(self): + ops = UnsupportedPlatformOps("win32") + with self.assertRaisesRegex( + ramdisk.RamdiskError, + UNSUPPORTED_PLATFORM_REASON, + ): + ops.process_start_boundary() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + UNSUPPORTED_PLATFORM_REASON, + ): + ops.managed_launch_processes( + nonce="a" * 48, + uid=1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_synthetic_cgroup_contracts_remain_platform_independent(self): + with tempfile.TemporaryDirectory() as temporary: + mountpoint = Path(temporary).resolve() / "cgroup" + leaf = mountpoint / "scope" + leaf.mkdir(parents=True) + (mountpoint / "memory.max").write_text("max", encoding="utf-8") + (mountpoint / "memory.current").write_text("0", encoding="utf-8") + (mountpoint / "memory.high").write_text("max", encoding="utf-8") + (leaf / "memory.max").write_text("4096", encoding="utf-8") + (leaf / "memory.current").write_text("1024", encoding="utf-8") + (leaf / "memory.high").write_text("2048", encoding="utf-8") + mountinfo_path = ( + mountpoint.as_posix() + .replace("\\", "\\134") + .replace(" ", "\\040") + .replace("\t", "\\011") + .replace("\n", "\\012") + ) + + with mock.patch.object( + discovery, + "get_platform_ops", + return_value=UnsupportedPlatformOps("darwin"), + ): + result = ramdisk._discover_cgroup_memory( + cgroup_text="0::/scope\n", + mountinfo_text=( + "36 25 0:32 / %s rw,nosuid,nodev,noexec " + "- cgroup2 cgroup rw\n" % mountinfo_path + ), + ) + + self.assertEqual(result["status"], "limited") + self.assertEqual(result["available_bytes"], 3072) + self.assertEqual(result["high_available_bytes"], 1024) + + def test_platform_skip_inventory_is_exact_and_drift_checked(self): + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["linux_operational"]["tests"]), + 39, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["sigterm_handler"]["tests"]), + 6, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["sigint_handler"]["tests"]), + 4, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["posix_pty"]["tests"]), + 1, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["posix_fifo"]["tests"]), + 1, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["native_dirfd"]["tests"]), + 4, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["linux_pidfd"]["tests"]), + 1, + ) + self.assertEqual( + len(PLATFORM_SKIP_INVENTORY["linux_stdlib_pidfd"]["tests"]), + 1, + ) + assert_platform_skip_inventory() + + def test_pidfd_markers_distinguish_managed_libc_from_stdlib_support(self): + target = os.environ.get("COLIBRI_TEST_TARGET_PLATFORM", sys.platform) + managed_pidfd = ( + target.startswith("linux") + and linux_ops._pidfd_process_control_supported() + ) + + self.assertEqual( + PLATFORM_SKIP_INVENTORY["linux_pidfd"]["supported"], + managed_pidfd, + ) + self.assertEqual( + PLATFORM_SKIP_INVENTORY["linux_stdlib_pidfd"]["supported"], + target.startswith("linux") + and callable(getattr(os, "pidfd_open", None)) + and callable(getattr(signal, "pidfd_send_signal", None)), + ) + + +class LinuxOperationalReadContractTest(unittest.TestCase): + @requires_linux_pidfd + def test_real_pidfd_group_signal_targets_each_exact_member(self): + nonce = "d" * 48 + state_dir = "/tmp/colibri-pidfd-test-state" + weights_dir = "/tmp/colibri-pidfd-test-weights" + environment = os.environ.copy() + environment.update( + COLI_MANAGED_NONCE=nonce, + COLI_STATE_DIR=state_dir, + COLI_WEIGHTS_DIR=weights_dir, + ) + program = ( + "import os,time; child=os.fork(); " + "print(child,flush=True) if child else None; time.sleep(60)" + ) + process = subprocess.Popen( + [sys.executable, "-c", program], + start_new_session=True, + stdout=subprocess.PIPE, + text=True, + env=environment, + ) + child_pid = int(process.stdout.readline().strip()) + open_pidfd, send_pidfd = linux_ops._pidfd_api() + cleanup_pidfds = [] + for pid in (process.pid, child_pid): + cleanup_pidfds.append(open_pidfd(pid, 0)) + real_killpg = linux_ops.os.killpg + + try: + leader = linux_ops._strict_process_identity(process.pid) + record = { + "pid": process.pid, + "pgid": process.pid, + "uid": os.getuid(), + "starttime": leader["starttime"], + "nonce": nonce, + "state_dir": state_dir, + "weights_dir": weights_dir, + } + with mock.patch.object( + linux_ops.os, + "killpg", + side_effect=lambda pgid, signum: real_killpg(pgid, signum), + ) as killpg: + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + ) + process.wait(timeout=5.0) + deadline = time.monotonic() + 5.0 + while ( + time.monotonic() < deadline + and linux_ops._process_group_alive(process.pid) + ): + time.sleep(0.05) + + self.assertEqual(result["status"], "signaled") + self.assertEqual( + set(result["signaled"]), + {process.pid, child_pid}, + ) + self.assertEqual(process.returncode, -signal.SIGTERM) + self.assertFalse(linux_ops._process_group_alive(process.pid)) + self.assertTrue( + all(call.args[1] == 0 for call in killpg.call_args_list) + ) + finally: + for descriptor in cleanup_pidfds: + try: + send_pidfd(descriptor, signal.SIGKILL, None, 0) + except OSError: + pass + os.close(descriptor) + if process.poll() is None: + process.kill() + process.wait(timeout=5.0) + process.stdout.close() + + def test_trusted_helper_rejects_foreign_group_writable_parent(self): + safe_directory = mock.Mock( + st_mode=stat.S_IFDIR | 0o755, + st_uid=0, + st_gid=0, + ) + unsafe_directory = mock.Mock( + st_mode=stat.S_IFDIR | 0o775, + st_uid=0, + st_gid=4321, + ) + safe_executable = mock.Mock( + st_mode=stat.S_IFREG | 0o755, + st_uid=0, + st_gid=0, + ) + + def file_stat(path): + if path == "/": + return safe_directory + if path == "/foreign/bin/fuser": + return safe_executable + if path == "/foreign/bin": + return unsafe_directory + if path == "/foreign": + return safe_directory + raise FileNotFoundError(errno.ENOENT, "not found", path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.shutil, + "which", + return_value="/foreign/bin/fuser", + ), mock.patch.object( + linux_ops.os.path, + "realpath", + side_effect=lambda path: path, + ), mock.patch.object( + linux_ops.os, "stat", side_effect=file_stat + ), mock.patch.object( + linux_ops.os, "access", return_value=False + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "rejected writable candidates: /foreign/bin/fuser", + ): + linux_ops._trusted_system_binary("fuser") + + def test_trusted_helper_accepts_sticky_nix_store_ancestor(self): + safe_directory = mock.Mock( + st_mode=stat.S_IFDIR | 0o555, + st_uid=0, + st_gid=0, + ) + root_directory = mock.Mock( + st_mode=stat.S_IFDIR | 0o755, + st_uid=0, + st_gid=0, + ) + nix_store = mock.Mock( + st_mode=stat.S_IFDIR | stat.S_ISVTX | 0o775, + st_uid=0, + st_gid=30000, + ) + safe_executable = mock.Mock( + st_mode=stat.S_IFREG | 0o555, + st_uid=0, + st_gid=0, + ) + executable = "/nix/store/abc123-psmisc/bin/fuser" + + def file_stat(path): + if path in ("/", "/nix"): + return root_directory + if path == "/nix/store": + return nix_store + if path in ( + "/nix/store/abc123-psmisc", + "/nix/store/abc123-psmisc/bin", + ): + return safe_directory + if path == executable: + return safe_executable + raise FileNotFoundError(errno.ENOENT, "not found", path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.shutil, "which", return_value=executable + ), mock.patch.object( + linux_ops.os.path, + "realpath", + side_effect=lambda path: path, + ), mock.patch.object( + linux_ops.os, "stat", side_effect=file_stat + ), mock.patch.object( + linux_ops.os, "access", return_value=False + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + self.assertEqual( + linux_ops._trusted_system_binary("fuser"), + executable, + ) + + def test_process_start_boundary_floors_uptime_to_boot_ticks(self): + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "sysconf", return_value=250 + ), mock.patch( + "builtins.open", + return_value=io.StringIO("123.456 99.0\n"), + ): + boundary = linux_ops.LinuxPlatformOps().process_start_boundary() + + self.assertEqual(boundary, 30864) + + def test_process_start_boundary_fails_closed(self): + cases = ( + ("unreadable", 100, None, "cannot read Linux boot uptime"), + ("malformed", 100, "unknown 0.0\n", "cannot parse Linux boot uptime"), + ("invalid-hz", 0, "123.45 0.0\n", "clock tick rate is invalid"), + ) + for case, ticks_per_second, uptime, message in cases: + def open_uptime(*args, **kwargs): + del args, kwargs + if uptime is None: + raise PermissionError( + errno.EACCES, + "permission denied", + "/proc/uptime", + ) + return io.StringIO(uptime) + + with self.subTest(case=case), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, + "sysconf", + return_value=ticks_per_second, + ), mock.patch( + "builtins.open", side_effect=open_uptime + ): + with self.assertRaisesRegex(ramdisk.RamdiskError, message): + linux_ops._process_start_boundary() + + def test_managed_launch_scan_returns_every_exact_attributed_identity(self): + nonce = "a" * 48 + owners = {731: 1000, 732: 1000, 900: 2000} + starttimes = {731: 17001, 732: 17002} + opened = [] + + def file_stat(path): + pid = int(path.rsplit("/", 1)[-1]) + return mock.Mock(st_uid=owners[pid]) + + def open_proc(path, mode, *args, **kwargs): + del args, kwargs + opened.append(path) + pid = int(path.split("/")[2]) + if path.endswith("/stat"): + self.assertEqual(mode, "r") + return io.StringIO( + _proc_stat_record(pid, 731, 731, starttimes[pid]) + ) + if path.endswith("/cmdline"): + self.assertEqual(mode, "rb") + return io.BytesIO( + ("coli\0serve\0--port\0%d\0" % (8000 + pid - 731)).encode() + ) + if path.endswith("/environ"): + self.assertEqual(mode, "rb") + return io.BytesIO( + ( + "COLI_MANAGED_NONCE=%s\0" + "COLI_STATE_DIR=/state/node-0\0" + "COLI_WEIGHTS_DIR=/mnt/weights\0" % nonce + ).encode() + ) + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, + "listdir", + return_value=["732", "self", "900", "731"], + ), mock.patch.object( + linux_ops.os, "stat", side_effect=file_stat + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + result = linux_ops.LinuxPlatformOps().managed_launch_processes( + nonce=nonce, + uid=1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + self.assertEqual([item["pid"] for item in result], [731, 732]) + self.assertEqual(result[0]["pgid"], 731) + self.assertEqual(result[0]["sid"], 731) + self.assertEqual(result[0]["starttime"], 17001) + self.assertEqual(result[0]["nonce"], nonce) + self.assertEqual(result[0]["cmdline"][:2], ["coli", "serve"]) + self.assertEqual(result[0]["state_dir"], "/state/node-0") + self.assertEqual(result[0]["weights_dir"], "/mnt/weights") + self.assertFalse(any(path.startswith("/proc/900/") for path in opened)) + + def test_process_identity_exposes_group_attribution_fields(self): + nonce = "a" * 48 + + def open_proc(path, mode, *args, **kwargs): + del args, kwargs + if path == "/proc/732/stat": + self.assertEqual(mode, "r") + return io.StringIO( + _proc_stat_record(732, 731, 731, 17002) + ) + if path == "/proc/732/cmdline": + self.assertEqual(mode, "rb") + return io.BytesIO(b"coli\0serve\0") + if path == "/proc/732/environ": + self.assertEqual(mode, "rb") + return io.BytesIO( + ( + "COLI_MANAGED_NONCE=%s\0" + "COLI_STATE_DIR=/state/node-0\0" + "COLI_WEIGHTS_DIR=/mnt/weights\0" % nonce + ).encode() + ) + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "getpgid", return_value=731 + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + identity = linux_ops._process_identity(732) + + self.assertEqual(identity["pid"], 732) + self.assertEqual(identity["uid"], 1000) + self.assertEqual(identity["starttime"], 17002) + self.assertEqual(identity["pgid"], 731) + self.assertEqual(identity["sid"], 731) + self.assertEqual(identity["nonce"], nonce) + self.assertEqual(identity["state_dir"], "/state/node-0") + self.assertEqual(identity["weights_dir"], "/mnt/weights") + + def test_process_identity_preserves_none_for_unreadable_attribution(self): + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(732, 731, 731, 17002) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"coli\0serve\0") + if path.endswith("/environ"): + raise PermissionError( + errno.EACCES, + "permission denied", + path, + ) + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "getpgid", return_value=731 + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + self.assertIsNone(linux_ops._process_identity(732)) + + def test_process_identity_rejects_a_hybrid_reused_pid_snapshot(self): + before = { + "state": "S", + "starttime": 17002, + "pgid": 731, + "sid": 731, + "num_threads": 1, + } + after = dict(before, starttime=17003) + + def open_proc(path, mode, *args, **kwargs): + del args, kwargs + if path.endswith("/cmdline"): + self.assertEqual(mode, "rb") + return io.BytesIO(b"coli\0serve\0") + if path.endswith("/environ"): + self.assertEqual(mode, "rb") + return io.BytesIO(b"OTHER=value\0") + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, + "_strict_proc_stat_identity", + side_effect=[before, after], + ), mock.patch.object( + linux_ops.os, "getpgid", return_value=731 + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch("builtins.open", side_effect=open_proc): + self.assertIsNone(linux_ops._process_identity(732)) + + def test_process_identity_proves_a_lone_thread_zombie_inert(self): + opened = [] + + def open_proc(path, mode, *args, **kwargs): + del args, kwargs + opened.append(path) + if path.endswith("/stat"): + self.assertEqual(mode, "r") + return io.StringIO( + _proc_stat_record(732, 731, 731, 17002, state="Z") + ) + raise AssertionError("zombie endpoint must not be read: %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["732"] + ), mock.patch("builtins.open", side_effect=open_proc): + identity = linux_ops._process_identity(732) + + self.assertTrue(identity["inert"]) + self.assertEqual(identity["state"], "Z") + self.assertEqual(identity["starttime"], 17002) + self.assertNotIn("/proc/732/cmdline", opened) + self.assertNotIn("/proc/732/environ", opened) + + def test_managed_launch_inspection_treats_a_stable_zombie_as_inert(self): + opened = [] + + def open_proc(path, mode, *args, **kwargs): + del args, kwargs + opened.append(path) + if path.endswith("/stat"): + self.assertEqual(mode, "r") + return io.StringIO( + _proc_stat_record(731, 731, 731, 17000, state="Z") + ) + raise AssertionError("zombie endpoint must not be read: %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch("builtins.open", side_effect=open_proc): + observation = linux_ops._inspect_managed_launch_pid( + 731, + 1000, + 17000, + ) + + self.assertEqual(observation["kind"], "inert-dead") + self.assertEqual(observation["state"], "Z") + self.assertNotIn("/proc/731/cmdline", opened) + self.assertNotIn("/proc/731/environ", opened) + + def test_inert_identity_rejects_incomplete_or_changing_task_snapshot(self): + before = { + "state": "Z", + "starttime": 17000, + "pgid": 731, + "sid": 731, + "num_threads": 1, + } + with self.subTest(case="unexpected-sibling"), mock.patch.object( + linux_ops.os, + "listdir", + return_value=["731", "732"], + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "incomplete or live task group", + ): + linux_ops._stable_inert_process_identity( + 731, + "/proc/731", + 1000, + before, + ) + + changed = dict(before, num_threads=2) + with self.subTest(case="stat-turnover"), mock.patch.object( + linux_ops.os, + "listdir", + return_value=["731"], + ), mock.patch.object( + linux_ops, + "_strict_proc_stat_identity", + return_value=changed, + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "changed during pending-launch recovery", + ): + linux_ops._stable_inert_process_identity( + 731, + "/proc/731", + 1000, + before, + ) + + def test_managed_launch_scan_skips_only_old_unreadable_same_uid(self): + for starttime, must_refuse in ((16999, False), (17000, True)): + opened = [] + + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + opened.append(path) + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(731, 731, 731, starttime) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"unrelated\0") + if path.endswith("/environ"): + raise PermissionError( + errno.EACCES, + "permission denied", + path, + ) + raise AssertionError("unexpected open %s" % path) + + with self.subTest( + starttime=starttime + ), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + if must_refuse: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot read Linux process identity /proc/731/environ", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + self.assertIn("/proc/731/environ", opened) + else: + self.assertEqual( + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ), + [], + ) + self.assertNotIn("/proc/731/cmdline", opened) + self.assertNotIn("/proc/731/environ", opened) + + def test_managed_launch_scan_rejects_recent_missing_or_wrong_nonce(self): + nonce = "a" * 48 + for case, actual_nonce in (("missing", None), ("wrong", "b" * 48)): + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(731, 731, 731, 17000) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"coli\0ramdisk\0start\0") + if path.endswith("/environ"): + nonce_field = ( + "COLI_MANAGED_NONCE=%s\0" % actual_nonce + if actual_nonce is not None + else "" + ) + return io.BytesIO( + ( + nonce_field + + "COLI_STATE_DIR=/state/node-0\0" + + "COLI_WEIGHTS_DIR=/mnt/weights\0" + ).encode() + ) + raise AssertionError("unexpected open %s" % path) + + with self.subTest(case=case), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "(ambiguous managed launch attribution|" + "missing or mismatched nonce attribution)", + ): + linux_ops._managed_launch_processes( + nonce, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_ignores_stable_readable_recent_bystander(self): + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(731, 731, 731, 17000) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"unrelated-worker\0") + if path.endswith("/environ"): + return io.BytesIO( + b"BROKEN\0" + b"COLI_MANAGED_NONCE=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\0" + b"COLI_MANAGED_NONCE\0" + b"COLI_STATE_DIR=/other/a\0" + b"COLI_STATE_DIR=/other/b\0" + b"COLI_WEIGHTS_DIR=/other/weights\0" + ) + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch("builtins.open", side_effect=open_proc): + self.assertEqual( + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ), + [], + ) + + def test_managed_launch_scan_rejects_path_attribution_without_target_nonce(self): + for case, actual_nonce in (("missing", None), ("wrong", "b" * 48)): + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(731, 731, 731, 17000) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"renamed-engine\0") + if path.endswith("/environ"): + nonce_field = ( + ("COLI_MANAGED_NONCE=%s\0" % actual_nonce).encode() + if actual_nonce is not None + else b"" + ) + return io.BytesIO( + nonce_field + + b"COLI_STATE_DIR=/state/node-0\0" + + b"COLI_WEIGHTS_DIR=/mnt/weights\0" + ) + raise AssertionError("unexpected open %s" % path) + + with self.subTest(case=case), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch("builtins.open", side_effect=open_proc): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "(ambiguous managed launch attribution|" + "missing or mismatched nonce attribution)", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_excludes_the_exact_original_launcher(self): + observation = { + "kind": "same-uid", + "pid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "cmdline": ["coli", "ramdisk", "start"], + "state_dir": None, + "weights_dir": None, + "environment_candidates": {}, + "environment_ambiguities": ( + "COLI_MANAGED_NONCE", + "COLI_STATE_DIR", + "COLI_WEIGHTS_DIR", + ), + } + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, + "_proc_pid_snapshot", + return_value={731: "/proc/731"}, + ), mock.patch.object( + linux_ops, + "_inspect_managed_launch_pid", + return_value=observation, + ): + self.assertEqual( + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=731, + launcher_starttime=17000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ), + [], + ) + + def test_managed_launch_scan_excludes_the_unattributed_recovery_process(self): + recovery_pid = linux_ops.os.getpid() + observation = { + "kind": "same-uid", + "pid": recovery_pid, + "uid": 1000, + "starttime": 18000, + "nonce": None, + "pgid": recovery_pid, + "sid": recovery_pid, + "cmdline": ["coli", "ramdisk", "start"], + "state_dir": None, + "weights_dir": None, + "environment_candidates": {}, + "environment_ambiguities": ( + "COLI_MANAGED_NONCE", + "COLI_STATE_DIR", + "COLI_WEIGHTS_DIR", + ), + } + snapshot = {recovery_pid: "/proc/%d" % recovery_pid} + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "_proc_pid_snapshot", return_value=snapshot + ), mock.patch.object( + linux_ops, + "_inspect_managed_launch_pid", + return_value=observation, + ): + self.assertEqual( + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ), + [], + ) + + def test_managed_launch_scan_rejects_turnover_after_final_reads(self): + stable = { + "kind": "same-uid", + "pid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "cmdline": ["unrelated-worker"], + "state_dir": None, + "weights_dir": None, + "environment_candidates": {}, + "environment_ambiguities": (), + } + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, + "_proc_pid_snapshot", + side_effect=[ + {731: "/proc/731"}, + {731: "/proc/731"}, + {731: "/proc/731"}, + {732: "/proc/732"}, + ], + ), mock.patch.object( + linux_ops, + "_inspect_managed_launch_pid", + side_effect=[stable, stable, None], + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unverified PID.*732", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_rechecks_final_snapshot_identity(self): + stable = { + "kind": "same-uid", + "pid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "cmdline": ["unrelated-worker"], + "state_dir": None, + "weights_dir": None, + } + reused = dict(stable, starttime=17001) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, + "_proc_pid_snapshot", + side_effect=[{731: "/proc/731"}] * 3, + ), mock.patch.object( + linux_ops, + "_inspect_managed_launch_pid", + side_effect=[stable, stable, reused], + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "changed during final", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_rechecks_same_pid_after_fourth_snapshot(self): + stable = { + "kind": "same-uid", + "pid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "cmdline": ["unrelated-worker"], + "state_dir": None, + "weights_dir": None, + "environment_candidates": {}, + "environment_ambiguities": (), + } + reused = dict(stable, starttime=17001) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, + "_proc_pid_snapshot", + return_value={731: "/proc/731"}, + ), mock.patch.object( + linux_ops, + "_inspect_managed_launch_pid", + side_effect=[stable, stable, stable, reused], + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "changed after the final", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_rechecks_same_pid_in_final_confirmation(self): + stable = { + "kind": "same-uid", + "pid": 731, + "uid": 1000, + "state": "S", + "inert": False, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "cmdline": ["unrelated-worker"], + "state_dir": None, + "weights_dir": None, + "environment_candidates": {}, + "environment_ambiguities": (), + } + reused = dict(stable, starttime=17001) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, + "_proc_pid_snapshot", + return_value={731: "/proc/731"}, + ), mock.patch.object( + linux_ops, + "_inspect_managed_launch_pid", + side_effect=[stable, stable, stable, stable, reused], + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "changed during final pending-launch identity confirmation", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_treats_pid_disappearance_as_benign(self): + owner_reads = 0 + + def file_stat(path): + nonlocal owner_reads + self.assertEqual(path, "/proc/731") + owner_reads += 1 + if owner_reads == 1: + return mock.Mock(st_uid=1000) + raise FileNotFoundError(errno.ENOENT, "process exited", path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, + "listdir", + side_effect=(["731"], [], [], [], []), + ), mock.patch.object( + linux_ops.os, "stat", side_effect=file_stat + ), mock.patch( + "builtins.open", + side_effect=FileNotFoundError( + errno.ENOENT, + "process exited", + "/proc/731/stat", + ), + ): + self.assertEqual( + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ), + [], + ) + + def test_managed_launch_scan_rejects_unreadable_or_malformed_same_uid(self): + cases = ( + ( + "denied-environ", + PermissionError( + errno.EACCES, + "permission denied", + "/proc/731/environ", + ), + "cannot read Linux process identity /proc/731/environ", + ), + ( + "malformed-environ", + None, + "ambiguous managed launch attribution", + ), + ( + "malformed-stat", + None, + "cannot parse Linux process identity /proc/731/stat", + ), + ) + for case, environ_error, message in cases: + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + if case == "malformed-stat": + return io.StringIO("731 (truncated) S 1\n") + return io.StringIO( + _proc_stat_record(731, 731, 731, 17001) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"coli\0serve\0") + if path.endswith("/environ"): + if environ_error is not None: + raise environ_error + payload = ( + b"not-an-environment-entry\0" + if case == "malformed-environ" + else b"OTHER=value\0" + ) + return io.BytesIO(payload) + raise AssertionError("unexpected open %s" % path) + + with self.subTest(case=case), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex(ramdisk.RamdiskError, message): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_rejects_new_uninspected_pid(self): + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(731, 731, 731, 17001) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"unrelated\0") + if path.endswith("/environ"): + return io.BytesIO(b"OTHER=value\0") + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, + "listdir", + side_effect=(["731"], ["731", "732"]), + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "uninspected PID.*732", + ): + linux_ops._managed_launch_processes( + "a" * 48, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_managed_launch_scan_rejects_attribution_or_session_ambiguity(self): + nonce = "a" * 48 + cases = ( + ( + "/wrong/state", + 731, + "mismatched state or weights attribution", + ), + ( + "/state/node-0", + 700, + "violates the new-session process-group identity", + ), + ) + for actual_state, session, message in cases: + def open_proc(path, mode, *args, **kwargs): + del mode, args, kwargs + if path.endswith("/stat"): + return io.StringIO( + _proc_stat_record(731, 731, session, 17001) + ) + if path.endswith("/cmdline"): + return io.BytesIO(b"coli\0serve\0") + if path.endswith("/environ"): + return io.BytesIO( + ( + "COLI_MANAGED_NONCE=%s\0" + "COLI_STATE_DIR=%s\0" + "COLI_WEIGHTS_DIR=/mnt/weights\0" + % (nonce, actual_state) + ).encode() + ) + raise AssertionError("unexpected open %s" % path) + + with self.subTest(message=message), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex(ramdisk.RamdiskError, message): + linux_ops._managed_launch_processes( + nonce, + 1000, + state_dir="/state/node-0", + weights_dir="/mnt/weights", + not_before_starttime=17000, + launcher_pid=700, + launcher_starttime=16000, + launcher_cmdline=["coli", "ramdisk", "start"], + expected_command=["coli", "serve"], + ) + + def test_process_group_scan_rejects_proc_enumeration_failures(self): + failures = ( + PermissionError(errno.EACCES, "permission denied", "/proc"), + OSError(errno.EIO, "input/output error", "/proc"), + ) + for failure in failures: + with self.subTest(errno=failure.errno), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=failure + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot enumerate Linux process table", + ) as raised: + linux_ops._process_group_member_pids(731) + self.assertIn( + "managed cleanup requires complete process-table visibility", + str(raised.exception), + ) + + def test_process_group_scan_rejects_unreadable_member_stat(self): + failures = ( + PermissionError( + errno.EACCES, + "permission denied", + "/proc/731/stat", + ), + OSError(errno.EIO, "input/output error", "/proc/731/stat"), + ) + for failure in failures: + with self.subTest(errno=failure.errno), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch( + "builtins.open", side_effect=failure + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot read Linux process identity /proc/731/stat", + ): + linux_ops._process_group_member_pids(731) + + def test_process_group_scan_rejects_truncated_member_stat(self): + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch( + "builtins.open", + mock.mock_open(read_data="731 (worker) S 1"), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot parse Linux process identity /proc/731/stat", + ): + linux_ops._process_group_member_pids(731) + + def test_process_group_scan_accepts_non_utf8_process_name_bytes(self): + raw_stat = b"731 (worker-\xff) S 1 731 731 0 -1 0\n" + + def open_proc(path, mode, *, encoding, errors, newline): + self.assertEqual(path, "/proc/731/stat") + self.assertEqual(mode, "r") + self.assertEqual(newline, "") + return io.TextIOWrapper( + io.BytesIO(raw_stat), + encoding=encoding, + errors=errors, + newline=newline, + ) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + self.assertEqual( + linux_ops._process_group_member_pids(731), + [731], + ) + + def test_process_group_scan_preserves_pid_disappearance(self): + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch( + "builtins.open", + side_effect=FileNotFoundError( + errno.ENOENT, + "no such process", + "/proc/731/stat", + ), + ): + self.assertEqual( + linux_ops._process_group_member_pids(731), + [], + ) + + def test_process_group_liveness_treats_only_stable_zombies_as_inert(self): + zombie = { + "state": "Z", + "starttime": 17000, + "pgid": 731, + "sid": 731, + "num_threads": 1, + } + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "killpg" + ) as killpg, mock.patch.object( + linux_ops, + "_process_group_member_pids", + side_effect=[[731], [731]], + ), mock.patch.object( + linux_ops.os, + "stat", + return_value=mock.Mock(st_uid=1000), + ), mock.patch.object( + linux_ops, + "_strict_proc_stat_identity", + return_value=zombie, + ), mock.patch.object( + linux_ops, + "_stable_inert_process_identity", + return_value=zombie, + ): + self.assertFalse(linux_ops._process_group_alive(731)) + + killpg.assert_called_once_with(731, 0) + + def test_empty_dead_process_group_is_not_running(self): + ops = mock.Mock() + ops.process_group_member_pids.return_value = [] + ops.process_group_alive.return_value = False + + members = processes._process_group_members( + 731, + ops=ops, + proc_identity=mock.Mock(), + ) + result = processes._process_matches( + {"pid": 731, "pgid": 731}, + proc_identity=lambda ignored: None, + process_group_members=lambda ignored: members, + ) + + self.assertEqual(members, ([], [])) + self.assertEqual(result, (False, "not-running", None)) + + def test_empty_live_process_group_is_unverified(self): + ops = mock.Mock() + ops.process_group_member_pids.return_value = [] + ops.process_group_alive.return_value = True + + members = processes._process_group_members( + 731, + ops=ops, + proc_identity=mock.Mock(), + ) + result = processes._process_matches( + {"pid": 731, "pgid": 731}, + proc_identity=lambda ignored: None, + process_group_members=lambda ignored: members, + ) + + self.assertEqual(members, ([], [731])) + self.assertFalse(result[0]) + self.assertEqual(result[1], "unverified-process-group") + + def test_process_group_membership_churn_is_unverified_not_absent(self): + zombie = { + "pid": 731, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "state_dir": None, + "weights_dir": None, + } + child = { + "pid": 732, + "uid": 1000, + "state": "S", + "inert": False, + "starttime": 17001, + "nonce": "a" * 48, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + ops = mock.Mock() + ops.process_group_member_pids.side_effect = ( + [731], + [731], + [731, 732], + [731, 732], + ) + identities = {731: zombie, 732: child} + + members, unreadable = processes._process_group_members( + 731, + ops=ops, + proc_identity=lambda pid: identities[pid], + ) + + self.assertTrue(unreadable) + result = processes._process_matches( + { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + }, + proc_identity=lambda ignored: zombie, + process_group_members=lambda ignored: (members, unreadable), + ) + self.assertFalse(result[0]) + self.assertEqual(result[1], "unverified-process-group") + + def test_process_group_stability_ignores_scheduler_state_transitions(self): + member = { + "pid": 731, + "uid": 1000, + "state": "R", + "inert": False, + "starttime": 17000, + "nonce": "a" * 48, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + sleeping = dict(member, state="S") + ops = mock.Mock() + ops.process_group_member_pids.return_value = [731] + + members, unreadable = processes._process_group_members( + 731, + ops=ops, + proc_identity=mock.Mock(side_effect=(member, sleeping)), + ) + + self.assertEqual(members, [sleeping]) + self.assertEqual(unreadable, []) + + def test_pidfd_group_signal_opens_all_targets_before_validation(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + identities = { + 731: dict( + record, + inert=False, + state="S", + sid=731, + ), + 732: dict( + record, + pid=732, + starttime=17001, + inert=False, + state="S", + sid=731, + ), + } + events = [] + + def pidfd_open(pid, flags): + events.append(("open", pid, flags)) + return pid + 1000 + + def identity(pid): + events.append(("identity", pid)) + return identities[pid] + + def pidfd_send(pidfd, signum, siginfo, flags): + events.append(("signal", pidfd, signum, siginfo, flags)) + + close = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"), mock.patch.object( + linux_ops.os, "killpg", create=True + ) as killpg: + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock( + side_effect=([731, 732], [731, 732], [731, 732]) + ), + process_identity=identity, + process_group_alive=mock.Mock(return_value=True), + pidfd_open=pidfd_open, + pidfd_send_signal=pidfd_send, + pidfd_exited=mock.Mock(return_value=False), + close_fd=close, + ) + + self.assertEqual(result["status"], "signaled") + self.assertEqual(result["signaled"], [731, 732]) + self.assertLess( + max(index for index, event in enumerate(events) if event[0] == "open"), + min(index for index, event in enumerate(events) if event[0] == "identity"), + ) + self.assertLess( + max(index for index, event in enumerate(events) if event[0] == "identity"), + min(index for index, event in enumerate(events) if event[0] == "signal"), + ) + killpg.assert_not_called() + self.assertEqual( + {call.args[0] for call in close.call_args_list}, + {1731, 1732}, + ) + + def test_pidfd_group_signal_refuses_reused_leader_after_binding(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + reused = dict( + record, + starttime=17001, + inert=False, + state="S", + sid=731, + ) + send = mock.Mock() + close = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock( + side_effect=([731], [731], [731]) + ), + process_identity=mock.Mock(return_value=reused), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(return_value=17), + pidfd_send_signal=send, + pidfd_exited=mock.Mock(return_value=False), + close_fd=close, + ) + + self.assertEqual(result["status"], "foreign") + self.assertEqual(result["reason"], "reused-pid") + send.assert_not_called() + close.assert_called_once_with(17) + + def test_pidfd_group_signal_treats_open_esrch_as_inconclusive(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + send = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731]), + process_identity=mock.Mock(), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(side_effect=ProcessLookupError()), + pidfd_send_signal=send, + pidfd_exited=mock.Mock(return_value=False), + close_fd=mock.Mock(), + ) + + self.assertEqual(result["status"], "inconclusive") + send.assert_not_called() + + def test_pidfd_group_signal_detects_same_number_reuse_by_fd_readiness(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + exact = dict(record, inert=False, state="S", sid=731) + send = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731]), + process_identity=mock.Mock(return_value=exact), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(return_value=17), + pidfd_send_signal=send, + pidfd_exited=mock.Mock( + side_effect=(False, False, True) + ), + close_fd=mock.Mock(), + ) + + self.assertEqual(result["status"], "inconclusive") + self.assertEqual( + result["reason"], + "pinned-member-exited-before-signal", + ) + send.assert_not_called() + + def test_pidfd_group_signal_validates_entire_batch_before_first_signal(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + leader = dict(record, inert=False, state="S", sid=731) + foreign = dict( + leader, + pid=732, + starttime=17001, + nonce="b" * 48, + ) + send = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731, 732]), + process_identity=mock.Mock(side_effect=(leader, foreign)), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(side_effect=(17, 18)), + pidfd_send_signal=send, + pidfd_exited=mock.Mock(return_value=False), + close_fd=mock.Mock(), + ) + + self.assertEqual(result["status"], "foreign") + self.assertEqual(result["reason"], "foreign-nonce") + send.assert_not_called() + + def test_pidfd_group_signal_reports_partial_non_esrch_failure(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + identities = ( + dict(record, inert=False, state="S", sid=731), + dict( + record, + pid=732, + starttime=17001, + inert=False, + state="S", + sid=731, + ), + ) + send = mock.Mock( + side_effect=(None, OSError(errno.EIO, "input/output error")) + ) + with mock.patch.object(linux_ops, "_require_linux"), mock.patch.object( + linux_ops.os, "killpg", create=True + ) as killpg: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "after signaling PID\\(s\\) 731.*numeric fallback is forbidden", + ): + linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock( + return_value=[731, 732] + ), + process_identity=mock.Mock(side_effect=identities), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(side_effect=(17, 18)), + pidfd_send_signal=send, + pidfd_exited=mock.Mock(return_value=False), + close_fd=mock.Mock(), + ) + + self.assertEqual(send.call_count, 2) + killpg.assert_not_called() + + def test_pidfd_group_signal_fails_closed_when_kernel_lacks_pidfd(self): + record = {"pid": 731, "pgid": 731} + send = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "requires Linux pidfd_open.*pidfd_open failed", + ): + linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731]), + process_identity=mock.Mock(), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock( + side_effect=OSError( + errno.ENOSYS, + "function not implemented", + ) + ), + pidfd_send_signal=send, + pidfd_exited=mock.Mock(return_value=False), + close_fd=mock.Mock(), + ) + + send.assert_not_called() + + def test_pidfd_group_signal_treats_send_esrch_as_gone(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + exact = dict(record, inert=False, state="S", sid=731) + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731]), + process_identity=mock.Mock(return_value=exact), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(return_value=17), + pidfd_send_signal=mock.Mock( + side_effect=ProcessLookupError() + ), + pidfd_exited=mock.Mock(return_value=False), + close_fd=mock.Mock(), + ) + + self.assertEqual(result["status"], "signaled") + self.assertEqual(result["signaled"], []) + self.assertEqual(result["exited"], [731]) + + def test_pidfd_group_signal_skips_exact_inert_members(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + inert = { + "pid": 731, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17000, + "pgid": 731, + "sid": 731, + } + live = dict( + record, + pid=732, + starttime=17001, + state="S", + inert=False, + sid=731, + ) + send = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731, 732]), + process_identity=mock.Mock(side_effect=(inert, live)), + process_group_alive=mock.Mock(return_value=True), + pidfd_open=mock.Mock(side_effect=(17, 18)), + pidfd_send_signal=send, + pidfd_exited=lambda pidfd: pidfd == 17, + close_fd=mock.Mock(), + ) + + self.assertEqual(result["status"], "signaled") + self.assertEqual(result["signaled"], [732]) + send.assert_called_once_with(18, signal.SIGTERM, None, 0) + + def test_pidfd_group_signal_accepts_only_stably_inert_absence(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + inert = { + "pid": 731, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17000, + "pgid": 731, + "sid": 731, + } + send = mock.Mock() + with mock.patch.object(linux_ops, "_require_linux"): + result = linux_ops._signal_verified_process_group( + record, + signal.SIGTERM, + process_group_member_pids=mock.Mock(return_value=[731]), + process_identity=mock.Mock(return_value=inert), + process_group_alive=mock.Mock(side_effect=(False, False)), + pidfd_open=mock.Mock(return_value=17), + pidfd_send_signal=send, + pidfd_exited=mock.Mock(return_value=True), + close_fd=mock.Mock(), + ) + + self.assertEqual(result["status"], "absent") + send.assert_not_called() + + def test_verified_termination_retries_inconclusive_pidfd_scan(self): + record = {"pid": 731, "pgid": 731} + ops = mock.Mock() + ops.signal_verified_process_group.side_effect = ( + { + "status": "inconclusive", + "reason": "membership-changed", + "members": [731], + }, + {"status": "signaled", "signaled": [731], "members": [731]}, + {"status": "absent", "members": []}, + ) + + with mock.patch.object(processes.time, "sleep") as sleep: + failure = processes._terminate_verified_group( + record, + term_seconds=1.0, + kill_seconds=1.0, + managed_child_liveness=lambda ignored: False, + ops=ops, + ) + + self.assertIsNone(failure) + self.assertEqual(ops.signal_verified_process_group.call_count, 3) + self.assertTrue( + all( + call.args[1] == signal.SIGTERM + for call in ops.signal_verified_process_group.call_args_list + ) + ) + sleep.assert_called() + + def test_verified_termination_retries_transient_strict_identity_error(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + exact = dict(record, inert=False, state="S", sid=731) + sends = [] + + class Ops: + calls = 0 + + def signal_verified_process_group(self, current, signum): + self.calls += 1 + if self.calls == 3: + members = mock.Mock(return_value=[]) + identity = mock.Mock() + alive = mock.Mock(return_value=False) + else: + members = mock.Mock(return_value=[731]) + if self.calls == 1: + identity = mock.Mock( + side_effect=ramdisk.RamdiskError( + "procfs changed during TERM transition" + ) + ) + else: + identity = mock.Mock(return_value=exact) + alive = mock.Mock(return_value=True) + return linux_ops._signal_verified_process_group( + current, + signum, + process_group_member_pids=members, + process_identity=identity, + process_group_alive=alive, + pidfd_open=mock.Mock(return_value=17), + pidfd_send_signal=lambda *args: sends.append(args), + pidfd_exited=mock.Mock(return_value=False), + close_fd=mock.Mock(), + ) + + ops = Ops() + with mock.patch.object(linux_ops, "_require_linux"), mock.patch.object( + processes.time, "sleep" + ): + failure = processes._terminate_verified_group( + record, + term_seconds=1.0, + kill_seconds=1.0, + managed_child_liveness=lambda ignored: False, + ops=ops, + ) + + self.assertIsNone(failure) + self.assertEqual(ops.calls, 3) + self.assertEqual(len(sends), 1) + self.assertEqual(sends[0][1], signal.SIGTERM) + + def test_verified_termination_resignals_new_exact_members(self): + record = {"pid": 731, "pgid": 731} + ops = mock.Mock() + ops.signal_verified_process_group.side_effect = ( + {"status": "signaled", "signaled": [731], "members": [731]}, + { + "status": "signaled", + "signaled": [731, 732], + "members": [731, 732], + }, + {"status": "absent", "members": []}, + ) + + with mock.patch.object(processes.time, "sleep"): + failure = processes._terminate_verified_group( + record, + term_seconds=1.0, + kill_seconds=1.0, + managed_child_liveness=lambda ignored: False, + ops=ops, + ) + + self.assertIsNone(failure) + self.assertEqual(ops.signal_verified_process_group.call_count, 3) + + def test_verified_termination_fails_foreign_attribution_immediately(self): + record = {"pid": 731, "pgid": 731} + ops = mock.Mock() + ops.signal_verified_process_group.return_value = { + "status": "foreign", + "reason": "reused-pid", + "members": [731], + } + + failure = processes._terminate_verified_group( + record, + term_seconds=1.0, + kill_seconds=1.0, + managed_child_liveness=lambda ignored: False, + ops=ops, + ) + + self.assertIn("reused-pid", failure) + ops.signal_verified_process_group.assert_called_once_with( + record, + signal.SIGTERM, + ) + + def test_process_match_revalidates_every_live_group_member_attribution(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + leader = { + "pid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + mismatches = ( + dict(leader, pid=732, pgid=999), + dict(leader, pid=732, sid=999), + dict(leader, pid=732, state_dir="/wrong/state"), + dict(leader, pid=732, weights_dir="/wrong/weights"), + ) + for member in mismatches: + with self.subTest(member=member): + result = processes._process_matches( + record, + proc_identity=lambda ignored: leader, + process_group_members=lambda ignored: ([member], []), + ) + self.assertFalse(result[0]) + self.assertEqual(result[1], "foreign-process-group") + + def test_dead_wrapper_group_requires_exact_member_paths_and_session(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + mismatched = { + "pid": 732, + "uid": 1000, + "starttime": 17001, + "nonce": nonce, + "pgid": 999, + "sid": 999, + "state_dir": "/wrong/state", + "weights_dir": "/wrong/weights", + } + result = processes._process_matches( + record, + proc_identity=lambda ignored: None, + process_group_members=lambda ignored: ([mismatched], []), + ) + self.assertFalse(result[0]) + self.assertEqual(result[1], "foreign-process-group") + + def test_inert_zombie_leader_does_not_hide_exact_live_child(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + zombie = { + "pid": 731, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "state_dir": None, + "weights_dir": None, + } + child = { + "pid": 732, + "uid": 1000, + "state": "S", + "inert": False, + "starttime": 17001, + "nonce": nonce, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + members = lambda ignored: ([zombie, child], []) + running = processes._process_matches( + record, + proc_identity=lambda ignored: zombie, + process_group_members=members, + ) + self.assertTrue(running[0]) + self.assertEqual(running[1], "running-group") + + stopped = processes._process_matches( + record, + proc_identity=lambda ignored: zombie, + process_group_members=lambda ignored: ([zombie], []), + ) + self.assertEqual(stopped, (False, "not-running", None)) + + def test_inert_nonleader_does_not_hide_exact_live_group_members(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + leader = { + "pid": 731, + "uid": 1000, + "inert": False, + "starttime": 17000, + "nonce": nonce, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + zombie = { + "pid": 732, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17001, + "nonce": None, + "pgid": 731, + "sid": 731, + "state_dir": None, + "weights_dir": None, + } + child = dict(leader, pid=733, starttime=17002) + + running = processes._process_matches( + record, + proc_identity=lambda ignored: leader, + process_group_members=lambda ignored: ([leader, zombie, child], []), + ) + + self.assertTrue(running[0]) + self.assertEqual(running[1], "running") + + def test_all_inert_group_is_not_running_with_or_without_leader(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + leader = { + "pid": 731, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17000, + "nonce": None, + "pgid": 731, + "sid": 731, + "state_dir": None, + "weights_dir": None, + } + child = dict(leader, pid=732, starttime=17001) + + for case, actual, members in ( + ("leader-present", leader, [leader, child]), + ("leader-absent", None, [child]), + ): + with self.subTest(case=case): + stopped = processes._process_matches( + record, + proc_identity=lambda ignored, value=actual: value, + process_group_members=lambda ignored, value=members: ( + value, + [], + ), + ) + self.assertEqual(stopped, (False, "not-running", None)) + + foreign = processes._process_matches( + record, + proc_identity=lambda ignored: None, + process_group_members=lambda ignored: ( + [dict(child, uid=1001, pgid=999, sid=999)], + [], + ), + ) + self.assertFalse(foreign[0]) + self.assertEqual(foreign[1], "foreign-process-group") + + def test_inert_group_member_requires_exact_uid_pgid_and_sid(self): + nonce = "a" * 48 + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": nonce, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + leader = { + "pid": 731, + "uid": 1000, + "inert": False, + "starttime": 17000, + "nonce": nonce, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + zombie = { + "pid": 732, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17001, + "nonce": None, + "pgid": 731, + "sid": 731, + "state_dir": None, + "weights_dir": None, + } + for field, value in (("uid", 1001), ("pgid", 999), ("sid", 999)): + with self.subTest(field=field): + mismatched = dict(zombie, **{field: value}) + result = processes._process_matches( + record, + proc_identity=lambda ignored: leader, + process_group_members=lambda ignored: ( + [leader, mismatched], + [], + ), + ) + self.assertFalse(result[0]) + self.assertEqual(result[1], "foreign-process-group") + + def test_inert_leader_requires_exact_persisted_starttime(self): + record = { + "pid": 731, + "pgid": 731, + "uid": 1000, + "starttime": 17000, + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + reused = { + "pid": 731, + "uid": 1000, + "state": "Z", + "inert": True, + "starttime": 17001, + "nonce": None, + "pgid": 731, + "sid": 731, + "state_dir": None, + "weights_dir": None, + } + child = { + "pid": 732, + "uid": 1000, + "state": "S", + "inert": False, + "starttime": 17002, + "nonce": "a" * 48, + "pgid": 731, + "sid": 731, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + + result = processes._process_matches( + record, + proc_identity=lambda ignored: reused, + process_group_members=lambda ignored: ([reused, child], []), + ) + + self.assertFalse(result[0]) + self.assertEqual(result[1], "foreign-process-group") + + def test_mount_table_rejects_operational_read_failures(self): + failures = ( + PermissionError( + errno.EACCES, + "permission denied", + "/proc/self/mountinfo", + ), + OSError( + errno.EIO, + "input/output error", + "/proc/self/mountinfo", + ), + ) + for failure in failures: + with self.subTest(errno=failure.errno), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch( + "builtins.open", side_effect=failure + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot read Linux mount table", + ): + linux_ops._mount_table() + + def test_mount_table_rejects_malformed_records(self): + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch( + "builtins.open", + mock.mock_open(read_data="36 25 truncated\n"), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot parse Linux mount table .* line 1", + ): + linux_ops._mount_table() + + def test_mount_table_preserves_successfully_read_empty_table(self): + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch( + "builtins.open", return_value=io.StringIO("") + ): + self.assertEqual(linux_ops._mount_table(), []) + + def test_busy_mount_scan_rejects_proc_enumeration_failure(self): + denied = PermissionError(errno.EACCES, "permission denied", "/proc") + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=denied + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed cleanup requires complete /proc visibility", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_rejects_unreadable_pid_reference(self): + denied = PermissionError( + errno.EACCES, + "permission denied", + "/proc/731/cwd", + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=denied + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot read process reference /proc/731/cwd.*hidepid", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_rejects_unreadable_or_malformed_maps(self): + failures = ( + OSError(errno.EIO, "input/output error", "/proc/731/maps"), + None, + ) + for failure in failures: + opened = ( + {"side_effect": failure} + if failure is not None + else {"return_value": io.StringIO("truncated\n")} + ) + with self.subTest(failure=type(failure).__name__), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, "readlink", return_value="/" + ), mock.patch( + "builtins.open", **opened + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "process mappings /proc/731/maps", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_rejects_unreadable_descriptor_table(self): + def list_directory(path): + if path == "/proc": + return ["731"] + raise PermissionError(errno.EACCES, "permission denied", path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", return_value="/" + ), mock.patch( + "builtins.open", return_value=io.StringIO("") + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot enumerate process descriptors /proc/731/fd", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_preserves_per_pid_disappearance(self): + vanished = FileNotFoundError( + errno.ENOENT, + "process exited", + "/proc/731/cwd", + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=vanished + ), mock.patch( + "builtins.open", + side_effect=FileNotFoundError( + errno.ENOENT, + "process exited", + "/proc/731/stat", + ), + ): + self.assertEqual( + linux_ops._busy_mount_references_proc("/mnt/colibri"), + [], + ) + + def test_busy_mount_scan_rejects_live_pid_with_missing_endpoint(self): + live_stat = "731 (worker) S 1 731 731 0 -1 0\n" + + for missing_leaf in ("cwd", "maps", "fd"): + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/fd": + if missing_leaf == "fd": + raise FileNotFoundError( + errno.ENOENT, + "leader endpoint unavailable", + path, + ) + return [] + raise AssertionError("unexpected listdir %s" % path) + + def read_link(path): + if path == "/proc/731/cwd" and missing_leaf == "cwd": + raise FileNotFoundError( + errno.ENOENT, + "leader endpoint unavailable", + path, + ) + return "/" + + def open_proc(path, *args, **kwargs): + del args, kwargs + if path == "/proc/731/stat": + return io.StringIO(live_stat) + if path == "/proc/731/maps": + if missing_leaf == "maps": + raise FileNotFoundError( + errno.ENOENT, + "leader endpoint unavailable", + path, + ) + return io.StringIO("") + raise AssertionError("unexpected open %s" % path) + + with self.subTest(endpoint=missing_leaf), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=read_link + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "missing process endpoint /proc/731/%s.*PID 731 remains" + % missing_leaf, + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_skips_kernel_thread_with_missing_cwd(self): + kernel_thread_stat = ( + "731 (kworker/0:1) I 2 0 0 0 -1 2097152\n" + ) + vanished_cwd = FileNotFoundError( + errno.ENOENT, + "kernel threads have no cwd", + "/proc/731/cwd", + ) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=vanished_cwd + ), mock.patch( + "builtins.open", return_value=io.StringIO(kernel_thread_stat) + ): + self.assertEqual( + linux_ops._busy_mount_references_proc("/mnt/colibri"), + [], + ) + + def test_busy_mount_scan_rejects_zombie_leader_with_live_sibling(self): + zombie_leader_stat = ( + "731 (worker) Z 1 731 731 0 -1 0 " + "0 0 0 0 0 0 0 0 20 0 2\n" + ) + vanished_cwd = FileNotFoundError( + errno.ENOENT, + "group leader exited", + "/proc/731/cwd", + ) + + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/task": + return ["731", "732"] + raise AssertionError("unexpected listdir %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=vanished_cwd + ), mock.patch( + "builtins.open", return_value=io.StringIO(zombie_leader_stat) + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "zombie/dead leader PID 731 still has sibling tasks 732", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_skips_proven_lone_zombie_leader(self): + zombie_leader_stat = ( + "731 (worker) Z 1 731 731 0 -1 0 " + "0 0 0 0 0 0 0 0 20 0 1\n" + ) + + def list_directory(path): + return ["731"] + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, + "readlink", + side_effect=FileNotFoundError(errno.ENOENT, "exited"), + ), mock.patch( + "builtins.open", return_value=io.StringIO(zombie_leader_stat) + ): + self.assertEqual( + linux_ops._busy_mount_references_proc("/mnt/colibri"), + [], + ) + + def test_busy_mount_scan_rejects_incomplete_zombie_task_snapshot(self): + cases = ( + (1, []), + (2, ["731"]), + (1, ["732"]), + ) + for num_threads, task_entries in cases: + zombie_leader_stat = ( + "731 (worker) Z 1 731 731 0 -1 0 " + "0 0 0 0 0 0 0 0 20 0 %d\n" % num_threads + ) + + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/task": + return task_entries + raise AssertionError("unexpected listdir %s" % path) + + with self.subTest( + num_threads=num_threads, + task_entries=task_entries, + ), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, + "readlink", + side_effect=FileNotFoundError(errno.ENOENT, "exited"), + ), mock.patch( + "builtins.open", + return_value=io.StringIO(zombie_leader_stat), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "incomplete process task snapshot", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_decodes_procfs_mapping_path_escapes(self): + mount_root = "/mnt/coli\\bri\tline\nroot" + mapped = (mount_root + "/model.bin").replace( + "\\", "\\134" + ).replace("\t", "\\011").replace("\n", "\\012") + maps = "1000-2000 r--p 00000000 00:01 7 %s\n" % mapped + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", return_value=["731"] + ), mock.patch.object( + linux_ops.os, "readlink", return_value="/" + ), mock.patch( + "builtins.open", return_value=io.StringIO(maps) + ): + self.assertEqual( + linux_ops._busy_mount_references_proc(mount_root), + [731], + ) + + def test_busy_mount_scan_finds_private_nonleader_task_references(self): + live_stat = ( + "731 (worker) S 1 731 731 0 -1 0 " + "0 0 0 0 0 0 0 0 20 0 2 0 100\n" + ) + mount_root = "/mnt/colibri" + mapped = "1000-2000 r--p 00000000 00:01 7 %s/model.bin\n" % mount_root + + for reference in ("cwd", "root", "exe", "maps", "fd"): + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/fd": + return [] + if path == "/proc/731/task": + return ["731", "732"] + if path == "/proc/731/task/732/fd": + return ["9"] if reference == "fd" else [] + raise AssertionError("unexpected listdir %s" % path) + + def read_link(path): + if path == "/proc/731/task/732/%s" % reference and reference in ( + "cwd", + "root", + "exe", + ): + return mount_root + "/weights" + if path == "/proc/731/task/732/fd/9": + return mount_root + "/weights" + return "/" + + def open_proc(path, *args, **kwargs): + del args, kwargs + if path == "/proc/731/stat": + return io.StringIO(live_stat) + if path == "/proc/731/maps": + return io.StringIO("") + if path == "/proc/731/task/732/maps": + return io.StringIO(mapped if reference == "maps" else "") + raise AssertionError("unexpected open %s" % path) + + with self.subTest(reference=reference), mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=read_link + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + self.assertEqual( + linux_ops._busy_mount_references_proc(mount_root), + [731], + ) + + def test_busy_mount_scan_rejects_incomplete_live_task_snapshot(self): + live_stat = ( + "731 (worker) S 1 731 731 0 -1 0 " + "0 0 0 0 0 0 0 0 20 0 2 0 100\n" + ) + + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/fd": + return [] + if path == "/proc/731/task": + return ["731"] + raise AssertionError("unexpected listdir %s" % path) + + def open_proc(path, *args, **kwargs): + del args, kwargs + if path == "/proc/731/stat": + return io.StringIO(live_stat) + if path == "/proc/731/maps": + return io.StringIO("") + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", return_value="/" + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "incomplete process task snapshot", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_rejects_live_nonleader_endpoint_disappearance(self): + live_stat = ( + "731 (worker) S 1 731 731 0 -1 0 " + "0 0 0 0 0 0 0 0 20 0 2 0 100\n" + ) + task_stat = live_stat.replace("731 (worker)", "732 (worker)", 1) + + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/fd": + return [] + if path == "/proc/731/task": + return ["731", "732"] + raise AssertionError("unexpected listdir %s" % path) + + def read_link(path): + if path == "/proc/731/task/732/cwd": + raise FileNotFoundError(errno.ENOENT, "task endpoint vanished", path) + return "/" + + def open_proc(path, *args, **kwargs): + del args, kwargs + if path == "/proc/731/stat": + return io.StringIO(live_stat) + if path == "/proc/731/maps": + return io.StringIO("") + if path == "/proc/731/task/732/stat": + return io.StringIO(task_stat) + raise AssertionError("unexpected open %s" % path) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=read_link + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "missing task endpoint .* task 732 .* remains live", + ): + linux_ops._busy_mount_references_proc("/mnt/colibri") + + def test_busy_mount_scan_preserves_non_utf8_leader_and_task_names(self): + leader_stat = ( + b"731 (leader-\xff) S 1 731 731 0 -1 0 " + b"0 0 0 0 0 0 0 0 20 0 2 0 100\n" + ) + task_stat = ( + b"732 (task-\xfe) Z 1 731 731 0 -1 0 " + b"0 0 0 0 0 0 0 0 20 0 1 0 101\n" + ) + stat_reads = [] + + def list_directory(path): + if path == "/proc": + return ["731"] + if path == "/proc/731/fd": + return [] + if path == "/proc/731/task": + return ["731", "732"] + raise AssertionError("unexpected listdir %s" % path) + + def read_link(path): + if path == "/proc/731/task/732/cwd": + raise FileNotFoundError( + errno.ENOENT, + "zombie task has no cwd", + path, + ) + return "/" + + def open_proc(path, mode, *, encoding, errors, newline=None): + self.assertEqual(mode, "r") + if path == "/proc/731/maps": + self.assertIsNone(newline) + return io.StringIO("") + self.assertEqual(newline, "") + if path == "/proc/731/stat": + stat_reads.append((path, errors)) + payload = leader_stat + elif path == "/proc/731/task/732/stat": + stat_reads.append((path, errors)) + payload = task_stat + else: + raise AssertionError("unexpected open %s" % path) + return io.TextIOWrapper( + io.BytesIO(payload), + encoding=encoding, + errors=errors, + newline=newline, + ) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops.os, "listdir", side_effect=list_directory + ), mock.patch.object( + linux_ops.os, "readlink", side_effect=read_link + ), mock.patch( + "builtins.open", side_effect=open_proc + ): + self.assertEqual( + linux_ops._busy_mount_references_proc("/mnt/colibri"), + [], + ) + + self.assertEqual( + stat_reads, + [ + ("/proc/731/stat", "surrogateescape"), + ("/proc/731/task/732/stat", "surrogateescape"), + ("/proc/731/stat", "surrogateescape"), + ], + ) + + def test_unprivileged_busy_scan_uses_trusted_fuser_without_shell(self): + trusted = mock.Mock( + side_effect=lambda name: "/usr/bin/" + name + ) + run = mock.Mock( + return_value=subprocess.CompletedProcess( + [], + 0, + " 732 731 732\n", + "/mnt/colibri: mmm\n", + ) + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + result = linux_ops._busy_mount_references( + "/mnt/colibri", + hardware=mock.sentinel.hardware, + run=run, + trusted_system_binary=trusted, + ) + + self.assertEqual(result, [731, 732]) + run.assert_called_once_with( + [ + "/usr/bin/sudo", + "--", + "/usr/bin/fuser", + "-mM", + "/mnt/colibri", + ], + timeout=10.0, + ) + self.assertEqual( + trusted.call_args_list, + [mock.call("fuser"), mock.call("sudo")], + ) + + def test_fuser_parser_distinguishes_empty_from_diagnostics(self): + empty = subprocess.CompletedProcess([], 1, "", "") + self.assertEqual( + linux_ops._parse_fuser_mount_references( + "/mnt/colibri", + empty, + ), + [], + ) + self.assertEqual( + linux_ops._parse_fuser_mount_references( + "/mnt/coli bri", + subprocess.CompletedProcess( + [], + 0, + "731 732\n", + "/mnt/coli bri: rcemF\n", + ), + ), + [731, 732], + ) + + failures = ( + subprocess.CompletedProcess( + [], + 1, + "", + "Specified filename is not a mountpoint", + ), + subprocess.CompletedProcess([], 2, "", "fatal"), + subprocess.CompletedProcess([], 0, "", "annotations"), + subprocess.CompletedProcess([], 0, "731 nope", "annotations"), + subprocess.CompletedProcess([], 0, "0", "annotations"), + subprocess.CompletedProcess( + [], + 0, + "731", + "/mnt/colibri: m\nCannot stat file: Permission denied\n", + ), + subprocess.CompletedProcess( + [], + 0, + "731", + "/mnt/colibri: m\nCannot open a network socket.\n", + ), + ) + for result in failures: + with self.subTest( + returncode=result.returncode, + stdout=result.stdout, + ): + with self.assertRaises(ramdisk.RamdiskError): + linux_ops._parse_fuser_mount_references( + "/mnt/colibri", + result, + ) + + def test_unprivileged_busy_scan_fails_closed_on_runner_exception(self): + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "trusted fuser could not inspect managed mount", + ): + linux_ops._busy_mount_references( + "/mnt/colibri", + run=mock.Mock( + side_effect=subprocess.TimeoutExpired("fuser", 10) + ), + trusted_system_binary=( + lambda name: "/usr/bin/" + name + ), + ) + + def test_prepare_cleanup_capability_resolves_fuser_sudo_and_umount(self): + trusted = mock.Mock( + side_effect=lambda name: "/usr/bin/" + name + ) + + def run_helper(command, **kwargs): + del kwargs + if command[-3:] == ["/usr/bin/fuser", "-mM", "/"]: + return subprocess.CompletedProcess( + command, + 0, + "731\n", + "/: m\n", + ) + if command[-2:] == ["/usr/bin/umount", "--help"]: + return subprocess.CompletedProcess( + command, + 0, + " -c, --no-canonicalize\n", + "", + ) + raise AssertionError("unexpected command %r" % (command,)) + + run = mock.Mock( + side_effect=run_helper + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ), linux_ops._noninteractive_privilege( + trusted_system_binary=trusted, + ): + linux_ops._ensure_busy_mount_scan_available( + "/mnt/colibri", + trusted_system_binary=trusted, + run=run, + ) + + self.assertEqual( + trusted.call_args_list, + [ + mock.call("fuser"), + mock.call("sudo"), + mock.call("umount"), + mock.call("sudo"), + ], + ) + self.assertEqual( + run.call_args_list, + [ + mock.call( + [ + "/usr/bin/sudo", + "-n", + "--", + "/usr/bin/fuser", + "-mM", + "/", + ], + timeout=10.0, + ), + mock.call( + [ + "/usr/bin/sudo", + "-n", + "--", + "/usr/bin/umount", + "--help", + ], + timeout=5.0, + ), + ], + ) + + def test_prepare_rejects_denied_privileged_umount_probe(self): + trusted = mock.Mock( + side_effect=lambda name: "/usr/bin/" + name + ) + + def run_helper(command, **kwargs): + del kwargs + if command[-3:] == ["/usr/bin/fuser", "-mM", "/"]: + return subprocess.CompletedProcess( + command, + 0, + "731\n", + "/: m\n", + ) + return subprocess.CompletedProcess( + command, + 1, + "", + "sudo: command not allowed\n", + ) + + run = mock.Mock(side_effect=run_helper) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "umount helper is incompatible or unauthorized", + ): + linux_ops._ensure_busy_mount_scan_available( + "/mnt/colibri", + trusted_system_binary=trusted, + run=run, + ) + + self.assertEqual( + run.call_args_list[-1], + mock.call( + [ + "/usr/bin/sudo", + "--", + "/usr/bin/umount", + "--help", + ], + timeout=5.0, + ), + ) + + def test_prepare_rejects_denied_fuser_probe_before_umount_check(self): + trusted = mock.Mock( + side_effect=lambda name: "/usr/bin/" + name + ) + denied = subprocess.CompletedProcess( + [], + 1, + "", + "sudo: a password is required\n", + ) + run = mock.Mock(return_value=denied) + + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "trusted fuser could not inspect managed mount /", + ): + linux_ops._ensure_busy_mount_scan_available( + "/mnt/colibri", + trusted_system_binary=trusted, + run=run, + ) + + self.assertEqual( + trusted.call_args_list, + [mock.call("fuser"), mock.call("sudo")], + ) + run.assert_called_once_with( + [ + "/usr/bin/sudo", + "--", + "/usr/bin/fuser", + "-mM", + "/", + ], + timeout=10.0, + ) + + def test_root_prepare_cleanup_capability_resolves_umount(self): + trusted = mock.Mock( + side_effect=lambda name: "/usr/bin/" + name + ) + run = mock.Mock( + return_value=subprocess.CompletedProcess( + [], 0, "--no-canonicalize", "" + ) + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=0 + ), mock.patch.object( + linux_ops, "_busy_mount_references_proc", return_value=[] + ) as scan: + linux_ops._ensure_busy_mount_scan_available( + "/mnt/colibri", + trusted_system_binary=trusted, + run=run, + ) + + scan.assert_called_once_with("/mnt/colibri") + trusted.assert_called_once_with("umount") + run.assert_called_once_with(["/usr/bin/umount", "--help"], timeout=5.0) + + def test_prepare_rejects_incompatible_umount_before_mount_mutation(self): + run = mock.Mock( + return_value=subprocess.CompletedProcess( + [], 0, "usage: umount TARGET", "" + ) + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=0 + ), mock.patch.object( + linux_ops, "_busy_mount_references_proc", return_value=[] + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "util-linux --no-canonicalize", + ): + linux_ops._ensure_busy_mount_scan_available( + "/mnt/colibri", + trusted_system_binary=lambda name: "/trusted/" + name, + run=run, + ) + + run.assert_called_once_with( + ["/trusted/umount", "--help"], + timeout=5.0, + ) + + def test_missing_fuser_names_the_required_psmisc_package(self): + trusted = mock.Mock( + side_effect=ramdisk.RamdiskError( + "trusted fuser executable was not found" + ) + ) + with mock.patch.object( + linux_ops, "_require_linux" + ), mock.patch.object( + linux_ops, "current_euid", return_value=1000 + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "install the psmisc package", + ): + linux_ops._ensure_busy_mount_scan_available( + "/mnt/colibri", + trusted_system_binary=trusted, + ) + + trusted.assert_called_once_with("fuser") + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_presentation_module.py b/c/tests/test_ramdisk_presentation_module.py new file mode 100644 index 000000000..fd57049b9 --- /dev/null +++ b/c/tests/test_ramdisk_presentation_module.py @@ -0,0 +1,88 @@ +"""Direct contracts for the dependency-injected presentation module.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import presentation + + +class PresentationModuleTest(unittest.TestCase): + def test_shared_and_replica_summaries_preserve_operator_consequences(self): + with ModelFixture() as fixture: + shared = ramdisk.build_plan( + plan_args(fixture.root), + hardware=hardware_fixture(nodes=2), + ) + replica = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), + hardware=hardware_fixture(nodes=2), + ) + + shared_summary = presentation._placement_summary(shared, base_port=9100) + replica_summary = presentation._placement_summary(replica, base_port=9100) + + self.assertEqual(shared_summary["copy_count"], 1) + self.assertEqual(shared_summary["ports"], [9100]) + self.assertEqual(replica_summary["copy_count"], 2) + self.assertEqual(replica_summary["ports"], [9100, 9101]) + self.assertIn("replication, not model sharding", replica_summary["explanation"]) + + def test_manifest_token_gets_persisted_port_through_its_callback(self): + manifest = { + "version": 1, + "deployment_id": "a" * 32, + "created_at": "2026-07-23T12:00:00+00:00", + "state": "ready", + "mounts": [], + "processes": [], + } + seen = [] + + def persisted_base_port(value): + seen.append(value) + return 9100 + + first = presentation._manifest_confirmation_token( + manifest, + persisted_base_port=persisted_base_port, + ) + second = presentation._manifest_confirmation_token( + manifest, + persisted_base_port=lambda _manifest: 9200, + ) + + self.assertEqual(seen, [manifest]) + self.assertNotEqual(first, second) + + def test_activity_rows_only_read_meminfo_for_a_present_workspace(self): + hardware = hardware_fixture() + + absent = presentation._tui_activity_rows( + {"present": False, "state": "absent"}, + hardware, + meminfo=lambda: self.fail("absent workspace must not probe meminfo"), + ) + active = presentation._tui_activity_rows( + { + "present": True, + "state": "ready", + "deep_validation": False, + "manifest_path": "/tmp/manifest.json", + "mounts": [], + "processes": [], + }, + hardware, + meminfo=lambda: {"Shmem": 4 * ramdisk.GIB}, + ) + + self.assertIn(("dim", "No RAM workspace exists yet."), absent) + self.assertEqual( + active[-1], + ("dim", "Host shared memory 4.00 GiB · swap 0.000 GiB"), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_presets.py b/c/tests/test_ramdisk_presets.py new file mode 100644 index 000000000..da2fae9f9 --- /dev/null +++ b/c/tests/test_ramdisk_presets.py @@ -0,0 +1,548 @@ +"""GPU discovery and first-run RAM-workspace preset contracts.""" + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import presets + + +def gpu_record(index, node, bus): + return { + "index": index, + "name": "GPU %d" % index, + "uuid": "GPU-test-%d" % index, + "pci_bus_id": bus, + "numa_node": node, + "locality": "resolved", + "total_bytes": 32 * ramdisk.GIB, + "free_bytes": 28 * ramdisk.GIB, + } + + +def gpu_hardware(available=128 * ramdisk.GIB): + hardware = hardware_fixture(available=available, nodes=4) + hardware["gpus"] = [ + gpu_record(0, 1, "0000:41:00.0"), + gpu_record(1, 3, "0000:c1:00.0"), + ] + hardware["gpu_discovery"] = { + "status": "available", + "error": None, + } + return hardware + + +class _GpuOps: + is_linux = True + + def __init__(self, nodes): + self.nodes = nodes + + @staticmethod + def executable_path(name): + return "/usr/bin/nvidia-smi" if name == "nvidia-smi" else None + + def read_text(self, path, default=None): + return self.nodes.get(path, default) + + +class GpuDiscoveryTest(unittest.TestCase): + def test_discovers_and_normalizes_gpu_pci_numa_locality(self): + output = ( + "0, RTX 5090, GPU-aaaa, 00000000:41:00.0, 32768, 30000\n" + "1, RTX 5090, GPU-bbbb, 00000000:C1:00.0, 32768, 29000\n" + ) + ops = _GpuOps( + { + "/sys/bus/pci/devices/0000:41:00.0/numa_node": "1\n", + "/sys/bus/pci/devices/0000:c1:00.0/numa_node": "3\n", + } + ) + run = mock.Mock( + return_value=argparse.Namespace( + returncode=0, + stdout=output, + stderr="", + ) + ) + + report = ramdisk._discover_gpus( + [0, 1, 2, 3], + ops=ops, + run=run, + ) + + self.assertEqual(report["status"], "available") + self.assertIn("uuid", run.call_args.args[0][1]) + self.assertEqual( + [ + ( + gpu["index"], + gpu["uuid"], + gpu["pci_bus_id"], + gpu["numa_node"], + ) + for gpu in report["devices"] + ], + [ + (0, "GPU-aaaa", "0000:41:00.0", 1), + (1, "GPU-bbbb", "0000:c1:00.0", 3), + ], + ) + self.assertTrue( + all(gpu["locality"] == "resolved" for gpu in report["devices"]) + ) + + def test_maps_minus_one_only_on_a_single_effective_node(self): + ops = _GpuOps( + {"/sys/bus/pci/devices/0000:01:00.0/numa_node": "-1\n"} + ) + run = mock.Mock( + return_value=argparse.Namespace( + returncode=0, + stdout=( + "0, GPU, GPU-aaaa, 0000:01:00.0, 1024, 512\n" + ), + stderr="", + ) + ) + + single = ramdisk._discover_gpus([7], ops=ops, run=run) + multiple = ramdisk._discover_gpus([0, 7], ops=ops, run=run) + + self.assertEqual(single["devices"][0]["numa_node"], 7) + self.assertEqual(single["devices"][0]["locality"], "single-node") + self.assertIsNone(multiple["devices"][0]["numa_node"]) + self.assertEqual(multiple["devices"][0]["locality"], "unknown") + + def test_query_failure_is_explicit_and_nonfatal(self): + ops = _GpuOps({}) + run = mock.Mock( + return_value=argparse.Namespace( + returncode=1, + stdout="", + stderr="Failed to initialize NVML", + ) + ) + + report = ramdisk._discover_gpus([0], ops=ops, run=run) + + self.assertEqual(report["status"], "unavailable") + self.assertEqual(report["devices"], []) + self.assertIn("Failed to initialize NVML", report["error"]) + + def test_records_ambient_cuda_visibility_as_selection_boundary(self): + ops = _GpuOps( + {"/sys/bus/pci/devices/0000:01:00.0/numa_node": "0\n"} + ) + run = mock.Mock( + return_value=argparse.Namespace( + returncode=0, + stdout=( + "0, GPU, GPU-aaaa, 0000:01:00.0, 1024, 512\n" + ), + stderr="", + ) + ) + + report = ramdisk._discover_gpus( + [0], + ops=ops, + run=run, + environ={"CUDA_VISIBLE_DEVICES": "0"}, + ) + + self.assertTrue(report["cuda_visible_devices_present"]) + self.assertEqual(report["cuda_visible_devices"], "0") + self.assertIn("relaunch", report["selection_error"]) + + +class PresetResolutionTest(unittest.TestCase): + def resolve( + self, + preset, + args, + hardware, + model, + cuda_capable=True, + ): + return presets.resolve_preset( + preset, + args, + hardware=hardware, + model=model, + build_plan=ramdisk.build_plan, + load_profile=ramdisk._load_profile, + cuda_capable=cuda_capable, + ) + + def test_gpu_fastest_selects_only_gpu_local_nodes_for_one_engine(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + result = self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args(fixture.root), + gpu_hardware(), + model, + ) + + args = result["args"] + plan = result["plan"] + self.assertEqual(args.memory_nodes, "1,3") + self.assertEqual(args.cpu_list, "2-3,6-7") + self.assertEqual(plan["topology"], "interleaved") + self.assertEqual(plan["staging"]["replica_count"], 1) + self.assertEqual(len(plan["placement"]["engine_cpu_sets"]), 1) + self.assertEqual( + [gpu["index"] for gpu in plan["managed_accelerator"]["devices"]], + [0, 1], + ) + self.assertTrue(plan["managed_accelerator"]["mmap"]) + self.assertFalse(plan["managed_accelerator"]["rammap"]) + self.assertEqual( + [gpu["uuid"] for gpu in plan["managed_accelerator"]["devices"]], + ["GPU-test-0", "GPU-test-1"], + ) + self.assertEqual( + plan["managed_accelerator"]["layout"], + "experts-only", + ) + self.assertIn("dense_tensor_bytes", plan["model"]) + + def test_gpu_fastest_uses_all_eligible_gpus_and_ignores_unusable_rows(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + hardware = gpu_hardware() + hardware["gpus"][0]["numa_node"] = None + hardware["gpus"][0]["locality"] = "unknown" + result = self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args(fixture.root), + hardware, + model, + ) + + self.assertIsNone(result["plan"]["preset"]["fallback"]) + self.assertEqual(result["args"].memory_nodes, "3") + self.assertEqual( + [ + gpu["index"] + for gpu in result["plan"]["managed_accelerator"]["devices"] + ], + [1], + ) + + def test_gpu_fastest_honors_exact_subset_and_dense_layout(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + result = self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args( + fixture.root, + gpu="1", + gpu_layout="dense-attention", + ), + gpu_hardware(), + model, + ) + + plan = result["plan"] + self.assertEqual(result["args"].memory_nodes, "3") + self.assertEqual( + [gpu["index"] for gpu in plan["managed_accelerator"]["devices"]], + [1], + ) + self.assertEqual( + plan["managed_accelerator"]["layout"], + "dense-attention", + ) + self.assertEqual( + plan["accelerator_projection"]["dense_gpu_bytes"], + plan["model"]["dense_tensor_bytes"], + ) + + def test_sharded_dense_layout_requires_multiple_selected_gpus(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "requires at least two", + ): + self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args( + fixture.root, + gpu="0", + gpu_layout="dense-attention-sharded", + ), + gpu_hardware(), + model, + ) + + def test_prepopulated_cuda_contract_cannot_bypass_topology_guard(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + hardware = gpu_hardware() + device = hardware["gpus"][0] + args = plan_args( + fixture.root, + topology="per-node", + managed_accelerator={ + "mode": "cuda", + "layout": "experts-only", + "devices": [dict(device)], + "mmap": True, + "rammap": False, + "async_copy": True, + "vram_budget": "auto", + "capability": "available", + }, + ) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "requires interleaved topology", + ): + ramdisk.build_plan( + args, + hardware=hardware, + model=model, + ) + + def test_explicit_unusable_gpu_is_rejected_instead_of_widened(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + hardware = gpu_hardware() + hardware["gpus"][0]["numa_node"] = None + hardware["gpus"][0]["locality"] = "unknown" + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "selects unusable NVIDIA device 0", + ): + self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args(fixture.root, gpu="0"), + hardware, + model, + ) + + def test_scriptable_plan_resolves_gpu_selector_without_a_preset(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + plan = ramdisk.build_plan( + plan_args( + fixture.root, + gpu="0", + gpu_layout="dense-attention", + ), + hardware=gpu_hardware(), + model=model, + ) + + self.assertEqual( + [gpu["index"] for gpu in plan["managed_accelerator"]["devices"]], + [0], + ) + self.assertEqual(plan["placement"]["memory_node_list"], "1") + self.assertEqual( + plan["managed_accelerator"]["layout"], + "dense-attention", + ) + + def test_custom_numa_masks_block_an_incompatible_gpu_change(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "outside the reviewed NUMA placement", + ): + ramdisk.build_plan( + plan_args( + fixture.root, + gpu="0", + memory_nodes="3", + cpu_list="6-7", + ), + hardware=gpu_hardware(), + model=model, + ) + + def test_dense_layout_blocks_obvious_gpu_capacity_failure(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + hardware = gpu_hardware() + hardware["gpus"][0]["free_bytes"] = ramdisk.GIB + plan = ramdisk.build_plan( + plan_args( + fixture.root, + gpu="0", + gpu_layout="dense-attention", + ), + hardware=hardware, + model=model, + ) + + self.assertIn( + "selected GPU free VRAM cannot hold the projected dense tensors " + "and per-device reserve", + plan["blockers"], + ) + self.assertEqual( + plan["accelerator_projection"]["expert_headroom_bytes"], + 0, + ) + + def test_dense_layout_checks_each_cards_balanced_projection(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + model["dense_tensor_bytes"] = 20 * ramdisk.GIB + hardware = gpu_hardware() + hardware["gpus"][0]["free_bytes"] = 3 * ramdisk.GIB + hardware["gpus"][1]["free_bytes"] = 30 * ramdisk.GIB + plan = ramdisk.build_plan( + plan_args( + fixture.root, + gpu="0,1", + gpu_layout="dense-attention", + ), + hardware=hardware, + model=model, + ) + + projection = plan["accelerator_projection"] + self.assertGreater( + projection["selected_free_bytes"], + projection["dense_gpu_bytes"] + + 2 * projection["vram_reserve_per_device_bytes"], + ) + self.assertFalse(projection["per_device"][0]["admission_ok"]) + self.assertTrue(projection["per_device"][1]["admission_ok"]) + self.assertIn( + "selected GPU free VRAM cannot hold the projected dense tensors " + "and per-device reserve", + plan["blockers"], + ) + self.assertIn("balanced estimate", " ".join(plan["warnings"])) + + def test_gpu_fastest_falls_back_to_single_when_locality_is_unavailable(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + hardware = hardware_fixture(nodes=4) + hardware["gpus"] = [] + hardware["gpu_discovery"] = { + "status": "unavailable", + "error": "NVML mismatch", + } + result = self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args(fixture.root), + hardware, + model, + ) + + self.assertEqual(result["args"].topology, "interleaved") + self.assertIsNone(result["args"].memory_nodes) + self.assertEqual(result["plan"]["preset"]["fallback"], "single") + self.assertIn("NVML mismatch", result["plan"]["preset"]["reason"]) + self.assertEqual(result["plan"]["managed_accelerator"]["mode"], "cpu") + + def test_gpu_fastest_falls_back_when_cuda_capability_is_unproven(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + result = self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args(fixture.root), + gpu_hardware(), + model, + cuda_capable=None, + ) + + self.assertEqual(result["plan"]["preset"]["fallback"], "single") + self.assertIn( + "CUDA engine capability could not be established", + result["plan"]["preset"]["reason"], + ) + self.assertEqual(result["plan"]["managed_accelerator"]["mode"], "cpu") + + def test_gpu_fastest_falls_back_to_partial_without_ever_replicating(self): + with ModelFixture() as fixture: + (fixture.root / ".coli_usage").write_text( + "0 1 100\n", + encoding="utf-8", + ) + model = ramdisk.scan_model(str(fixture.root)) + for shard in model["shards"]: + shard["size_bytes"] = 10 * ramdisk.GIB + model["total_shard_bytes"] = 20 * ramdisk.GIB + result = self.resolve( + presets.PRESET_GPU_FASTEST, + plan_args(fixture.root), + gpu_hardware(available=64 * ramdisk.GIB), + model, + ) + + self.assertEqual(result["args"].mode, "partial") + self.assertEqual(result["plan"]["mode"], "partial") + self.assertEqual(result["plan"]["topology"], "interleaved") + self.assertEqual(result["plan"]["staging"]["replica_count"], 1) + self.assertFalse( + any("replica" in blocker for blocker in result["plan"]["blockers"]) + ) + + def test_minimal_missing_profile_is_an_actionable_blocker(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + result = self.resolve( + presets.PRESET_MINIMAL, + plan_args(fixture.root), + hardware_fixture(), + model, + ) + + self.assertEqual(result["args"].mode, "partial") + self.assertTrue( + any( + "profile-guided staging is unavailable" in blocker + for blocker in result["plan"]["blockers"] + ) + ) + + def test_replicas_is_explicit_and_multiplies_engines(self): + with ModelFixture() as fixture: + model = ramdisk.scan_model(str(fixture.root)) + result = self.resolve( + presets.PRESET_REPLICAS, + plan_args(fixture.root), + hardware_fixture(nodes=4), + model, + ) + + self.assertEqual(result["plan"]["topology"], "per-node") + self.assertEqual(result["plan"]["staging"]["replica_count"], 4) + self.assertEqual( + len(result["plan"]["placement"]["engine_cpu_sets"]), + 4, + ) + + def test_advanced_edit_marks_custom_without_dropping_accelerator(self): + args = argparse.Namespace( + ramdisk_preset="gpu-fastest", + ramdisk_preset_label="Fastest GPU staging", + ramdisk_preset_reason="GPU-local nodes.", + ramdisk_preset_fallback=None, + managed_accelerator={"mode": "cuda", "devices": [{"index": 0}]}, + ) + accelerator = args.managed_accelerator + + presets.mark_preset_custom(args) + + self.assertEqual(args.ramdisk_preset, "custom") + self.assertEqual(args.ramdisk_preset_label, "Custom") + self.assertIn("Fastest GPU staging", args.ramdisk_preset_reason) + self.assertIs(args.managed_accelerator, accelerator) + + +if __name__ == "__main__": + unittest.main() diff --git a/c/tests/test_ramdisk_processes.py b/c/tests/test_ramdisk_processes.py new file mode 100644 index 000000000..07dcbbf49 --- /dev/null +++ b/c/tests/test_ramdisk_processes.py @@ -0,0 +1,2321 @@ +"""RAM-disk managed-process launch and cleanup tests.""" + +import linecache + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import lifecycle as lifecycle_support +from ramdisk_support import state as state_support + + +class ManagedLaunchTest(unittest.TestCase): + def _exercise_launch_line_interrupt( + self, + source_fragment, + *, + identity_mutation=None, + ): + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class LiveProcess: + pid = 7298 + + def poll(self): + return None + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired("engine", timeout) + + nonce = "6" * 48 + merge_id = "7" * 32 + process = LiveProcess() + identity = { + "pid": process.pid, + "pgid": process.pid, + "uid": host_uid(), + "starttime": 17298, + "nonce": nonce, + } + snapshots = [] + trace_hits = [] + verified_records = [] + + def trace_interrupt(frame, event, arg): + if ( + event == "line" + and frame.f_code is lifecycle_support.start.__code__ + and source_fragment + in linecache.getline(frame.f_code.co_filename, frame.f_lineno) + ): + trace_hits.append(frame.f_lineno) + sys.settrace(None) + raise KeyboardInterrupt("line-boundary interruption") + return trace_interrupt + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ): + expected_state_root = ramdisk._state_root() + expected_manifest_path = ramdisk._manifest_path() + expected_benchmarks_path = ramdisk._benchmarks_path() + plan["durable_state"] = { + "root": expected_state_root, + "manifest": expected_manifest_path, + "benchmarks": expected_benchmarks_path, + } + mount = dict(plan["mounts"][0]) + mount.update( + { + "ownership": "managed", + "identity": {"mount_id": 4, "device": "0:9"}, + } + ) + manifest = { + "version": ramdisk.MANIFEST_VERSION, + "deployment_id": "8" * 32, + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [mount], + "processes": [], + "ports": [], + } + identity.update( + { + "inert": False, + "sid": process.pid, + "state_dir": os.path.join( + expected_state_root, + "engines", + plan["model"]["fingerprint"].split(":", 1)[-1], + "interleaved", + ), + "weights_dir": mount["path"], + } + ) + if identity_mutation is not None: + identity = identity_mutation(dict(identity)) + + def save(current): + snapshot = json.loads(json.dumps(current)) + snapshots.append(snapshot) + ramdisk._atomic_json(ramdisk._manifest_path(), snapshot) + + caught = None + saved_manifest_path = None + saved_state_root = None + saved_benchmarks_path = None + def terminate_verified(record): + verified_records.append( + json.loads(json.dumps(record)) + ) + return "process group survived SIGKILL" + + clock = iter((0.0, 0.0, 2.0)) + monotonic_patch = ( + mock.patch.object( + lifecycle_support.time, + "monotonic", + side_effect=lambda: next(clock, 2.0), + ) + if identity_mutation is not None + else contextlib.nullcontext() + ) + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.multiple( + ramdisk, + _filesystem_for_path=mock.Mock(return_value="ext4"), + _load_manifest=mock.Mock(return_value=manifest), + _assert_effective_masks_unchanged=mock.Mock(), + _assert_ready_mounts=mock.Mock(), + _save_manifest=mock.Mock(side_effect=save), + _admit_concurrent_runtimes=mock.Mock(), + _recover_delta=mock.Mock(), + _usage_read=mock.Mock(return_value={}), + _usage_write=mock.Mock(), + _proc_identity=mock.Mock(return_value=identity), + _wait_managed_ready=mock.Mock(), + _process_matches=mock.Mock( + return_value=(True, "running", identity) + ), + _terminate_verified_group=mock.Mock( + side_effect=terminate_verified + ), + _terminate_direct_child=mock.Mock( + return_value="direct child survived SIGKILL" + ), + _group_alive=mock.Mock(return_value=True), + _track_managed_child=mock.Mock(), + _forget_managed_child=mock.Mock(), + _merge_usage=mock.Mock(), + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk.subprocess, "Popen", return_value=process + ), mock.patch.object( + ramdisk.secrets, + "token_hex", + side_effect=lambda size: nonce if size == 24 else merge_id, + ), monotonic_patch: + try: + sys.settrace(trace_interrupt) + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + except BaseException as exc: + caught = exc + finally: + sys.settrace(None) + saved_manifest_path = ramdisk._manifest_path() + saved_state_root = ramdisk._state_root() + saved_benchmarks_path = ramdisk._benchmarks_path() + + load_error = None + try: + state_support._load_manifest( + required=True, + filesystem_for_path=lambda ignored: "ext4", + read_json=ramdisk._read_json, + manifest_path=lambda: saved_manifest_path, + state_root=lambda: saved_state_root, + benchmarks_path=lambda: saved_benchmarks_path, + assert_durable_state_dir=lambda path, plan=None: None, + uid_provider=host_uid, + ) + except ramdisk.RamdiskError as exc: + load_error = exc + + return ( + manifest, + snapshots, + trace_hits, + caught, + load_error, + verified_records, + ) + + def _launch_authorities(self, manifest): + return [ + (kind, entry["state_dir"]) + for kind, entries in ( + ("pending", manifest.get("pending_launches", [])), + ("published", manifest.get("processes", [])), + ( + "retained", + manifest.get("recovery", {}).get( + "retained_processes", [] + ), + ), + ) + for entry in entries + ] + + def _exact_launch_identity(self, identity, plan, node=None): + mount = next( + record + for record in plan["mounts"] + if record.get("node") == node + ) + label = "interleaved" if node is None else "node-%d" % node + return dict( + identity, + inert=False, + sid=identity["pid"], + state_dir=os.path.join( + ramdisk._state_root(), + "engines", + plan["model"]["fingerprint"].split(":", 1)[-1], + label, + ), + weights_dir=mount["path"], + ) + + def _exercise_prepublication_popen_outcome( + self, + *, + popen_effect=None, + popen_factory=None, + cancel_after_pending=False, + log_open_effect=None, + terminate_direct_child_effect=None, + group_alive_effect=None, + merge_effect=None, + ): + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + nonce = "d" * 48 + merge_id = "e" * 32 + cancel = threading.Event() + snapshots = [] + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + manifest = { + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + "ports": [], + } + + def save(current): + snapshot = json.loads(json.dumps(current)) + snapshots.append(snapshot) + if cancel_after_pending and snapshot.get("pending_launches"): + cancel.set() + + merge = mock.Mock(side_effect=merge_effect) + if popen_factory is None: + popen = mock.Mock(side_effect=popen_effect) + else: + self.assertIsNone(popen_effect) + popen = popen_factory + terminate_direct_child = mock.Mock( + side_effect=terminate_direct_child_effect + ) + group_alive = ( + mock.Mock(return_value=False) + if group_alive_effect is None + else mock.Mock(side_effect=group_alive_effect) + ) + real_open = open + caught = None + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.multiple( + ramdisk, + _filesystem_for_path=mock.Mock(return_value="ext4"), + _load_manifest=mock.Mock(return_value=manifest), + _assert_effective_masks_unchanged=mock.Mock(), + _assert_ready_mounts=mock.Mock(), + _save_manifest=mock.Mock(side_effect=save), + _admit_concurrent_runtimes=mock.Mock(), + _recover_delta=mock.Mock(), + _usage_read=mock.Mock(return_value={}), + _usage_write=mock.Mock(), + _process_matches=mock.Mock(), + _terminate_verified_group=mock.Mock(), + _terminate_direct_child=terminate_direct_child, + _group_alive=group_alive, + _track_managed_child=mock.Mock(), + _forget_managed_child=mock.Mock(), + _merge_usage=merge, + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk.subprocess, "Popen", popen + ), mock.patch.object( + ramdisk.secrets, + "token_hex", + side_effect=lambda size: nonce if size == 24 else merge_id, + ), mock.patch.object( + lifecycle_support, + "open", + side_effect=( + log_open_effect + if log_open_effect is not None + else real_open + ), + create=True, + ): + try: + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + cancel_event=cancel, + ) + except BaseException as exc: + caught = exc + + return ( + manifest, + snapshots, + popen, + merge, + terminate_direct_child, + group_alive, + caught, + ) + + def _exercise_exact_popen_line_interrupt( + self, + *, + target_code, + source_fragment, + constructor_error=False, + ): + real_popen = subprocess.Popen + attempts = [] + trace_hits = [] + + class ProbePopen(real_popen): + def __init__(self, _command, **kwargs): + super().__init__( + [ + sys.executable, + "-c", + "import time; time.sleep(30)", + ], + **kwargs, + ) + attempts.append(self) + if constructor_error: + raise OSError("post-init constructor failure") + + def reap_attempts(): + for process in attempts: + if process.poll() is None: + process.kill() + process.wait(timeout=2) + + self.addCleanup(reap_attempts) + + def terminate_direct_child(process): + self.assertIs(process, attempts[-1]) + process.kill() + return None + + def group_alive(pgid): + self.assertEqual(pgid, attempts[-1].pid) + self.assertIsNotNone(attempts[-1].poll()) + return False + + def trace_interrupt(frame, event, arg): + if ( + event == "line" + and frame.f_code is target_code + and source_fragment + in linecache.getline(frame.f_code.co_filename, frame.f_lineno) + ): + trace_hits.append(frame.f_lineno) + sys.settrace(None) + raise KeyboardInterrupt("exact Popen handle boundary") + return trace_interrupt + + try: + sys.settrace(trace_interrupt) + result = self._exercise_prepublication_popen_outcome( + popen_factory=ProbePopen, + terminate_direct_child_effect=terminate_direct_child, + group_alive_effect=group_alive, + ) + finally: + sys.settrace(None) + return result, attempts, trace_hits + + def test_launch_rollback_keeps_live_direct_child_when_proc_identity_is_inconclusive(self): + class LiveProcess: + pid = 7100 + + def poll(self): + return None + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired("engine", timeout) + + record = {"pid": 7100, "pgid": 7100} + forget = mock.Mock() + direct_terminate = mock.Mock( + return_value="direct child PID 7100 survived SIGKILL" + ) + failures, surviving = lifecycle_support._rollback_launched_children( + [LiveProcess()], + [record], + process_matches=lambda ignored: ( + False, + "identity-unavailable", + None, + ), + group_alive=mock.Mock(return_value=True), + track_managed_child=mock.Mock(), + terminate_verified_group=lambda ignored: ( + "cannot revalidate managed process identity" + ), + terminate_direct_child=direct_terminate, + forget_managed_child=forget, + ) + + self.assertEqual(surviving, {7100}) + self.assertIn("cannot revalidate", failures[0]) + self.assertIn("direct child is still alive", failures[0]) + direct_terminate.assert_called_once() + forget.assert_not_called() + + def test_launch_rollback_discards_termination_failure_only_after_proven_absence(self): + class ExitedProcess: + pid = 7200 + + def poll(self): + return 17 + + def wait(self, timeout=None): + return 17 + + forget = mock.Mock() + failures, surviving = lifecycle_support._rollback_launched_children( + [ExitedProcess()], + [{"pid": 7200, "pgid": 7200}], + process_matches=lambda ignored: ( + False, + "not-running", + None, + ), + group_alive=mock.Mock(return_value=False), + track_managed_child=mock.Mock(), + terminate_verified_group=lambda ignored: "late SIGKILL timeout", + terminate_direct_child=mock.Mock(), + forget_managed_child=forget, + ) + + self.assertEqual(failures, []) + self.assertEqual(surviving, set()) + forget.assert_called_once_with(7200) + + def test_launch_rollback_retains_forked_engine_after_wrapper_exits(self): + class ExitedWrapper: + pid = 7250 + + def poll(self): + return 1 + + def wait(self, timeout=None): + return 1 + + forget = mock.Mock() + failures, surviving = lifecycle_support._rollback_launched_children( + [ExitedWrapper()], + [{"pid": 7250, "pgid": 7250}], + process_matches=lambda ignored: ( + True, + "running-group", + {"pgid": 7250, "members": [{"pid": 7251}]}, + ), + group_alive=mock.Mock(return_value=True), + track_managed_child=mock.Mock(), + terminate_verified_group=lambda ignored: ( + "process group 7250 survived SIGKILL" + ), + terminate_direct_child=mock.Mock(), + forget_managed_child=forget, + ) + + self.assertEqual(surviving, {7250}) + self.assertIn("persisted process identity is still running", failures[0]) + forget.assert_not_called() + + def test_launch_rollback_retains_unpublished_group_after_wrapper_exits(self): + class ExitedWrapper: + pid = 7275 + + def poll(self): + return 1 + + def wait(self, timeout=None): + return 1 + + context = { + "state_dir": "/state/unpublished", + "usage_baseline": {}, + "record": None, + } + group_alive = mock.Mock(return_value=True) + forget = mock.Mock() + failures, surviving = lifecycle_support._rollback_launched_children( + [ExitedWrapper()], + [], + process_matches=mock.Mock(), + group_alive=group_alive, + track_managed_child=mock.Mock(), + terminate_verified_group=mock.Mock(), + terminate_direct_child=mock.Mock(return_value=None), + forget_managed_child=forget, + launch_contexts=[context], + ) + + self.assertEqual(surviving, {7275}) + self.assertIn( + "direct-created process group 7275 is still alive", + failures[0], + ) + self.assertTrue(context["rollback_process_alive"]) + self.assertEqual(context["rollback_pid"], 7275) + group_alive.assert_called_once_with(7275) + forget.assert_not_called() + + def test_launch_rollback_treats_interrupted_group_scan_as_unproven(self): + class ExitedWrapper: + pid = 7280 + + def poll(self): + return 1 + + def wait(self, timeout=None): + return 1 + + context = { + "state_dir": "/state/interrupted-scan", + "usage_baseline": {}, + "record": None, + } + failures, surviving = lifecycle_support._rollback_launched_children( + [ExitedWrapper()], + [], + process_matches=mock.Mock(), + group_alive=mock.Mock(side_effect=KeyboardInterrupt()), + track_managed_child=mock.Mock(), + terminate_verified_group=mock.Mock(), + terminate_direct_child=mock.Mock(return_value=None), + forget_managed_child=mock.Mock(), + launch_contexts=[context], + ) + + self.assertEqual(surviving, {7280}) + self.assertIn( + "could not establish direct-created process group 7280 absence", + failures[0], + ) + self.assertTrue(context["rollback_process_alive"]) + + def test_pending_launch_is_durable_until_exact_process_promotion(self): + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class LiveProcess: + pid = 7290 + + def poll(self): + return None + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired("engine", timeout) + + nonce = "a" * 48 + merge_id = "b" * 32 + process = LiveProcess() + identity = { + "pid": process.pid, + "pgid": process.pid, + "uid": host_uid(), + "starttime": 17290, + "nonce": nonce, + } + snapshots = [] + successful = [] + + def save(current): + snapshot = json.loads(json.dumps(current)) + snapshots.append(snapshot) + if snapshot.get("processes") and not snapshot.get( + "pending_launches" + ): + raise OSError("exact process promotion write failed") + successful.append(snapshot) + + def popen(*args, **kwargs): + pending = successful[-1]["pending_launches"][0] + self.assertEqual(successful[-1]["processes"], []) + self.assertEqual(pending["nonce"], nonce) + self.assertEqual(pending["usage_merge_id"], merge_id) + self.assertEqual(pending["operation_id"], "start:" + merge_id) + return process + + def proc_identity(_pid): + # Popen has returned, but exact PID/PGID publication has not. A + # hard crash here leaves the durable pending record intact. + self.assertEqual(successful[-1]["processes"], []) + self.assertEqual( + successful[-1]["pending_launches"][0]["state_dir"], + snapshots[-1]["pending_launches"][0]["state_dir"], + ) + return dict( + identity, + inert=False, + sid=process.pid, + state_dir=snapshots[-1]["pending_launches"][0]["state_dir"], + weights_dir=snapshots[-1]["pending_launches"][0][ + "weights_dir" + ], + ) + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + manifest = { + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + "ports": [], + } + merge = mock.Mock() + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.multiple( + ramdisk, + _filesystem_for_path=mock.Mock(return_value="ext4"), + _load_manifest=mock.Mock(return_value=manifest), + _assert_effective_masks_unchanged=mock.Mock(), + _assert_ready_mounts=mock.Mock(), + _save_manifest=mock.Mock(side_effect=save), + _admit_concurrent_runtimes=mock.Mock(), + _recover_delta=mock.Mock(), + _usage_read=mock.Mock(return_value={}), + _usage_write=mock.Mock(), + _proc_identity=mock.Mock(side_effect=proc_identity), + _wait_managed_ready=mock.Mock(), + _process_matches=mock.Mock(), + _terminate_verified_group=mock.Mock(), + _terminate_direct_child=mock.Mock( + return_value="direct child PID 7290 survived SIGKILL" + ), + _group_alive=mock.Mock(return_value=True), + _track_managed_child=mock.Mock(), + _forget_managed_child=mock.Mock(), + _merge_usage=merge, + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk.subprocess, "Popen", side_effect=popen + ), mock.patch.object( + ramdisk.secrets, + "token_hex", + side_effect=lambda size: nonce if size == 24 else merge_id, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "exact process promotion write failed.*direct child", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + merge.assert_not_called() + self.assertEqual(manifest["state"], "error") + self.assertEqual(len(manifest["processes"]), 1) + self.assertEqual(manifest["pending_launches"], []) + self.assertFalse( + manifest.get("recovery", {}).get("retained_processes") + ) + published = manifest["processes"][0] + self.assertEqual(published["pid"], process.pid) + self.assertEqual(published["uid"], host_uid()) + self.assertEqual(published["starttime"], 17290) + self.assertEqual(published["nonce"], nonce) + self.assertEqual(published["usage_baseline"], {}) + self.assertEqual(published["usage_merge_id"], merge_id) + + def test_cancel_after_pending_save_is_rechecked_before_popen(self): + manifest, snapshots, popen, _merge, _terminate, _group, caught = ( + self._exercise_prepublication_popen_outcome( + cancel_after_pending=True + ) + ) + + self.assertIsInstance(caught, ramdisk._OperationCancelled) + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + popen.assert_not_called() + self.assertEqual(manifest["state"], "ready") + self.assertEqual(manifest["pending_launches"], []) + self.assertEqual(manifest["processes"], []) + + def test_pre_spawn_log_open_oserror_clears_pending_launch(self): + manifest, snapshots, popen, _merge, _terminate, _group, caught = ( + self._exercise_prepublication_popen_outcome( + log_open_effect=OSError("log open failed") + ) + ) + + self.assertIsInstance(caught, OSError) + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + popen.assert_not_called() + self.assertEqual(manifest["state"], "error") + self.assertEqual(manifest["pending_launches"], []) + self.assertEqual(manifest["processes"], []) + + def test_mocked_popen_oserror_retains_unknown_without_inspected_attempt(self): + manifest, snapshots, popen, merge, _terminate, _group, caught = ( + self._exercise_prepublication_popen_outcome( + popen_effect=OSError("parent-side Popen failure") + ) + ) + + self.assertIsInstance(caught, ramdisk.RamdiskError) + self.assertIn("process creation outcome is unknown", str(caught)) + self.assertIsInstance(caught.__cause__, OSError) + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + popen.assert_called_once() + merge.assert_not_called() + self.assertEqual(manifest["state"], "error") + self.assertEqual(manifest["processes"], []) + self.assertEqual(len(manifest["pending_launches"]), 1) + + def test_inspected_prefork_popen_exception_proves_child_absence(self): + real_popen = subprocess.Popen + + class PreForkFailurePopen(real_popen): + def _execute_child(self, *args, **kwargs): + del args, kwargs + raise OSError("inspected pre-fork failure") + + ( + manifest, + snapshots, + _popen, + merge, + terminate, + group, + caught, + ) = self._exercise_prepublication_popen_outcome( + popen_factory=PreForkFailurePopen + ) + + self.assertIsInstance(caught, OSError) + self.assertEqual(str(caught), "inspected pre-fork failure") + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + terminate.assert_not_called() + group.assert_not_called() + merge.assert_called_once() + self.assertEqual(manifest["state"], "error") + self.assertEqual(manifest["processes"], []) + self.assertEqual(manifest["pending_launches"], []) + + @requires_linux_operational + def test_postfork_popen_exception_retains_and_reaps_exact_child(self): + real_popen = subprocess.Popen + attempts = [] + attempt_owned_devnull = [] + child_stdin_streams = [] + + class PostForkFailurePopen(real_popen): + def __init__(self, _command, **kwargs): + child_stdin_streams.append(kwargs["stdin"]) + super().__init__( + [ + sys.executable, + "-c", + "import time; time.sleep(30)", + ], + **kwargs, + ) + + def _close_pipe_fds(self, *args): + super()._close_pipe_fds(*args) + attempts.append(self) + attempt_owned_devnull.append(hasattr(self, "_devnull")) + raise OSError("parent-side post-fork failure") + + def reap_attempt(): + if not attempts: + return + process = attempts[-1] + if process.poll() is None: + process.kill() + process.wait(timeout=2) + + self.addCleanup(reap_attempt) + + def terminate_direct_child(process): + self.assertIs(process, attempts[-1]) + process.kill() + return None + + ( + manifest, + snapshots, + _popen, + merge, + terminate, + group, + caught, + ) = self._exercise_prepublication_popen_outcome( + popen_factory=PostForkFailurePopen, + terminate_direct_child_effect=terminate_direct_child, + ) + + self.assertEqual(len(attempts), 1) + self.assertEqual(attempt_owned_devnull, [False]) + self.assertEqual(len(child_stdin_streams), 1) + self.assertTrue(child_stdin_streams[0].closed) + process = attempts[0] + self.assertIsInstance(caught, OSError) + self.assertEqual(str(caught), "parent-side post-fork failure") + self.assertTrue(process._child_created) + self.assertIsNotNone(process.returncode) + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + terminate.assert_called_once_with(process) + group.assert_called_once_with(process.pid) + merge.assert_called_once() + self.assertEqual(manifest["state"], "error") + self.assertEqual(manifest["processes"], []) + self.assertEqual(manifest["pending_launches"], []) + + @requires_linux_operational + def test_exact_popen_attempt_is_reaped_across_registration_boundaries(self): + cases = ( + ( + "helper-success-return", + lifecycle_support._construct_retained_popen.__code__, + "return attempt, None, True", + False, + ), + ( + "helper-error-return", + lifecycle_support._construct_retained_popen.__code__, + "return attempt, exc, True", + True, + ), + ( + "caller-success-registration", + lifecycle_support.start.__code__, + "spawned.append(process)", + False, + ), + ( + "caller-error-registration", + lifecycle_support.start.__code__, + "spawned.append(process)", + True, + ), + ) + for name, target_code, source_fragment, constructor_error in cases: + with self.subTest(boundary=name): + ( + result, + attempts, + trace_hits, + ) = self._exercise_exact_popen_line_interrupt( + target_code=target_code, + source_fragment=source_fragment, + constructor_error=constructor_error, + ) + ( + manifest, + snapshots, + _popen, + merge, + terminate, + group, + caught, + ) = result + + self.assertEqual(len(trace_hits), 1) + self.assertEqual(len(attempts), 1) + process = attempts[0] + self.assertIsInstance(caught, KeyboardInterrupt) + self.assertIsNotNone(process.returncode) + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + terminate.assert_called_once_with(process) + group.assert_called_once_with(process.pid) + merge.assert_called_once() + self.assertEqual(manifest["state"], "error") + self.assertEqual(manifest["processes"], []) + self.assertEqual(manifest["pending_launches"], []) + self.assertFalse( + manifest.get("recovery", {}).get("retained_processes") + ) + + def test_async_popen_interruption_retains_outcome_unknown_pending_launch(self): + manifest, snapshots, popen, merge, _terminate, _group, caught = ( + self._exercise_prepublication_popen_outcome( + popen_effect=KeyboardInterrupt("asynchronous interrupt") + ) + ) + + self.assertIsInstance(caught, ramdisk.RamdiskError) + self.assertIn("process creation outcome is unknown", str(caught)) + self.assertIsInstance(caught.__cause__, KeyboardInterrupt) + self.assertTrue( + any(snapshot.get("pending_launches") for snapshot in snapshots) + ) + popen.assert_called_once() + merge.assert_not_called() + self.assertEqual(manifest["state"], "error") + self.assertEqual(manifest["processes"], []) + self.assertEqual(len(manifest["pending_launches"]), 1) + pending = manifest["pending_launches"][0] + self.assertEqual(pending["operation_id"], "start:" + "e" * 32) + self.assertEqual(pending["nonce"], "d" * 48) + self.assertEqual(pending["usage_merge_id"], "e" * 32) + + def test_log_close_failure_rolls_back_returned_process_before_merge(self): + events = [] + + class ReturnedProcess: + pid = 7295 + alive = True + + def poll(self): + return None if self.alive else 0 + + def wait(self, timeout=None): + if self.alive: + raise subprocess.TimeoutExpired("engine", timeout) + return 0 + + class FaultyLog: + def close(self): + events.append("close") + raise OSError("log close failed") + + process = ReturnedProcess() + + def terminate_direct_child(child): + self.assertIs(child, process) + events.append("terminate") + child.alive = False + + def group_alive(pgid): + self.assertEqual(pgid, process.pid) + self.assertFalse(process.alive) + events.append("absence-proven") + return False + + def merge_usage(*args, **kwargs): + self.assertIn("absence-proven", events) + events.append("merge") + + ( + manifest, + _snapshots, + popen, + merge, + terminate, + group, + caught, + ) = self._exercise_prepublication_popen_outcome( + popen_effect=lambda *args, **kwargs: process, + log_open_effect=lambda *args, **kwargs: FaultyLog(), + terminate_direct_child_effect=terminate_direct_child, + group_alive_effect=group_alive, + merge_effect=merge_usage, + ) + + self.assertIsInstance(caught, OSError) + popen.assert_called_once() + terminate.assert_called_once_with(process) + group.assert_called_once_with(process.pid) + merge.assert_called_once() + self.assertLess(events.index("terminate"), events.index("merge")) + self.assertLess(events.index("absence-proven"), events.index("merge")) + self.assertEqual(manifest["pending_launches"], []) + self.assertEqual(manifest["processes"], []) + + def test_interrupt_after_handle_registration_keeps_one_loadable_authority(self): + manifest, _snapshots, hits, caught, load_error, _verified = ( + self._exercise_launch_line_interrupt( + 'context["spawn_outcome"] = "created"' + ) + ) + + self.assertTrue(hits) + self.assertIsInstance(caught, ramdisk.RamdiskError) + self.assertIsNone(load_error) + authorities = self._launch_authorities(manifest) + self.assertEqual(len(authorities), 1) + self.assertEqual(authorities[0][0], "retained") + + def test_interrupt_after_process_publication_keeps_published_authority_only(self): + manifest, _snapshots, hits, caught, load_error, _verified = ( + self._exercise_launch_line_interrupt("records.append(record)") + ) + + self.assertTrue(hits) + self.assertIsInstance(caught, ramdisk.RamdiskError) + self.assertIsNone(load_error) + authorities = self._launch_authorities(manifest) + self.assertEqual(len(authorities), 1) + self.assertEqual(authorities[0][0], "published") + + def test_post_replace_interrupt_does_not_downgrade_exact_authority(self): + real_fsync_directory = state_support._fsync_bound_directory + interrupted = [] + + def interrupt_after_promotion_replace(descriptor): + real_fsync_directory(descriptor) + if interrupted: + return + durable = ramdisk._read_json(ramdisk._manifest_path()) + if durable and durable.get("processes") and not durable.get( + "pending_launches" + ): + interrupted.append(ramdisk._manifest_path()) + raise KeyboardInterrupt( + "after exact process authority replacement" + ) + + with mock.patch.object( + state_support, + "_fsync_bound_directory", + side_effect=interrupt_after_promotion_replace, + ): + ( + manifest, + _snapshots, + hits, + caught, + load_error, + verified_records, + ) = self._exercise_launch_line_interrupt( + "fragment-that-does-not-exist" + ) + + self.assertTrue(interrupted) + self.assertEqual(hits, []) + self.assertIsInstance(caught, ramdisk.RamdiskError) + self.assertIsNone(load_error) + self.assertEqual( + [kind for kind, _ in self._launch_authorities(manifest)], + ["published"], + ) + self.assertEqual(len(verified_records), 1) + published = manifest["processes"][0] + self.assertEqual( + { + key: verified_records[0][key] + for key in ( + "pid", + "pgid", + "uid", + "starttime", + "nonce", + "state_dir", + "weights_dir", + "usage_merge_id", + ) + }, + { + key: published[key] + for key in ( + "pid", + "pgid", + "uid", + "starttime", + "nonce", + "state_dir", + "weights_dir", + "usage_merge_id", + ) + }, + ) + self.assertEqual(published["uid"], host_uid()) + self.assertEqual(published["starttime"], 17298) + self.assertEqual(published["nonce"], "6" * 48) + + def test_launch_promotion_requires_complete_observed_identity(self): + valid = { + "pid": 7298, + "uid": host_uid(), + "inert": False, + "starttime": 17298, + "nonce": "6" * 48, + "pgid": 7298, + "sid": 7298, + "state_dir": "/state/exact", + "weights_dir": "/weights/exact", + } + contract = { + "pid": 7298, + "uid": host_uid(), + "nonce": "6" * 48, + "state_dir": "/state/exact", + "weights_dir": "/weights/exact", + } + self.assertTrue( + lifecycle_support._launch_identity_matches(valid, **contract) + ) + cases = { + "not-a-dict": None, + "pid": dict(valid, pid=7299), + "uid": dict(valid, uid=host_uid() + 1), + "starttime-zero": dict(valid, starttime=0), + "starttime-bool": dict(valid, starttime=True), + "inert": dict(valid, inert=True), + "inert-missing": { + key: value for key, value in valid.items() if key != "inert" + }, + "nonce": dict(valid, nonce="7" * 48), + "pgid": dict(valid, pgid=7299), + "sid": dict(valid, sid=7299), + "state-dir": dict(valid, state_dir="/state/foreign"), + "weights-dir": dict(valid, weights_dir="/weights/foreign"), + } + for case, identity in cases.items(): + with self.subTest(case=case): + self.assertFalse( + lifecycle_support._launch_identity_matches( + identity, + **contract, + ) + ) + + def test_mismatched_launch_identity_retains_pending_group_authority(self): + cases = { + "pid": lambda value: dict(value, pid=value["pid"] + 1), + "uid": lambda value: dict(value, uid=value["uid"] + 1), + "starttime": lambda value: dict(value, starttime=0), + "inert": lambda value: dict(value, inert=True), + "nonce": lambda value: dict(value, nonce="8" * 48), + "pgid": lambda value: dict(value, pgid=value["pgid"] + 1), + "sid": lambda value: dict(value, sid=value["sid"] + 1), + "state-dir": lambda value: dict( + value, + state_dir=value["state_dir"] + "-foreign", + ), + "weights-dir": lambda value: dict( + value, + weights_dir=value["weights_dir"] + "-foreign", + ), + } + for case, mutation in cases.items(): + with self.subTest(case=case): + ( + manifest, + _snapshots, + hits, + caught, + load_error, + verified_records, + ) = self._exercise_launch_line_interrupt( + "fragment-that-does-not-exist", + identity_mutation=mutation, + ) + self.assertEqual(hits, []) + self.assertIsInstance(caught, ramdisk.RamdiskError) + self.assertIsNone(load_error) + self.assertEqual(manifest["processes"], []) + self.assertEqual(len(manifest["pending_launches"]), 1) + pending = manifest["pending_launches"][0] + self.assertEqual( + pending["observed_group"]["pgid"], + 7298, + ) + self.assertEqual( + pending["observed_group"]["uid"], + host_uid(), + ) + self.assertFalse( + manifest.get("recovery", {}).get("retained_processes") + ) + self.assertEqual(verified_records, []) + self.assertNotIn("usage_merged_at", pending) + + def test_failed_launch_never_merges_usage_while_direct_child_is_alive(self): + nonce = "e" * 48 + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class LiveProcess: + pid = 7300 + + def poll(self): + return None + + def wait(self, timeout=None): + raise subprocess.TimeoutExpired("engine", timeout) + + identity = { + "pid": 7300, + "pgid": 7300, + "sid": 7300, + "uid": host_uid(), + "inert": False, + "starttime": 17300, + "nonce": nonce, + } + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + identity.update( + { + "state_dir": os.path.join( + state, + "colibri", + "ramdisk", + "engines", + plan["model"]["fingerprint"].split(":", 1)[-1], + "interleaved", + ), + "weights_dir": plan["mounts"][0]["path"], + } + ) + manifest = { + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + "ports": [], + } + forget = mock.Mock() + merge = mock.Mock() + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.multiple( + ramdisk, + _filesystem_for_path=mock.Mock(return_value="ext4"), + _load_manifest=mock.Mock(return_value=manifest), + _assert_effective_masks_unchanged=mock.Mock(), + _assert_ready_mounts=mock.Mock(), + _save_manifest=mock.Mock(), + _admit_concurrent_runtimes=mock.Mock(), + _recover_delta=mock.Mock(), + _usage_read=mock.Mock(return_value={}), + _usage_write=mock.Mock(), + _proc_identity=mock.Mock(return_value=identity), + _wait_managed_ready=mock.Mock( + side_effect=ramdisk.RamdiskError("not ready") + ), + _process_matches=mock.Mock( + return_value=(False, "identity-unavailable", None) + ), + _terminate_verified_group=mock.Mock( + return_value="could not revalidate process identity" + ), + _terminate_direct_child=mock.Mock( + return_value="direct child PID 7300 survived SIGKILL" + ), + _track_managed_child=mock.Mock(), + _forget_managed_child=forget, + _merge_usage=merge, + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk.subprocess, "Popen", return_value=LiveProcess() + ), mock.patch.object( + ramdisk.secrets, + "token_hex", + side_effect=lambda size: nonce if size == 24 else "a" * 32, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "not ready.*direct child is still alive", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + merge.assert_not_called() + forget.assert_not_called() + self.assertEqual(manifest["state"], "error") + self.assertNotIn("stopped_at", manifest["processes"][0]) + self.assertNotIn("usage_merged_at", manifest["processes"][0]) + self.assertIn("direct child is still alive", manifest["processes"][0]["stop_error"]) + self.assertFalse( + manifest.get("recovery", {}).get("retained_processes", []) + ) + + # A published survivor stays in the normal process list so a fresh + # invocation of stop can reach its verified group terminator instead + # of deadlocking behind unpublished-process recovery. + reloaded = json.loads(json.dumps(manifest)) + process_matches = mock.Mock( + side_effect=[ + (True, "running", dict(identity)), + (False, "not-running", None), + ] + ) + terminate = mock.Mock(return_value=None) + merge_after_stop = mock.Mock() + with mock.patch.multiple( + ramdisk, + _load_manifest=mock.Mock(return_value=reloaded), + _process_matches=process_matches, + _managed_child_liveness=mock.Mock(return_value=False), + _save_manifest=mock.Mock(), + _terminate_verified_group=terminate, + _merge_usage=merge_after_stop, + _bind_usage_transaction=mock.Mock( + side_effect=lambda record, plan=None, reserved_ids=None: ( + record["usage_merge_id"] + ) + ), + ): + stopped = ramdisk.stop.__wrapped__() + + terminate.assert_called_once_with(reloaded["processes"][0]) + merge_after_stop.assert_called_once() + self.assertEqual(stopped["state"], "stopped") + self.assertNotIn("stop_error", stopped["processes"][0]) + + def test_two_engine_start_retains_survivor_when_sibling_dies_before_readiness( + self, + ): + # Owner interleaving: a multi-engine start publishes engine #1, then + # the real _wait_managed_ready death branch fires for engine #2 (the + # sibling exits before readiness). Rollback then runs over the + # surviving, already-published engine #1 and must retain it durably: + # state=error, no stopped_at, no premature usage merge, and a later + # verified stop recovers it. The fault is injected at _process_matches + # (the real trigger at processes.py:_wait_managed_ready), not by + # mocking readiness itself. + nonce = "f" * 48 + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return b'{"status":"ok"}' + + captures = [] + processes_by_pid = {} + + class FakeProcess: + next_pid = 7400 + + def __init__(self): + type(self).next_pid += 1 + self.pid = type(self).next_pid + self._alive = True + + def poll(self): + return None if self._alive else 0 + + def wait(self, timeout=None): + if self._alive: + raise subprocess.TimeoutExpired("engine", timeout) + + def popen(command, **kwargs): + process = FakeProcess() + processes_by_pid[process.pid] = process + captures.append( + {"pid": process.pid, "environment": dict(kwargs["env"])} + ) + return process + + def identity(pid): + launch = next(item for item in captures if item["pid"] == pid) + environment = launch["environment"] + return { + "pid": pid, + "pgid": pid, + "sid": pid, + "uid": host_uid(), + "inert": False, + "starttime": 1000 + pid, + "nonce": nonce, + "state_dir": environment["COLI_STATE_DIR"], + "weights_dir": environment["COLI_WEIGHTS_DIR"], + } + + survivor_identity = {} + + def process_matches_by_record(record): + # First-published engine stays running and becomes ready. The + # sibling is reported not-running so the un-mocked readiness loop + # raises "exited before readiness"; it has now exited, so mark its + # process dead for the rollback that follows. + pid = record["pid"] + if not captures or pid != captures[0]["pid"]: + sibling = processes_by_pid.get(pid) + if sibling is not None: + sibling._alive = False + return (False, "not-running", None) + survivor_identity.clear() + survivor_identity.update(identity(pid)) + return (True, "running", dict(survivor_identity)) + + usage_ids = iter(("a" * 32, "b" * 32)) + + def deterministic_token(size): + return nonce if size == 24 else next(usage_ids) + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + hardware = hardware_fixture(nodes=2) + set_asymmetric_node_cores(hardware) + plan = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), + hardware=hardware, + ) + manifest = { + "state": "ready", + "base_port": 8100, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(item) for item in plan["mounts"]], + "processes": [], + "best_runtime": { + "per-node": { + "variant": "partial_direct", + "knobs": { + "PIPE": 1, + "OMP_NUM_THREADS": 3, + "OMP_PROC_BIND": "spread", + }, + } + }, + } + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.multiple( + ramdisk, + _filesystem_for_path=mock.Mock(return_value="ext4"), + _load_manifest=mock.Mock(return_value=manifest), + _assert_effective_masks_unchanged=mock.Mock(), + _assert_ready_mounts=mock.Mock(), + _save_manifest=mock.Mock(), + _admit_concurrent_runtimes=mock.Mock(), + _recover_delta=mock.Mock(), + _usage_read=mock.Mock(return_value={}), + _usage_write=mock.Mock(), + _fresh_user_binary=mock.Mock(return_value="/usr/bin/numactl"), + _proc_identity=mock.Mock(side_effect=identity), + _process_matches=mock.Mock( + side_effect=process_matches_by_record + ), + _terminate_verified_group=mock.Mock( + return_value="could not revalidate process identity" + ), + _terminate_direct_child=mock.Mock( + return_value="direct child survived SIGKILL" + ), + _track_managed_child=mock.Mock(), + _forget_managed_child=mock.Mock(), + _merge_usage=mock.Mock(), + ), mock.patch.object( + ramdisk.socket, "socket", side_effect=lambda *a, **k: FakeSocket() + ), mock.patch.object( + ramdisk.subprocess, "Popen", side_effect=popen + ), mock.patch.object( + ramdisk.secrets, "token_hex", side_effect=deterministic_token + ), mock.patch( + "urllib.request.urlopen", return_value=FakeResponse() + ) as urlopen_mock: + with self.assertRaisesRegex( + ramdisk.RamdiskError, "exited before readiness" + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + self.assertEqual(manifest["state"], "error") + survivor_pid = captures[0]["pid"] + survivors = [ + process + for process in manifest["processes"] + if process["pid"] == survivor_pid + ] + self.assertEqual(len(survivors), 1) + survivor = survivors[0] + # Lock the readiness ordering: the survivor must have actually become + # ready (real readiness loop returned) BEFORE the sibling died. A + # reversed-iteration mutant makes the sibling die first with no engine + # ready, which would leave ready_at unset and record zero health + # requests. + self.assertIn("ready_at", survivor) + self.assertEqual(urlopen_mock.call_count, 1) + self.assertEqual( + urlopen_mock.call_args.args[0].full_url, + "http://127.0.0.1:%d/health" % survivor["port"], + ) + self.assertNotIn("stopped_at", survivor) + self.assertNotIn("usage_merged_at", survivor) + self.assertIn("alive", survivor["stop_error"]) + self.assertFalse( + manifest.get("recovery", {}).get("retained_processes", []) + ) + + # A later verified stop recovers the retained survivor exactly once. + reloaded = json.loads(json.dumps(manifest)) + reloaded["processes"] = [ + process + for process in reloaded["processes"] + if process["pid"] == survivor_pid and "stopped_at" not in process + ] + self.assertEqual(len(reloaded["processes"]), 1) + stop_matches = mock.Mock( + side_effect=[ + (True, "running", dict(survivor_identity)), + (False, "not-running", None), + ] + ) + stop_terminate = mock.Mock(return_value=None) + stop_merge = mock.Mock() + with mock.patch.multiple( + ramdisk, + _load_manifest=mock.Mock(return_value=reloaded), + _process_matches=stop_matches, + _managed_child_liveness=mock.Mock(return_value=False), + _save_manifest=mock.Mock(), + _terminate_verified_group=stop_terminate, + _merge_usage=stop_merge, + _bind_usage_transaction=mock.Mock( + side_effect=lambda record, plan=None, reserved_ids=None: ( + record["usage_merge_id"] + ) + ), + ): + stopped = ramdisk.stop.__wrapped__() + + stop_terminate.assert_called_once_with(reloaded["processes"][0]) + stop_merge.assert_called_once() + self.assertEqual(stopped["state"], "stopped") + self.assertNotIn("stop_error", stopped["processes"][0]) + # A verified recovery must not advertise stale launch-time errors. + self.assertNotIn("launch_error", stopped) + self.assertNotIn("cleanup_errors", stopped) + + def test_full_mode_start_refuses_wrong_usage_identity_before_seed_or_spawn(self): + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan( + plan_args(fixture.root, mode="full"), + hardware=hardware_fixture(), + ) + manifest = { + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + "ports": [], + } + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ), mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_assert_effective_masks_unchanged" + ), mock.patch.object( + ramdisk, "_assert_ready_mounts" + ), mock.patch.object( + ramdisk, "_save_manifest" + ), mock.patch.object( + ramdisk, "_admit_concurrent_runtimes" + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk, "_usage_write" + ) as usage_write, mock.patch.object( + ramdisk.subprocess, "Popen" + ) as popen: + for header, message in ( + ("-1 1 2\n-2 1 1\n", "engine identity"), + ( + "-1 9 2\n-2 1 3815245270\n", + "dimensions", + ), + ): + with self.subTest(message=message): + (fixture.root / ".coli_usage").write_text( + header + "0 1 10\n", + encoding="utf-8", + ) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + message, + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + usage_write.assert_not_called() + popen.assert_not_called() + self.assertEqual(manifest["state"], "ready") + + def test_start_stop_preserve_headered_usage_identity(self): + engine_id = 3815245270 + usage_header = "-1 1 2\n-2 1 %d\n" % engine_id + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class FakeProcess: + pid = 4300 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + nonce = "a" * 48 + identity = { + "pid": 4300, + "pgid": 4300, + "sid": 4300, + "uid": host_uid(), + "inert": False, + "starttime": 14300, + "nonce": nonce, + } + process = FakeProcess() + with ModelFixture() as fixture, canonical_temporary_directory() as state: + canonical = fixture.root / ".coli_usage" + canonical.write_text( + usage_header + "0 1 10\n", + encoding="utf-8", + ) + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + identity.update( + { + "state_dir": os.path.join( + state, + "colibri", + "ramdisk", + "engines", + plan["model"]["fingerprint"].split(":", 1)[-1], + "interleaved", + ), + "weights_dir": plan["mounts"][0]["path"], + } + ) + manifest = { + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + "ports": [], + } + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ), mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_assert_effective_masks_unchanged" + ), mock.patch.object( + ramdisk, "_assert_ready_mounts" + ), mock.patch.object( + ramdisk, "_save_manifest" + ), mock.patch.object( + ramdisk, "_admit_concurrent_runtimes" + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk.subprocess, "Popen", return_value=process + ), mock.patch.object( + ramdisk, "_proc_identity", return_value=identity + ), mock.patch.object( + ramdisk, "_wait_managed_ready" + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(False, "not-running", None), + ), mock.patch.object( + ramdisk.secrets, + "token_hex", + side_effect=lambda size: "a" * (size * 2), + ): + launched = ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + record = launched["processes"][0] + state_usage = Path(record["state_dir"]) / ".coli_usage" + self.assertTrue( + state_usage.read_text(encoding="utf-8").startswith( + usage_header + ) + ) + current = ramdisk._usage_read(str(state_usage)) + current["0:1"] = 12 + ramdisk._usage_write(str(state_usage), current) + process.returncode = 0 + stopped = ramdisk.stop.__wrapped__() + + self.addCleanup(ramdisk._forget_managed_child, 4300) + merged = ramdisk._usage_read(str(canonical)) + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(merged["0:1"], 12) + self.assertEqual(merged["-1:1"], 2) + self.assertEqual(merged["-2:1"], engine_id) + + def test_per_node_launch_forces_durable_kv_and_node_local_core_counts(self): + captures = [] + nonce = "a" * 48 + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class FakeProcess: + next_pid = 4100 + + def __init__(self): + type(self).next_pid += 1 + self.pid = type(self).next_pid + + def poll(self): + return None + + def popen(command, **kwargs): + process = FakeProcess() + captures.append( + { + "command": list(command), + "environment": dict(kwargs["env"]), + "pid": process.pid, + } + ) + return process + + def identity(pid): + launch = next( + item for item in captures if item["pid"] == pid + ) + environment = launch["environment"] + return { + "pid": pid, + "pgid": pid, + "sid": pid, + "uid": host_uid(), + "inert": False, + "starttime": 1000 + pid, + "nonce": nonce, + "state_dir": environment["COLI_STATE_DIR"], + "weights_dir": environment["COLI_WEIGHTS_DIR"], + } + + usage_ids = iter(("a" * 32, "b" * 32)) + + def deterministic_token(size): + return nonce if size == 24 else next(usage_ids) + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + hardware = hardware_fixture(nodes=2) + set_asymmetric_node_cores(hardware) + plan = ramdisk.build_plan( + plan_args(fixture.root, topology="per-node"), hardware=hardware + ) + manifest = { + "state": "ready", + "base_port": 8100, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(item) for item in plan["mounts"]], + "processes": [], + "best_runtime": { + "per-node": { + "variant": "partial_direct", + "knobs": { + "PIPE": 1, + "OMP_NUM_THREADS": 3, + "OMP_PROC_BIND": "spread", + }, + } + }, + } + with mock.patch.dict( + os.environ, + { + "XDG_STATE_HOME": state, + "KVSAVE": "0", + "COLI_NO_OMP_TUNE": "1", + "COLI_OMP_TUNED": "1", + "COLI_USAGE_DECAY": "0.5", + }, + ), mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ), mock.patch.object(ramdisk, "_load_manifest", return_value=manifest), mock.patch.object( + ramdisk, "_assert_ready_mounts" + ), mock.patch.object(ramdisk, "_save_manifest"), mock.patch.object( + ramdisk, "_admit_concurrent_runtimes" + ) as admit, mock.patch.object(ramdisk, "_recover_delta"), mock.patch.object( + ramdisk, "_usage_read", return_value={} + ) as usage_read, mock.patch.object( + ramdisk, "_usage_write" + ) as usage_write, mock.patch.object( + ramdisk, "_fresh_user_binary", return_value="/usr/bin/numactl" + ), mock.patch.object( + ramdisk.socket, "socket", side_effect=lambda *args, **kwargs: FakeSocket() + ), mock.patch.object(ramdisk.subprocess, "Popen", side_effect=popen), mock.patch.object( + ramdisk, "_proc_identity", side_effect=identity + ), mock.patch.object(ramdisk, "_wait_managed_ready"), mock.patch.object( + ramdisk.secrets, + "token_hex", + side_effect=deterministic_token, + ): + result = ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), cli_path=sys.executable + ) + for launch in captures: + self.addCleanup(ramdisk._forget_managed_child, launch["pid"]) + + self.assertEqual(result["state"], "running") + admit.assert_called_once_with(plan, manifest["mounts"], benchmark=False) + usage_read.assert_called_once_with( + os.path.join(plan["model"]["path"], ".coli_usage"), + plan=plan, + ) + self.assertEqual(usage_write.call_count, 2) + self.assertTrue( + all( + call.kwargs.get("plan") is plan + for call in usage_write.call_args_list + ) + ) + self.assertEqual(len(captures), 2) + for index, (expected_cores, expected_cpus) in enumerate( + ((3, "0-2"), (5, "3-7")) + ): + launch = captures[index] + environment = launch["environment"] + self.assertEqual(environment["KVSAVE"], "1") + self.assertEqual(environment["PROF"], "1") + self.assertEqual(environment["PIPE"], "1") + self.assertEqual(environment["OMP_NUM_THREADS"], str(expected_cores)) + self.assertEqual(environment["OMP_PROC_BIND"], "spread") + self.assertEqual(environment["COLI_NUMA"], "0") + self.assertEqual(environment["COLI_USAGE_DECAY"], "1.0") + self.assertNotIn("COLI_NO_OMP_TUNE", environment) + self.assertNotIn("COLI_OMP_TUNED", environment) + self.assertEqual(environment["COLI_NUMA_NODES"], str(index)) + self.assertEqual(environment["COLI_CPU_AFFINITY"], expected_cpus) + self.assertEqual( + launch["command"][:3], + [ + "/usr/bin/numactl", + "--physcpubind=%s" % expected_cpus, + "--membind=%d" % index, + ], + ) + self.assertTrue(environment["COLI_STATE_DIR"].endswith("node-%d" % index)) + self.assertEqual([record["port"] for record in result["processes"]], [8100, 8101]) + self.assertEqual(result["base_port"], 8100) + + def test_gpu_plan_launch_applies_reviewed_devices_and_mmap_path(self): + captures = [] + nonce = "d" * 48 + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class FakeProcess: + pid = 4200 + + def poll(self): + return None + + def popen(command, **kwargs): + captures.append(dict(kwargs["env"])) + return FakeProcess() + + identity = { + "pid": 4200, + "pgid": 4200, + "sid": 4200, + "uid": host_uid(), + "inert": False, + "starttime": 14200, + "nonce": nonce, + } + with ModelFixture() as fixture, canonical_temporary_directory() as state: + hardware = hardware_fixture() + hardware["gpus"] = [ + { + "index": 2, + "name": "GPU 2", + "uuid": "GPU-test-2", + "pci_bus_id": "0000:41:00.0", + "numa_node": 0, + "locality": "resolved", + "total_bytes": 32 * ramdisk.GIB, + "free_bytes": 28 * ramdisk.GIB, + } + ] + model = ramdisk.scan_model(str(fixture.root)) + result = ramdisk._resolve_preset( + ramdisk.PRESET_GPU_FASTEST, + plan_args(fixture.root), + hardware=hardware, + model=model, + build_plan=ramdisk.build_plan, + load_profile=ramdisk._load_profile, + cuda_capable=True, + ) + plan = result["plan"] + identity.update( + { + "state_dir": os.path.join( + state, + "colibri", + "ramdisk", + "engines", + plan["model"]["fingerprint"].split(":", 1)[-1], + "interleaved", + ), + "weights_dir": plan["mounts"][0]["path"], + } + ) + manifest = { + "state": "ready", + "base_port": 8000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + } + with mock.patch.dict( + os.environ, + { + "XDG_STATE_HOME": state, + "COLI_GPUS": "7,8", + "CUDA_EXPERT_GB": "4", + "COLI_RAMMAP": "1", + }, + ), mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ), mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_assert_effective_masks_unchanged" + ), mock.patch.object( + ramdisk, "_assert_ready_mounts" + ), mock.patch.object( + ramdisk, "_save_manifest" + ), mock.patch.object( + ramdisk, "_admit_concurrent_runtimes" + ), mock.patch.object( + ramdisk, "_recover_delta" + ), mock.patch.object( + ramdisk, "_usage_read", return_value={} + ), mock.patch.object( + ramdisk, "_usage_write" + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + ramdisk.subprocess, + "Popen", + side_effect=popen, + ), mock.patch.object( + ramdisk, + "_proc_identity", + return_value=identity, + ), mock.patch.object( + ramdisk, "_wait_managed_ready" + ), mock.patch.object( + ramdisk.secrets, "token_hex", return_value=nonce + ): + launched = ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + engine_path=sys.executable, + ) + self.addCleanup(ramdisk._forget_managed_child, 4200) + + self.assertEqual(launched["state"], "running") + self.assertEqual(len(captures), 1) + environment = captures[0] + self.assertEqual(environment["COLI_CUDA"], "1") + self.assertEqual(environment["COLI_GPU"], "0") + self.assertNotIn("COLI_GPUS", environment) + self.assertEqual( + environment["CUDA_VISIBLE_DEVICES"], + "GPU-test-2", + ) + self.assertEqual(environment["CUDA_EXPERT_GB"], "auto") + self.assertEqual(environment["REPIN"], "16") + self.assertEqual(environment["COLI_MMAP"], "1") + self.assertEqual(environment["COLI_RAMMAP"], "0") + self.assertEqual(environment["COLI_RAM_PREFAULT"], "0") + self.assertEqual(environment["PIN"], "auto") + self.assertEqual( + environment["COLI_ENGINE"], + os.path.realpath(sys.executable), + ) + self.assertEqual( + launched["processes"][0]["accelerator_environment"]["COLI_GPU"], + "0", + ) + + def test_clean_start_cancellation_restores_retryable_manifest(self): + cancel = threading.Event() + nonce = "c" * 48 + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class FakeProcess: + pid = 6100 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + self.returncode = 0 + return self.returncode + + def cancel_ready(*args, **kwargs): + cancel.set() + ramdisk._raise_if_cancelled(cancel) + + identity = { + "pid": 6100, + "pgid": 6100, + "sid": 6100, + "uid": host_uid(), + "inert": False, + "starttime": 16100, + "nonce": nonce, + } + process = FakeProcess() + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan( + plan_args(fixture.root), hardware=hardware_fixture() + ) + identity.update( + { + "state_dir": os.path.join( + state, + "colibri", + "ramdisk", + "engines", + plan["model"]["fingerprint"].split(":", 1)[-1], + "interleaved", + ), + "weights_dir": plan["mounts"][0]["path"], + } + ) + manifest = { + "state": "ready", + "base_port": 9000, + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + "ports": [], + } + with mock.patch.dict( + os.environ, {"XDG_STATE_HOME": state} + ), mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ), mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object(ramdisk, "_assert_ready_mounts"), mock.patch.object( + ramdisk, "_save_manifest" + ), mock.patch.object(ramdisk, "_admit_concurrent_runtimes"), mock.patch.object( + ramdisk, "_recover_delta" + ), mock.patch.object(ramdisk, "_usage_read", return_value={}), mock.patch.object( + ramdisk, "_usage_write" + ), mock.patch.object( + ramdisk.socket, "socket", side_effect=lambda *args, **kwargs: FakeSocket() + ), mock.patch.object( + ramdisk.subprocess, "Popen", return_value=process + ) as popen, mock.patch.object( + ramdisk, + "_proc_identity", + side_effect=lambda ignored: ( + identity if process.poll() is None else None + ), + ), mock.patch.object( + ramdisk, "_wait_managed_ready", side_effect=cancel_ready + ), mock.patch.object( + ramdisk, "_terminate_verified_group", return_value=None + ), mock.patch.object( + ramdisk, "_group_alive", return_value=False + ), mock.patch.object(ramdisk, "_merge_usage"), mock.patch.object( + ramdisk.secrets, "token_hex", return_value=nonce + ): + with self.assertRaises(ramdisk._OperationCancelled): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + cancel_event=cancel, + ) + with self.assertRaises(ramdisk._OperationCancelled): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + cancel_event=cancel, + ) + + popen.assert_called_once() + + self.assertEqual(manifest["state"], "ready") + self.assertEqual(manifest["base_port"], 9000) + self.assertEqual(manifest["processes"], []) + self.assertEqual(manifest["ports"], []) + self.assertNotIn("launch_error", manifest) + + def test_launch_rollback_merges_every_context_when_manifest_saves_fail(self): + nonce = "b" * 48 + + class FakeSocket: + def bind(self, address): + pass + + def close(self): + pass + + class FakeProcess: + pid = 5100 + + def __init__(self): + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + self.returncode = 0 + return self.returncode + + with ModelFixture() as fixture, canonical_temporary_directory() as state: + plan = ramdisk.build_plan(plan_args(fixture.root), hardware=hardware_fixture()) + manifest = { + "state": "ready", + "model_fingerprint": plan["model"]["fingerprint"], + "plan": plan, + "mounts": [dict(plan["mounts"][0])], + "processes": [], + } + + def save(current): + if current.get("state") == "error" or any( + record.get("usage_merge_id") for record in current.get("processes", []) + ): + raise OSError("state filesystem full") + + identity = { + "pid": 5100, + "pgid": 5100, + "uid": host_uid(), + "starttime": 15100, + "nonce": nonce, + } + process = FakeProcess() + with mock.patch.dict(os.environ, {"XDG_STATE_HOME": state}), mock.patch.object( + ramdisk, "_filesystem_for_path", return_value="ext4" + ), mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object(ramdisk, "_assert_ready_mounts"), mock.patch.object( + ramdisk, "_save_manifest", side_effect=save + ), mock.patch.object(ramdisk, "_admit_concurrent_runtimes"), mock.patch.object( + ramdisk, "_recover_delta" + ), mock.patch.object(ramdisk, "_usage_read", return_value={}), mock.patch.object( + ramdisk, "_usage_write" + ), mock.patch.object( + ramdisk.socket, "socket", side_effect=lambda *args, **kwargs: FakeSocket() + ), mock.patch.object( + ramdisk.subprocess, "Popen", return_value=process + ), mock.patch.object( + ramdisk, + "_proc_identity", + side_effect=lambda ignored: ( + identity if process.poll() is None else None + ), + ), mock.patch.object( + ramdisk, "_wait_managed_ready", side_effect=ramdisk.RamdiskError("not ready") + ), mock.patch.object( + ramdisk, "_terminate_verified_group", return_value=None + ), mock.patch.object(ramdisk, "_group_alive", return_value=False), mock.patch.object( + ramdisk, "_merge_usage" + ) as merge, mock.patch.object(ramdisk.secrets, "token_hex", return_value=nonce): + with self.assertRaisesRegex(ramdisk.RamdiskError, "rollback/reporting errors"): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=8200), cli_path=sys.executable + ) + + merge.assert_called_once() + self.assertTrue(merge.call_args.kwargs["keep_journal"]) diff --git a/c/tests/test_ramdisk_state_lifecycle.py b/c/tests/test_ramdisk_state_lifecycle.py new file mode 100644 index 000000000..041b59331 --- /dev/null +++ b/c/tests/test_ramdisk_state_lifecycle.py @@ -0,0 +1,4655 @@ +"""RAM-disk durable state, safety, and lifecycle tests.""" + +import copy + +if __package__: + from .ramdisk_test_support import * # noqa: F401,F403 +else: + from ramdisk_test_support import * # noqa: F401,F403 + +from ramdisk_support import lifecycle as lifecycle_support +from ramdisk_support import linux_ops +from ramdisk_support import processes as process_support +from ramdisk_support import state as state_support + + +_REAL_BOUND_PARENT_DESCRIPTOR = state_support._bound_parent_descriptor +_REAL_FSYNC_DIRECTORY = state_support._fsync_directory + + +@contextlib.contextmanager +def _portable_descriptor_seam(): + """Exercise descriptor-gated state logic without weakening production.""" + + def allow_portable_binding(*args, **kwargs): + kwargs["require_native"] = False + return _REAL_BOUND_PARENT_DESCRIPTOR(*args, **kwargs) + + with mock.patch.object( + state_support, + "_bound_parent_descriptor", + new=allow_portable_binding, + ), mock.patch.object( + state_support, + "_fsync_bound_directory", + new=lambda descriptor: None, + ), mock.patch.object( + state_support, + "_fsync_directory", + new=lambda path: None, + ): + yield + + +class StateAndSafetyTest(unittest.TestCase): + FINGERPRINT = "sha256:" + ("a" * 64) + GLM_ENGINE_ID = 3815245270 + USAGE_HEADER = "-1 1 2\n-2 1 %d\n" % GLM_ENGINE_ID + + def setUp(self): + self.descriptor_seam = contextlib.ExitStack() + self.addCleanup(self.descriptor_seam.close) + if not state_support._supports_native_dirfd(): + self.descriptor_seam.enter_context(_portable_descriptor_seam()) + self.temp = tempfile.TemporaryDirectory() + self.root = str(Path(self.temp.name).resolve()) + self.env = mock.patch.dict( + os.environ, + { + "XDG_STATE_HOME": os.path.join(self.root, "state"), + "COLI_RAMDISK_MANIFEST": os.path.join(self.root, "manifest.json"), + }, + ) + self.env.start() + self.filesystem = mock.patch.object(ramdisk, "_filesystem_for_path", return_value="ext4") + self.filesystem.start() + + def tearDown(self): + self.descriptor_seam.close() + self.filesystem.stop() + self.env.stop() + self.temp.cleanup() + + def manifest(self, state="ready", mount_paths=None, processes=None): + """Return a schema-valid lifecycle record for focused safety tests.""" + mount_paths = mount_paths or ["/mnt/colibri-test"] + processes = processes or [] + topology = "per-node" if len(mount_paths) > 1 else "interleaved" + mount_root = mount_paths[0] if topology == "interleaved" else os.path.dirname(mount_paths[0]) + nodes = list(range(len(mount_paths))) if topology == "per-node" else [0] + planned = [ + {"path": path, "node": nodes[index] if topology == "per-node" else None} + for index, path in enumerate(mount_paths) + ] + mounted = [ + { + "path": path, + "node": planned[index]["node"], + "identity": { + "mount_id": index + 4, + "device": "0:%d" % (index + 9), + }, + } + for index, path in enumerate(mount_paths) + ] + fingerprint_dir = self.FINGERPRINT.split(":", 1)[1] + complete_processes = [] + for index, partial in enumerate(processes): + record = dict(partial) + node = planned[index]["node"] + port = 8000 + index + label = "interleaved" if node is None else "node-%d" % node + record.update( + { + "pgid": record["pid"], + "uid": host_uid(), + "starttime": 100 + record["pid"], + "nonce": "%048x" % (index + 1), + "port": port, + "node": node, + "weights_dir": planned[index]["path"], + "state_dir": os.path.join( + ramdisk._state_root(), "engines", fingerprint_dir, label + ), + "usage_baseline": {}, + "command": [ + str(C_DIR / "coli"), + "serve", + "--model", + os.path.join(self.root, "model"), + "--port", + str(port), + ], + } + ) + complete_processes.append(record) + return { + "version": ramdisk.MANIFEST_VERSION, + "state": state, + "model_fingerprint": self.FINGERPRINT, + "plan": { + "topology": topology, + "mount_root": mount_root, + "mounts": planned, + "hardware": hardware_fixture(nodes=len(mount_paths) if topology == "per-node" else 1), + "model": { + "path": os.path.join(self.root, "model"), + "fingerprint": self.FINGERPRINT, + }, + "durable_state": { + "root": ramdisk._state_root(), + "manifest": ramdisk._manifest_path(), + "benchmarks": ramdisk._benchmarks_path(), + }, + "source_shards": [{"name": "model.safetensors"}], + }, + "mounts": mounted, + "processes": complete_processes, + } + + def recovery_state_dir(self, node=None): + label = "interleaved" if node is None else "node-%d" % node + return os.path.join( + ramdisk._state_root(), + "engines", + self.FINGERPRINT.split(":", 1)[1], + label, + ) + + def test_usage_delta_merge_and_crash_recovery_are_idempotent(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir, mode=0o750) + os.makedirs(state_dir, mode=0o700) + ramdisk._usage_write(canonical, {"0:1": 10, "0:2": 3}) + ramdisk._usage_write(os.path.join(state_dir, ".coli_usage"), {"0:1": 14, "0:2": 3, "0:3": 2}) + record = {"state_dir": state_dir, "usage_baseline": {"0:1": 10, "0:2": 3}} + ramdisk._merge_usage(record, canonical) + self.assertEqual(ramdisk._usage_read(canonical), {"0:1": 14, "0:2": 3, "0:3": 2}) + ramdisk._merge_usage(record, canonical) + self.assertEqual(ramdisk._usage_read(canonical), {"0:1": 14, "0:2": 3, "0:3": 2}) + + merge_id = "a" * 32 + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + ramdisk._usage_write(canonical, {"0:1": 15}, merge_id=merge_id) + ramdisk._atomic_json(delta_path, {"version": 1, "id": merge_id, "delta": {"0:1": 1}}) + ramdisk._recover_delta(state_dir, canonical) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 15) + self.assertFalse(os.path.exists(delta_path)) + self.assertEqual(os.stat(model_dir).st_mode & 0o777, 0o750) + + def test_recovery_uses_one_canonical_counts_and_markers_snapshot(self): + model_dir = os.path.join(self.root, "model") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + merge_id = "4" * 32 + ramdisk._usage_write(canonical, {"0:1": 10}) + ramdisk._atomic_json( + os.path.join(state_dir, ".coli_usage.delta.json"), + {"version": 1, "id": merge_id, "delta": {"0:1": 2}}, + ) + real_snapshot = state_support._usage_snapshot_from_bound + canonical_snapshots = [] + + def observe_snapshot(bound, name, path, **kwargs): + snapshot = real_snapshot(bound, name, path, **kwargs) + if kwargs.get("source") == "canonical usage target": + canonical_snapshots.append(snapshot["text"]) + return snapshot + + with mock.patch.object( + state_support, + "_usage_snapshot_from_bound", + side_effect=observe_snapshot, + ): + ramdisk._recover_delta(state_dir, canonical) + + self.assertEqual(len(canonical_snapshots), 1) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 12) + self.assertEqual(ramdisk._usage_merge_ids(canonical), {merge_id}) + + def test_usage_delta_recovery_rejects_nonpositive_or_coerced_counts(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + + invalid_deltas = ( + {"0:1": 0}, + {"0:1": -3}, + {"0:1": True}, + {"0:1": 2.5}, + {"0:1": "3"}, + {"invalid": 3}, + ) + for index, delta in enumerate(invalid_deltas, 1): + with self.subTest(delta=delta): + ramdisk._atomic_json( + delta_path, + { + "version": 1, + "id": ("%x" % index) * 32, + "delta": delta, + }, + ) + journal_before = Path(delta_path).read_bytes() + canonical_before = Path(canonical).read_bytes() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "usage delta journal has invalid counts", + ): + ramdisk._recover_delta(state_dir, canonical) + + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + self.assertEqual(Path(delta_path).read_bytes(), journal_before) + + def test_present_null_usage_journal_is_malformed_and_retained(self): + model_dir = os.path.join(self.root, "model") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + ramdisk._usage_write(canonical, {"0:1": 10}) + Path(delta_path).write_text("null\n", encoding="utf-8") + canonical_before = Path(canonical).read_bytes() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "must contain a JSON object", + ): + ramdisk._recover_delta(state_dir, canonical) + + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + self.assertEqual(Path(delta_path).read_text(encoding="utf-8"), "null\n") + + def test_absent_usage_journal_retry_reproves_parent_directory(self): + model_dir = os.path.join(self.root, "model") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + ramdisk._usage_write(canonical, {"0:1": 10}) + + with mock.patch.object( + state_support, + "_fsync_bound_directory", + ) as sync_directory, mock.patch.object( + state_support, + "_fsync_directory", + new=sync_directory, + ): + ramdisk._recover_delta(state_dir, canonical) + + sync_directory.assert_called_once() + + def test_record_merge_refuses_mismatched_journal_transaction(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + ramdisk._atomic_json( + delta_path, + { + "version": 1, + "id": "b" * 32, + "delta": {"0:1": 7}, + }, + ) + canonical_before = Path(canonical).read_bytes() + journal_before = Path(delta_path).read_bytes() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "journal transaction.*managed record", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": "a" * 32, + }, + canonical, + ) + + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + self.assertEqual(Path(delta_path).read_bytes(), journal_before) + + def test_recover_delta_facade_forwards_expected_transaction(self): + plan = {"identity": "test-plan"} + merge_id = "a" * 32 + with mock.patch.object( + ramdisk, + "_state_recover_delta", + return_value=merge_id, + ) as recover: + result = ramdisk._recover_delta( + "/durable/state", + "/model/.coli_usage", + plan=plan, + expected_merge_id=merge_id, + ) + + self.assertEqual(result, merge_id) + recover.assert_called_once_with( + "/durable/state", + "/model/.coli_usage", + plan=plan, + expected_merge_id=merge_id, + filesystem_for_path=ramdisk._filesystem_for_path, + source_still_matches=ramdisk._source_still_matches, + ) + + def test_matching_record_journal_and_standalone_legacy_journal_recover(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + standalone_dir = os.path.join(self.root, "standalone-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + os.makedirs(standalone_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + merge_id = "c" * 32 + ramdisk._atomic_json( + os.path.join(state_dir, ".coli_usage.delta.json"), + {"version": 1, "id": merge_id, "delta": {"0:1": 2}}, + ) + + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": merge_id, + }, + canonical, + ) + + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 12) + standalone_id = "d" * 32 + ramdisk._atomic_json( + os.path.join(standalone_dir, ".coli_usage.delta.json"), + { + "version": 1, + "id": standalone_id, + "delta": {"0:1": 1}, + }, + ) + ramdisk._recover_delta(standalone_dir, canonical) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 13) + self.assertEqual( + ramdisk._usage_merge_ids(canonical), + {merge_id, standalone_id}, + ) + + def test_legacy_record_adopts_existing_journal_transaction_idempotently(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + merge_id = "9" * 32 + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + journal = { + "version": 1, + "id": merge_id, + "delta": {"0:1": 2}, + } + ramdisk._atomic_json(delta_path, journal) + record = { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + } + + ramdisk._merge_usage(record, canonical) + + self.assertEqual(record["usage_merge_id"], merge_id) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 12) + self.assertFalse(os.path.exists(delta_path)) + + # A stale replay of the adopted legacy journal must remain idempotent. + ramdisk._atomic_json(delta_path, journal) + ramdisk._merge_usage(record, canonical) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 12) + self.assertFalse(os.path.exists(delta_path)) + + def test_usage_history_read_is_optional_only_when_absent(self): + missing = os.path.join(self.root, "missing-usage") + self.assertEqual(state_support._usage_read(missing), {}) + + denied = os.path.join(self.root, "denied-usage") + Path(denied).write_text("0 1 7\n", encoding="utf-8") + real_open = state_support.os.open + + def deny_usage_open(path, flags, *args, **kwargs): + if path in (denied, os.path.basename(denied)): + raise PermissionError("permission denied") + return real_open(path, flags, *args, **kwargs) + + with mock.patch.object( + state_support.os, + "open", + side_effect=deny_usage_open, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot read usage state.*permission denied", + ): + state_support._usage_read(denied) + + invalid = os.path.join(self.root, "invalid-utf8-usage") + with open(invalid, "wb") as stream: + stream.write(b"0 1 7\n\xff") + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "cannot read usage state", + ): + state_support._usage_read(invalid) + + def test_managed_usage_merge_requires_regular_nonsymlink_state_file(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + os.makedirs(model_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + + for kind in ("missing", "directory", "symlink"): + with self.subTest(kind=kind): + state_dir = os.path.join(self.root, "state-" + kind) + os.makedirs(state_dir) + state_usage = os.path.join(state_dir, ".coli_usage") + if kind == "directory": + os.mkdir(state_usage) + elif kind == "symlink": + target = os.path.join(self.root, "usage-target") + Path(target).write_text("0 1 12\n", encoding="utf-8") + os.symlink(target, state_usage) + canonical_before = Path(canonical).read_bytes() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed usage history.*regular non-symlink file", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": "e" * 32, + }, + canonical, + ) + + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + + @requires_native_dirfd + def test_managed_usage_merge_rejects_symlink_swap_during_verified_open(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "state-native") + state_usage = os.path.join(state_dir, ".coli_usage") + attacker = os.path.join(self.root, "attacker-native") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + Path(state_usage).write_text("0 1 12\n", encoding="utf-8") + Path(attacker).write_text("0 1 99\n", encoding="utf-8") + real_open = state_support.os.open + swapped = False + + def swap_before_open(path, flags, *args, **kwargs): + nonlocal swapped + if path in (state_usage, ".coli_usage") and not swapped: + swapped = True + os.unlink(state_usage) + os.symlink(attacker, state_usage) + return real_open(path, flags, *args, **kwargs) + + with mock.patch.object( + state_support.os, + "open", + side_effect=swap_before_open, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed usage history.*regular non-symlink|" + "managed usage history changed|" + "cannot read managed usage history", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": "8" * 32, + }, + canonical, + ) + + self.assertTrue(swapped) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 10) + + def test_managed_usage_merge_fails_closed_without_native_dirfd(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "state-portable") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + canonical_before = Path(canonical).read_bytes() + + with mock.patch.object( + state_support, + "_bound_parent_descriptor", + new=_REAL_BOUND_PARENT_DESCRIPTOR, + ), mock.patch.object( + state_support, + "_supports_native_dirfd", + return_value=False, + ), mock.patch.object( + state_support.os, + "open", + wraps=state_support.os.open, + ) as open_file: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed state requires descriptor-relative filesystem operations", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": "9" * 32, + }, + canonical, + ) + + open_file.assert_not_called() + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + + def test_managed_usage_read_binds_parent_identity(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + original_dir = os.path.join(self.root, "original-state") + replacement_dir = os.path.join(self.root, "replacement-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + os.makedirs(replacement_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + ramdisk._usage_write( + os.path.join(replacement_dir, ".coli_usage"), + {"0:1": 99}, + ) + state_usage = os.path.join(state_dir, ".coli_usage") + real_open = state_support.os.open + swapped = False + + def swap_parent_before_open(path, flags, *args, **kwargs): + nonlocal swapped + if path in (state_dir, state_usage) and not swapped: + swapped = True + os.rename(state_dir, original_dir) + os.rename(replacement_dir, state_dir) + return real_open(path, flags, *args, **kwargs) + + with mock.patch.object( + state_support.os, + "open", + side_effect=swap_parent_before_open, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed state|parent.*changed|managed usage history", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": "7" * 32, + }, + canonical, + ) + + self.assertTrue(swapped) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 10) + + def test_managed_usage_seed_write_rejects_symlink_target(self): + state_dir = os.path.join(self.root, "node-state") + os.makedirs(state_dir) + state_usage = os.path.join(state_dir, ".coli_usage") + attacker = os.path.join(self.root, "attacker-usage") + Path(attacker).write_text("0 1 99\n", encoding="utf-8") + os.symlink(attacker, state_usage) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed usage history.*regular non-symlink", + ): + state_support._managed_usage_write( + state_usage, + {"0:1": 10}, + filesystem_for_path=lambda path: "ext4", + ) + + self.assertTrue(os.path.islink(state_usage)) + self.assertIn("0 1 99", Path(attacker).read_text(encoding="utf-8")) + + @requires_native_dirfd + def test_managed_usage_seed_write_binds_parent_identity(self): + state_dir = os.path.join(self.root, "node-state") + original_dir = os.path.join(self.root, "original-state") + replacement_dir = os.path.join(self.root, "replacement-state") + os.makedirs(state_dir) + os.makedirs(replacement_dir) + state_usage = os.path.join(state_dir, ".coli_usage") + real_open = state_support.os.open + swapped = False + + def swap_parent_before_open(path, flags, *args, **kwargs): + nonlocal swapped + if path == state_dir and not swapped: + swapped = True + os.rename(state_dir, original_dir) + os.rename(replacement_dir, state_dir) + return real_open(path, flags, *args, **kwargs) + + with mock.patch.object( + state_support.os, + "open", + side_effect=swap_parent_before_open, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "managed state.*parent identity changed|" + "managed state parent identity changed", + ): + state_support._managed_usage_write( + state_usage, + {"0:1": 10}, + filesystem_for_path=lambda path: "ext4", + ) + + self.assertTrue(swapped) + self.assertFalse(os.path.exists(os.path.join(state_dir, ".coli_usage"))) + self.assertFalse(os.path.exists(os.path.join(original_dir, ".coli_usage"))) + + @requires_posix_fifo + def test_managed_usage_swap_to_fifo_uses_nonblocking_open(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + state_usage = os.path.join(state_dir, ".coli_usage") + ramdisk._usage_write(state_usage, {"0:1": 12}) + real_open = state_support.os.open + swapped = False + opened_flags = [] + + def swap_to_fifo_before_open(path, flags, *args, **kwargs): + nonlocal swapped + if path in (state_usage, ".coli_usage") and not swapped: + swapped = True + os.unlink(state_usage) + os.mkfifo(state_usage) + opened_flags.append(flags) + if not flags & getattr(os, "O_NONBLOCK", 0): + raise AssertionError("FIFO open would block") + return real_open(path, flags, *args, **kwargs) + + with mock.patch.object( + state_support.os, + "open", + side_effect=swap_to_fifo_before_open, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "regular non-symlink", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": "6" * 32, + }, + canonical, + ) + + self.assertTrue(swapped) + self.assertTrue(opened_flags[0] & os.O_NONBLOCK) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 10) + + def test_canonical_usage_symlink_cannot_authorize_marker_shortcut(self): + model_dir = os.path.join(self.root, "model") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + attacker = os.path.join(self.root, "attacker-usage") + merge_id = "5" * 32 + Path(attacker).write_text( + "0 1 99\n# coli-ramdisk-merge %s\n" % merge_id, + encoding="utf-8", + ) + os.symlink(attacker, canonical) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "canonical usage.*regular non-symlink|canonical usage target", + ): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": merge_id, + }, + canonical, + ) + + self.assertTrue(os.path.islink(canonical)) + self.assertIn("0 1 99", Path(attacker).read_text(encoding="utf-8")) + + def test_managed_usage_merge_rejects_missing_or_regressed_positive_baseline(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10, "0:2": 4}) + state_usage = os.path.join(state_dir, ".coli_usage") + + for label, current, message in ( + ("missing", {"0:2": 4}, "missing positive baseline counter"), + ("regressed", {"0:1": 9, "0:2": 4}, "regressed below baseline"), + ): + with self.subTest(label=label): + ramdisk._usage_write(state_usage, current) + canonical_before = Path(canonical).read_bytes() + with self.assertRaisesRegex(ramdisk.RamdiskError, message): + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10, "0:2": 4}, + "usage_merge_id": ("1" if label == "missing" else "2") * 32, + }, + canonical, + ) + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + + def test_zero_baseline_omission_and_zero_byte_history_remain_valid(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + Path(canonical).touch() + state_usage = os.path.join(state_dir, ".coli_usage") + Path(state_usage).touch() + + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": {"0:1": 0}, + "usage_merge_id": "3" * 32, + }, + canonical, + ) + + self.assertEqual(Path(canonical).read_bytes(), b"") + self.assertEqual(Path(state_usage).read_bytes(), b"") + + def test_headered_usage_copy_merge_and_recovery_preserve_metadata(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + Path(canonical).write_text( + self.USAGE_HEADER + "0 1 10\n", + encoding="utf-8", + ) + + baseline = ramdisk._usage_read(canonical) + self.assertEqual(baseline["-1:1"], 2) + self.assertEqual(baseline["-2:1"], self.GLM_ENGINE_ID) + state_usage = os.path.join(state_dir, ".coli_usage") + ramdisk._usage_write(state_usage, baseline) + self.assertTrue( + Path(state_usage).read_text(encoding="utf-8").startswith( + self.USAGE_HEADER + ) + ) + + current = dict(baseline) + current["0:1"] = 13 + ramdisk._usage_write(state_usage, current) + record = { + "state_dir": state_dir, + "usage_baseline": baseline, + "usage_merge_id": "b" * 32, + } + ramdisk._merge_usage(record, canonical) + merged = ramdisk._usage_read(canonical) + self.assertEqual(merged["0:1"], 13) + self.assertEqual(merged["-1:1"], 2) + self.assertEqual(merged["-2:1"], self.GLM_ENGINE_ID) + + recovery_dir = os.path.join(self.root, "recovery-state") + os.makedirs(recovery_dir) + recovery_id = "c" * 32 + ramdisk._atomic_json( + os.path.join(recovery_dir, ".coli_usage.delta.json"), + { + "version": 1, + "id": recovery_id, + "delta": {"0:1": 2}, + "headers": { + "-1:1": 2, + "-2:1": self.GLM_ENGINE_ID, + }, + }, + ) + ramdisk._recover_delta(recovery_dir, canonical) + recovered = ramdisk._usage_read(canonical) + self.assertEqual(recovered["0:1"], 15) + self.assertEqual(recovered["-1:1"], 2) + self.assertEqual(recovered["-2:1"], self.GLM_ENGINE_ID) + + def test_multilayer_usage_header_survives_a_second_managed_merge(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + header = state_support._usage_header_counts( + { + "n_layers": 40, + "n_experts": 8, + "format_version": 1, + "engine_id": self.GLM_ENGINE_ID, + } + ) + first_baseline = dict(header) + first_baseline["0:1"] = 10 + manifest = self.manifest( + state="running", + processes=[{"pid": 731}], + ) + manifest["processes"][0]["usage_baseline"] = dict(first_baseline) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + loaded = ramdisk._load_manifest(required=True) + self.assertEqual( + loaded["processes"][0]["usage_baseline"]["-1:40"], + 8, + ) + + ramdisk._usage_write(canonical, first_baseline) + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + dict(first_baseline, **{"0:1": 12}), + ) + + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": first_baseline, + "usage_merge_id": "1" * 32, + }, + canonical, + ) + second_baseline = ramdisk._usage_read(canonical) + self.assertEqual(second_baseline["-1:40"], 8) + self.assertEqual(second_baseline["0:1"], 12) + + second_current = dict(second_baseline) + second_current["0:1"] = 15 + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + second_current, + ) + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": second_baseline, + "usage_merge_id": "2" * 32, + }, + canonical, + ) + + final = ramdisk._usage_read(canonical) + self.assertEqual(final["-1:40"], 8) + self.assertEqual(final["-2:1"], self.GLM_ENGINE_ID) + self.assertEqual(final["0:1"], 15) + + def test_manifest_rejects_nonpositive_or_malformed_usage_headers(self): + invalid_baselines = ( + {"-1:0": 8, "-2:1": self.GLM_ENGINE_ID, "0:1": 10}, + {"-1:40": 8, "-2:0": self.GLM_ENGINE_ID, "0:1": 10}, + {"-1:40": 0, "-2:1": self.GLM_ENGINE_ID, "0:1": 10}, + {"-1:40": 8, "-2:1": 0, "0:1": 10}, + {"-1:forty": 8, "-2:1": self.GLM_ENGINE_ID, "0:1": 10}, + ) + for baseline in invalid_baselines: + with self.subTest(baseline=baseline): + manifest = self.manifest( + state="running", + processes=[{"pid": 731}], + ) + manifest["processes"][0]["usage_baseline"] = baseline + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unsafe managed process record", + ): + ramdisk._load_manifest(required=True) + + def test_usage_reader_rejects_zero_header_metadata(self): + invalid_headers = ( + ("-1 0 8\n-2 1 %d\n" % self.GLM_ENGINE_ID, "dimensions"), + ("-1 40 8\n-2 0 %d\n" % self.GLM_ENGINE_ID, "version"), + ("-1 40 8\n-2 1 0\n", "engine identity"), + ) + for index, (contents, message) in enumerate(invalid_headers): + with self.subTest(contents=contents): + path = os.path.join(self.root, "invalid-header-%d" % index) + Path(path).write_text(contents, encoding="utf-8") + with self.assertRaisesRegex(ramdisk.RamdiskError, message): + ramdisk._usage_read(path) + + def test_usage_metadata_must_be_complete_and_compatible(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + Path(canonical).write_text( + self.USAGE_HEADER + "0 1 10\n", + encoding="utf-8", + ) + baseline = ramdisk._usage_read(canonical) + state_usage = os.path.join(state_dir, ".coli_usage") + Path(state_usage).write_text( + "-1 1 2\n-2 1 1\n0 1 12\n", + encoding="utf-8", + ) + record = {"state_dir": state_dir, "usage_baseline": baseline} + with self.assertRaisesRegex(ramdisk.RamdiskError, "engine"): + ramdisk._merge_usage(record, canonical) + self.assertEqual(ramdisk._usage_read(canonical), baseline) + + incomplete = os.path.join(self.root, "incomplete.coli_usage") + Path(incomplete).write_text("-1 1 2\n0 1 3\n", encoding="utf-8") + with self.assertRaisesRegex(ramdisk.RamdiskError, "both"): + ramdisk._usage_read(incomplete) + + def test_legacy_seed_upgrades_to_engine_header_and_old_journal_recovers(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + Path(canonical).write_text("0 1 10\n", encoding="utf-8") + baseline = ramdisk._usage_read(canonical) + Path(state_dir, ".coli_usage").write_text( + self.USAGE_HEADER + "0 1 10\n", + encoding="utf-8", + ) + ramdisk._merge_usage( + {"state_dir": state_dir, "usage_baseline": baseline}, + canonical, + ) + upgraded = ramdisk._usage_read(canonical) + self.assertEqual(upgraded["0:1"], 10) + self.assertEqual(upgraded["-1:1"], 2) + self.assertEqual(upgraded["-2:1"], self.GLM_ENGINE_ID) + + current = dict(upgraded) + current["0:1"] = 12 + ramdisk._usage_write( + os.path.join(state_dir, ".coli_usage"), + current, + ) + ramdisk._merge_usage( + { + "state_dir": state_dir, + "usage_baseline": baseline, + "usage_merge_id": "e" * 32, + }, + canonical, + ) + + # Journals created by the pre-header RAM-disk manager have no metadata. + # They remain valid legacy deltas, while the canonical identity wins. + recovery_dir = os.path.join(self.root, "old-journal") + os.makedirs(recovery_dir) + ramdisk._atomic_json( + os.path.join(recovery_dir, ".coli_usage.delta.json"), + { + "version": 1, + "id": "d" * 32, + "delta": {"0:1": 1}, + }, + ) + ramdisk._recover_delta(recovery_dir, canonical) + recovered = ramdisk._usage_read(canonical) + self.assertEqual(recovered["0:1"], 13) + self.assertEqual(recovered["-1:1"], 2) + self.assertEqual(recovered["-2:1"], self.GLM_ENGINE_ID) + + def test_atomic_json_never_chmods_an_existing_override_parent(self): + parent = os.path.join(self.root, "shared-parent") + os.mkdir(parent, 0o755) + before = os.stat(parent).st_mode & 0o777 + ramdisk._atomic_json(os.path.join(parent, "manifest.json"), {"ok": True}) + self.assertEqual(os.stat(parent).st_mode & 0o777, before) + + def test_atomic_json_stream_close_cannot_mask_primary_error(self): + stream = mock.MagicMock() + stream.__enter__.return_value = stream + stream.__exit__.side_effect = OSError("secondary stream close") + stream.close.side_effect = OSError("secondary stream close") + with mock.patch.object( + state_support.os, + "fdopen", + return_value=stream, + ), mock.patch.object( + state_support.json, + "dump", + side_effect=ValueError("primary JSON serialization"), + ): + with self.assertRaisesRegex( + ValueError, + "primary JSON serialization", + ): + state_support._atomic_json( + os.path.join(self.root, "atomic.json"), + {"unsafe": object()}, + ) + + @requires_native_dirfd + def test_atomic_temp_creation_stays_inside_bound_parent(self): + parent = os.path.join(self.root, "usage-parent") + original = os.path.join(self.root, "usage-parent-original") + replacement = os.path.join(self.root, "usage-parent-replacement") + os.makedirs(parent) + os.makedirs(replacement) + target = os.path.join(parent, ".coli_usage") + real_open = state_support.os.open + swapped = False + + def swap_parent_at_temp_open(path, flags, *args, **kwargs): + nonlocal swapped + if ( + isinstance(path, str) + and path.startswith(".usage-") + and kwargs.get("dir_fd") is not None + and not swapped + ): + swapped = True + os.rename(parent, original) + os.rename(replacement, parent) + return real_open(path, flags, *args, **kwargs) + + with mock.patch.object( + state_support.os, + "open", + side_effect=swap_parent_at_temp_open, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "parent identity changed", + ): + state_support._usage_write( + target, + {"0:1": 1}, + require_native=True, + ) + + self.assertTrue(swapped) + self.assertEqual(list(Path(original).glob(".usage-*")), []) + self.assertEqual(list(Path(parent).glob(".usage-*")), []) + + def test_usage_write_stream_close_cannot_mask_primary_error(self): + stream = mock.MagicMock() + stream.__enter__.return_value = stream + stream.__exit__.side_effect = OSError("secondary stream close") + stream.close.side_effect = OSError("secondary stream close") + path = os.path.join(self.root, ".coli_usage") + with mock.patch.object( + state_support.os, + "fdopen", + return_value=stream, + ): + stream.write.side_effect = ValueError("primary usage write") + with self.assertRaisesRegex(ValueError, "primary usage write"): + state_support._usage_write(path, {"0:1": 1}) + + def test_managed_usage_close_cannot_mask_parse_error(self): + state_usage = os.path.join(self.root, ".coli_usage") + Path(state_usage).write_text("-1 malformed\n", encoding="utf-8") + real_close = state_support.os.close + + def fail_after_close(descriptor): + real_close(descriptor) + raise OSError("secondary descriptor close") + + with mock.patch.object( + state_support.os, + "close", + side_effect=fail_after_close, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "malformed usage header", + ): + state_support._managed_usage_read(state_usage) + + def test_directory_fsync_error_propagates_and_closes_descriptor(self): + with mock.patch.object( + state_support.os, + "open", + return_value=731, + ), mock.patch.object( + state_support.os, + "fsync", + side_effect=OSError(5, "synthetic directory EIO"), + ), mock.patch.object(state_support.os, "close") as close: + with self.assertRaisesRegex(OSError, "synthetic directory EIO"): + _REAL_FSYNC_DIRECTORY(self.root) + + close.assert_called_once_with(731) + + def test_directory_fsync_preserves_primary_error_over_close_error(self): + with mock.patch.object( + state_support.os, + "open", + return_value=732, + ), mock.patch.object( + state_support.os, + "fsync", + side_effect=OSError(5, "primary directory EIO"), + ), mock.patch.object( + state_support.os, + "close", + side_effect=OSError(9, "secondary close failure"), + ) as close: + with self.assertRaisesRegex(OSError, "primary directory EIO"): + _REAL_FSYNC_DIRECTORY(self.root) + + close.assert_called_once_with(732) + + def test_directory_fsync_reports_close_error_after_successful_sync(self): + with mock.patch.object( + state_support.os, + "open", + return_value=733, + ), mock.patch.object( + state_support.os, + "fsync", + ), mock.patch.object( + state_support.os, + "close", + side_effect=OSError(9, "directory close failure"), + ) as close: + with self.assertRaisesRegex(OSError, "directory close failure"): + _REAL_FSYNC_DIRECTORY(self.root) + + close.assert_called_once_with(733) + + def test_pending_manifest_save_reports_unproven_directory_commit(self): + path = os.path.join(self.root, "pending-manifest.json") + pending = { + "state": "starting", + "pending_launches": [{"operation_id": "start:" + ("a" * 32)}], + } + durability_error = OSError(5, "synthetic directory EIO") + with mock.patch.object( + state_support, + "_fsync_bound_directory", + side_effect=durability_error, + ), mock.patch.object( + state_support, + "_fsync_directory", + side_effect=durability_error, + ): + with self.assertRaisesRegex(OSError, "synthetic directory EIO"): + state_support._save_manifest( + pending, + manifest_path=lambda: path, + ) + + persisted = json.loads(Path(path).read_text(encoding="utf-8")) + self.assertEqual( + persisted["pending_launches"][0]["operation_id"], + "start:" + ("a" * 32), + ) + + def test_uncertain_canonical_marker_keeps_journal_for_retry(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + merge_id = "f" * 32 + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + ramdisk._atomic_json( + delta_path, + {"version": 1, "id": merge_id, "delta": {"0:1": 2}}, + ) + + durability_error = OSError(5, "synthetic directory EIO") + with mock.patch.object( + state_support, + "_fsync_bound_directory", + side_effect=durability_error, + ), mock.patch.object( + state_support, + "_fsync_directory", + side_effect=durability_error, + ): + with self.assertRaisesRegex(OSError, "synthetic directory EIO"): + ramdisk._recover_delta(state_dir, canonical) + + self.assertTrue(os.path.exists(delta_path)) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 12) + self.assertIn(merge_id, ramdisk._usage_merge_ids(canonical)) + + ramdisk._recover_delta(state_dir, canonical) + self.assertFalse(os.path.exists(delta_path)) + self.assertEqual(ramdisk._usage_read(canonical)["0:1"], 12) + + @requires_native_dirfd + def test_existing_marker_reproves_canonical_parent_before_journal_unlink(self): + model_dir = os.path.join(self.root, "model") + state_dir = os.path.join(self.root, "node-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + merge_id = "0" * 32 + delta_path = os.path.join(state_dir, ".coli_usage.delta.json") + ramdisk._usage_write(canonical, {"0:1": 12}, merge_id=merge_id) + ramdisk._atomic_json( + delta_path, + {"version": 1, "id": merge_id, "delta": {"0:1": 2}}, + ) + canonical_before = Path(canonical).read_bytes() + synced = [] + real_sync = state_support._fsync_bound_directory + canonical_parent = state_support._stat_identity(os.stat(model_dir)) + managed_parent = state_support._stat_identity(os.stat(state_dir)) + + def fail_canonical_sync(descriptor): + info = os.fstat(descriptor) + identity = (info.st_dev, info.st_ino) + if identity == canonical_parent: + raise OSError("canonical parent durability uncertain") + return real_sync(descriptor) + + with mock.patch.object( + state_support, + "_fsync_bound_directory", + side_effect=fail_canonical_sync, + ): + with self.assertRaisesRegex( + OSError, + "canonical parent durability uncertain", + ): + ramdisk._recover_delta(state_dir, canonical) + + self.assertTrue(os.path.exists(delta_path)) + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + + def track_sync(descriptor): + info = os.fstat(descriptor) + synced.append((info.st_dev, info.st_ino)) + return real_sync(descriptor) + + with mock.patch.object( + state_support, + "_fsync_bound_directory", + side_effect=track_sync, + ): + ramdisk._recover_delta(state_dir, canonical) + + self.assertIn(canonical_parent, synced) + self.assertIn(managed_parent, synced) + self.assertLess( + synced.index(canonical_parent), + synced.index(managed_parent), + ) + self.assertFalse(os.path.exists(delta_path)) + + def test_durable_unlink_reports_unproven_directory_commit(self): + path = os.path.join(self.root, "journal.json") + Path(path).write_text("{}\n", encoding="utf-8") + with mock.patch.object( + state_support, + "_fsync_directory", + side_effect=( + OSError(5, "synthetic directory EIO"), + None, + ), + ) as sync_directory: + with self.assertRaisesRegex(OSError, "synthetic directory EIO"): + state_support._durable_unlink(path) + state_support._durable_unlink(path) + + self.assertFalse(os.path.exists(path)) + self.assertEqual(sync_directory.call_count, 2) + sync_directory.assert_has_calls( + [mock.call(self.root), mock.call(self.root)] + ) + + def test_private_state_directory_rejects_existing_symlink_without_chmod(self): + target = os.path.join(self.root, "redirect-target") + link = os.path.join(self.root, "redirect-link") + os.mkdir(target, 0o755) + os.symlink(target, link) + before = os.stat(target).st_mode & 0o777 + with self.assertRaisesRegex(ramdisk.RamdiskError, "contains a symlink"): + ramdisk._ensure_private_dir(link) + self.assertEqual(os.stat(target).st_mode & 0o777, before) + + def test_derived_state_directory_must_remain_on_durable_filesystem(self): + state_dir = os.path.join(self.root, "engine-state") + os.mkdir(state_dir) + with mock.patch.object(ramdisk, "_filesystem_for_path", return_value="tmpfs"): + with self.assertRaisesRegex(ramdisk.RamdiskError, "volatile filesystem"): + ramdisk._assert_durable_state_dir(state_dir, plan=self.manifest()["plan"]) + + def test_two_node_usage_markers_survive_crash_between_manifest_saves(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + os.makedirs(model_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + records = [] + for index, value in enumerate((12, 13), 1): + state_dir = os.path.join(self.root, "node-%d" % index) + os.makedirs(state_dir) + ramdisk._usage_write(os.path.join(state_dir, ".coli_usage"), {"0:1": value}) + record = { + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": ("%x" % index) * 32, + } + ramdisk._merge_usage(record, canonical) + records.append(record) + self.assertEqual(ramdisk._usage_read(canonical), {"0:1": 15}) + self.assertEqual(ramdisk._usage_merge_ids(canonical), {"1" * 32, "2" * 32}) + + # Simulate both records still looking uncommitted after a manager crash. + for index, record in enumerate(records, 1): + ramdisk._atomic_json( + os.path.join(record["state_dir"], ".coli_usage.delta.json"), + {"version": 1, "id": record["usage_merge_id"], "delta": {"0:1": index + 1}}, + ) + ramdisk._recover_delta(record["state_dir"], canonical) + self.assertEqual(ramdisk._usage_read(canonical), {"0:1": 15}) + + def test_process_identity_rejects_uid_starttime_and_nonce_mismatch(self): + record = {"pid": 44, "uid": 1000, "starttime": 99, "nonce": "expected"} + with mock.patch.object( + ramdisk, + "_proc_identity", + return_value={"pid": 44, "uid": 1000, "starttime": 99, "nonce": "other", "pgid": 44}, + ): + matches, reason, _ = ramdisk._process_matches(record) + self.assertFalse(matches) + self.assertEqual(reason, "foreign-nonce") + + @requires_linux_operational + def test_manifest_rejects_missing_nonce_before_process_signaling(self): + manifest = self.manifest( + state="running", processes=[{"pid": 12345}] + ) + manifest["processes"][0].pop("nonce") + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with self.assertRaisesRegex(ramdisk.RamdiskError, "unsafe managed process"): + ramdisk._load_manifest(required=True) + + def test_manifest_rejects_corrupt_process_accounting_metadata(self): + manifest = self.manifest( + state="running", + processes=[{"pid": 12346}], + ) + manifest["processes"][0]["usage_baseline"] = {"0:1": True} + manifest["processes"][0]["usage_merge_id"] = "UPPERCASE" + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unsafe managed process", + ): + ramdisk._load_manifest(required=True) + + def test_pending_operation_id_must_bind_usage_transaction(self): + pending = { + "operation_id": "start:" + ("1" * 32), + "nonce": "2" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "3" * 32, + } + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [pending] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pending launch recovery", + ): + ramdisk._load_manifest(required=True) + + def test_usage_transaction_ids_are_unique_across_all_authorities(self): + mounts = [ + "/mnt/colibri-test/node0", + "/mnt/colibri-test/node1", + "/mnt/colibri-test/node2", + ] + base = self.manifest( + state="error", + mount_paths=mounts, + processes=[{"pid": 12020}], + ) + base["processes"][0]["usage_merge_id"] = "4" * 32 + base["pending_launches"] = [ + { + "operation_id": "start:" + ("5" * 32), + "nonce": "6" * 48, + "uid": host_uid(), + "port": 8001, + "node": 1, + "state_dir": self.recovery_state_dir(node=1), + "weights_dir": mounts[1], + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "5" * 32, + } + ] + base["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": [ + { + "pid": 12022, + "pgid": 12022, + "node": 2, + "state_dir": self.recovery_state_dir(node=2), + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "7" * 32, + "error": "group absence unproven", + } + ], + } + + ramdisk._atomic_json(ramdisk._manifest_path(), base) + loaded = ramdisk._load_manifest(required=True) + self.assertEqual(loaded["processes"][0]["usage_merge_id"], "4" * 32) + + duplicate_cases = ( + ("process-pending", ("pending_launches", 0), "4" * 32), + ( + "process-retained", + ("recovery", "retained_processes", 0), + "4" * 32, + ), + ( + "pending-retained", + ("recovery", "retained_processes", 0), + "5" * 32, + ), + ) + for label, path, duplicate in duplicate_cases: + with self.subTest(label=label): + manifest = copy.deepcopy(base) + target = manifest + for key in path: + target = target[key] + target["usage_merge_id"] = duplicate + if path[0] == "pending_launches": + target["operation_id"] = "start:" + duplicate + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "duplicate usage transaction", + ): + ramdisk._load_manifest(required=True) + + legacy = copy.deepcopy(base) + legacy["processes"][0].pop("usage_merge_id") + ramdisk._atomic_json(ramdisk._manifest_path(), legacy) + self.assertIsNone( + ramdisk._load_manifest(required=True)["processes"][0].get( + "usage_merge_id" + ) + ) + + def test_manifest_rejects_untrusted_process_usage_merge_marker(self): + cases = ( + (True, "e" * 32), + ("not-a-timestamp", "e" * 32), + ("2026-08-01T00:00:00Z", None), + ) + for marker, merge_id in cases: + with self.subTest(marker=marker, merge_id=merge_id): + manifest = self.manifest( + state="stopped", + processes=[ + { + "pid": 12347, + "usage_merge_id": merge_id, + "usage_merged_at": marker, + } + ], + ) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unsafe managed process", + ): + ramdisk._load_manifest(required=True) + + def test_manifest_rejects_recovery_state_dir_outside_exact_node_path(self): + manifest = self.manifest(state="error") + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": [ + { + "pid": 12347, + "pgid": 12347, + "node": None, + "state_dir": os.path.join(self.root, "arbitrary-state"), + "usage_baseline": {}, + "usage_merge_id": "a" * 32, + "error": "absence unproven", + } + ], + } + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unsafe or duplicate state directory", + ): + ramdisk._load_manifest(required=True) + + def test_manifest_rejects_mount_layout_outside_v1_root(self): + manifest = self.manifest() + manifest["plan"]["mount_root"] = os.path.join(self.root, "mount") + manifest["plan"]["mounts"][0]["path"] = manifest["plan"]["mount_root"] + manifest["mounts"][0]["path"] = manifest["plan"]["mount_root"] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with self.assertRaisesRegex(ramdisk.RamdiskError, "unsafe mount root"): + ramdisk._load_manifest(required=True) + + def test_error_manifest_accepts_pending_mount_without_an_identity(self): + manifest = self.manifest(state="error") + manifest["mounts"][0].pop("identity") + manifest["mounts"][0]["ownership"] = "pending" + manifest["error"] = "mount helper outcome is unknown" + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + loaded = ramdisk._load_manifest(required=True) + + self.assertEqual(loaded["state"], "error") + self.assertEqual(loaded["mounts"][0]["ownership"], "pending") + self.assertNotIn("identity", loaded["mounts"][0]) + + def test_ready_manifest_rejects_pending_mount_ownership(self): + manifest = self.manifest(state="ready") + manifest["mounts"][0].pop("identity") + manifest["mounts"][0]["ownership"] = "pending" + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with self.assertRaisesRegex(ramdisk.RamdiskError, "pending mount"): + ramdisk._load_manifest(required=True) + + def test_manifest_rejects_retained_process_recovery_outside_error_state(self): + manifest = self.manifest(state="stopped") + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": [ + { + "pid": 12001, + "pgid": 12001, + "node": None, + "state_dir": self.recovery_state_dir(), + "usage_baseline": {"0:1": 7}, + "usage_merge_id": "1" * 32, + "error": "group absence unproven", + } + ], + } + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "retained process recovery.*error state", + ): + ramdisk._load_manifest(required=True) + + def test_outcome_unknown_pending_launch_is_reconciled_only_by_stop(self): + pending = { + "operation_id": "start:" + ("6" * 32), + "nonce": "7" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "6" * 32, + } + + def write_pending(): + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [copy.deepcopy(pending)] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + write_pending() + with mock.patch.object(ramdisk.subprocess, "Popen") as popen: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pre-spawn managed launch has an unknown outcome", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + popen.assert_not_called() + + write_pending() + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + side_effect=[[], []], + ) as discover, mock.patch.object( + ramdisk, "_terminate_verified_group" + ) as terminate, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + stopped = ramdisk.stop() + terminate.assert_not_called() + self.assertEqual(discover.call_count, 2) + merge.assert_called_once() + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(stopped["pending_launches"], []) + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "stopped") + self.assertEqual(persisted["pending_launches"], []) + + def test_pending_live_group_identity_is_persisted_before_stop_signals(self): + pending = { + "operation_id": "start:" + ("8" * 32), + "nonce": "9" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "8" * 32, + } + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [copy.deepcopy(pending)] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + candidate = { + "pid": 12010, + "uid": host_uid(), + "starttime": 400, + "nonce": pending["nonce"], + "pgid": 12010, + "sid": 12010, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["coli", "serve"], + } + + def terminate(record): + durable = ramdisk._load_manifest(required=True) + observed = durable["pending_launches"][0]["observed_group"] + self.assertEqual(observed["pgid"], 12010) + self.assertEqual(observed["leader_starttime"], 400) + self.assertEqual(record["starttime"], 400) + return None + + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + side_effect=[[candidate], []], + ), mock.patch.object( + ramdisk, + "_process_group_members", + return_value=([candidate], []), + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(True, "running", candidate), + ), mock.patch.object( + ramdisk, "_group_alive", return_value=False + ), mock.patch.object( + ramdisk, "_terminate_verified_group", side_effect=terminate + ) as terminate_group, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + stopped = ramdisk.stop() + + terminate_group.assert_called_once() + merge.assert_called_once() + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(stopped["pending_launches"], []) + + def test_manifest_rejects_untrusted_pending_usage_merge_timestamp(self): + pending = { + "operation_id": "start:" + ("4" * 32), + "nonce": "5" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "4" * 32, + } + for marker in (True, "not-a-timestamp"): + with self.subTest(marker=marker): + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [ + dict(pending, usage_merged_at=marker) + ] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unsafe pending launch recovery", + ): + ramdisk._load_manifest(required=True) + + def test_pending_wrapper_dead_group_uses_member_verified_pgid(self): + pending = { + "operation_id": "start:" + ("a" * 32), + "nonce": "b" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "a" * 32, + "observed_group": { + "pgid": 12011, + "uid": host_uid(), + "leader_starttime": 401, + }, + } + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [copy.deepcopy(pending)] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + child = { + "pid": 12012, + "uid": host_uid(), + "starttime": 402, + "nonce": pending["nonce"], + "pgid": 12011, + "sid": 12011, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["colibri"], + } + + def terminate(record): + self.assertEqual(record["pid"], 12011) + self.assertEqual(record["pgid"], 12011) + self.assertIsNone(record["starttime"]) + durable = ramdisk._load_manifest(required=True) + self.assertEqual( + durable["pending_launches"][0]["observed_group"] + ["leader_starttime"], + 401, + ) + return None + + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + side_effect=[[child], []], + ), mock.patch.object( + ramdisk, + "_process_group_members", + return_value=([child], []), + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(True, "running-group", {"members": [child]}), + ), mock.patch.object( + ramdisk, "_group_alive", return_value=False + ), mock.patch.object( + ramdisk, "_terminate_verified_group", side_effect=terminate + ) as terminate_group, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + stopped = ramdisk.stop() + + terminate_group.assert_called_once() + merge.assert_called_once() + self.assertEqual(stopped["state"], "stopped") + + def test_pending_preflight_accepts_the_exact_inert_zombie_leader(self): + pending = { + "operation_id": "start:" + ("a" * 32), + "nonce": "b" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "a" * 32, + } + zombie = { + "pid": 12011, + "uid": host_uid(), + "state": "Z", + "inert": True, + "starttime": 401, + "nonce": None, + "pgid": 12011, + "sid": 12011, + "state_dir": None, + "weights_dir": None, + } + child = { + "pid": 12012, + "uid": host_uid(), + "state": "S", + "inert": False, + "starttime": 402, + "nonce": pending["nonce"], + "pgid": 12011, + "sid": 12011, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["colibri"], + } + + def process_matches(record): + self.assertEqual(record["pid"], 12011) + self.assertEqual(record["starttime"], 401) + return True, "running-group", {"members": [zombie, child]} + + preflights, failures = lifecycle_support._preflight_pending_launches( + {"pending_launches": [pending]}, + discover_managed_launches=mock.Mock(return_value=[child]), + process_matches=process_matches, + process_group_members=mock.Mock(return_value=([zombie, child], [])), + group_alive=mock.Mock(return_value=True), + ) + + self.assertEqual(failures, []) + self.assertEqual(preflights[0]["observed_group"]["leader_starttime"], 401) + + def test_pending_preflight_accepts_exact_inert_nonleader(self): + pending = { + "operation_id": "start:" + ("a" * 32), + "nonce": "b" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "a" * 32, + } + leader = { + "pid": 12011, + "uid": host_uid(), + "inert": False, + "starttime": 401, + "nonce": pending["nonce"], + "pgid": 12011, + "sid": 12011, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["coli", "serve"], + } + zombie = { + "pid": 12012, + "uid": host_uid(), + "state": "Z", + "inert": True, + "starttime": 402, + "nonce": None, + "pgid": 12011, + "sid": 12011, + "state_dir": None, + "weights_dir": None, + } + process_matches = mock.Mock( + return_value=(True, "running", {"members": [leader, zombie]}) + ) + + preflights, failures = lifecycle_support._preflight_pending_launches( + {"pending_launches": [pending]}, + discover_managed_launches=mock.Mock(return_value=[leader]), + process_matches=process_matches, + process_group_members=mock.Mock( + return_value=([leader, zombie], []) + ), + group_alive=mock.Mock(return_value=True), + ) + + self.assertEqual(failures, []) + self.assertEqual(preflights[0]["record"]["starttime"], 401) + process_matches.assert_called_once() + + def test_pending_preflight_rejects_mismatched_inert_nonleader(self): + pending = { + "operation_id": "start:" + ("a" * 32), + "nonce": "b" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "a" * 32, + } + leader = { + "pid": 12011, + "uid": host_uid(), + "inert": False, + "starttime": 401, + "nonce": pending["nonce"], + "pgid": 12011, + "sid": 12011, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["coli", "serve"], + } + zombie = { + "pid": 12012, + "uid": host_uid(), + "state": "Z", + "inert": True, + "starttime": 402, + "nonce": None, + "pgid": 12011, + "sid": 12011, + "state_dir": None, + "weights_dir": None, + } + for field, value in ( + ("uid", host_uid() + 1), + ("pgid", 12099), + ("sid", 12099), + ): + with self.subTest(field=field): + process_matches = mock.Mock() + mismatched = dict(zombie, **{field: value}) + preflights, failures = ( + lifecycle_support._preflight_pending_launches( + {"pending_launches": [pending]}, + discover_managed_launches=mock.Mock( + return_value=[leader] + ), + process_matches=process_matches, + process_group_members=mock.Mock( + return_value=([leader, mismatched], []) + ), + group_alive=mock.Mock(return_value=True), + ) + ) + + self.assertEqual(preflights, []) + self.assertRegex(failures[0], "foreign or mismatched member") + process_matches.assert_not_called() + + def test_pending_launch_ambiguous_groups_refuse_without_side_effects(self): + pending = { + "operation_id": "start:" + ("c" * 32), + "nonce": "d" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "c" * 32, + } + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [copy.deepcopy(pending)] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + def candidate(pid, pgid): + return { + "pid": pid, + "uid": host_uid(), + "starttime": 500 + pid, + "nonce": pending["nonce"], + "pgid": pgid, + "sid": pgid, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["colibri"], + } + + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + return_value=[candidate(12020, 12020), candidate(12021, 12021)], + ), mock.patch.object( + ramdisk, "_terminate_verified_group" + ) as terminate, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "multiple process groups", + ): + ramdisk.stop() + + terminate.assert_not_called() + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(len(persisted["pending_launches"]), 1) + with mock.patch.object(ramdisk, "_mount_at", return_value=None): + recovery_status = ramdisk.status(deep=False) + self.assertNotIn( + pending["usage_merge_id"], + json.dumps(recovery_status, sort_keys=True), + ) + + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + side_effect=[[], []], + ), mock.patch.object(ramdisk, "_merge_usage"): + stopped = ramdisk.stop() + self.assertEqual(stopped["state"], "stopped") + self.assertNotIn("cleanup_errors", stopped) + + def test_pending_launch_waits_for_ordinary_process_global_preflight(self): + pending = { + "operation_id": "start:" + ("e" * 32), + "nonce": "f" * 48, + "uid": host_uid(), + "port": 8001, + "node": 1, + "state_dir": os.path.join(self.root, "pending-state"), + "weights_dir": "/mnt/colibri-test/node1", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "e" * 32, + } + candidate = { + "pid": 12030, + "uid": host_uid(), + "starttime": 530, + "nonce": pending["nonce"], + "pgid": 12030, + "sid": 12030, + "state_dir": pending["state_dir"], + "weights_dir": pending["weights_dir"], + "cmdline": ["colibri"], + } + ordinary = { + "pid": 12031, + "pgid": 12031, + "uid": host_uid(), + "nonce": "1" * 48, + "state_dir": os.path.join(self.root, "ordinary-state"), + "usage_baseline": {}, + } + manifest = { + "state": "starting", + "plan": { + "model": {"path": os.path.join(self.root, "model")}, + "mounts": [ + {"node": 0, "path": "/mnt/colibri-test/node0"}, + {"node": 1, "path": pending["weights_dir"]}, + ], + }, + "mounts": [ + {"node": 0, "path": "/mnt/colibri-test/node0"}, + {"node": 1, "path": pending["weights_dir"]}, + ], + "processes": [ordinary], + "pending_launches": [pending], + } + process_matches = mock.Mock( + side_effect=[ + (True, "running", candidate), + (False, "foreign-uid", None), + ] + ) + terminate = mock.Mock() + merge = mock.Mock() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unverified processes", + ): + lifecycle_support.stop( + load_manifest=mock.Mock(return_value=manifest), + discover_managed_launches=mock.Mock(return_value=[candidate]), + process_matches=process_matches, + process_group_members=mock.Mock(return_value=([candidate], [])), + group_alive=mock.Mock(), + managed_child_liveness=mock.Mock(return_value=None), + save_manifest=mock.Mock(), + terminate_verified_group=terminate, + merge_usage=merge, + bind_usage_transaction=mock.Mock(), + ) + + terminate.assert_not_called() + merge.assert_not_called() + self.assertNotIn("observed_group", pending) + + def test_pending_launch_retry_does_not_repeat_merged_transaction(self): + pending = { + "operation_id": "start:" + ("2" * 32), + "nonce": "3" * 48, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": self.recovery_state_dir(), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 3}, + "usage_merge_id": "2" * 32, + } + manifest = self.manifest(state="starting") + manifest["pending_launches"] = [copy.deepcopy(pending)] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + real_save = ramdisk._save_manifest + save_calls = [] + + def fail_pending_removal(value): + save_calls.append(copy.deepcopy(value)) + if len(save_calls) == 2: + raise OSError("pending removal save failed") + return real_save(value) + + applied_transactions = set() + applications = [] + + def idempotent_merge(record, _canonical, plan=None): + del plan + merge_id = record["usage_merge_id"] + if merge_id not in applied_transactions: + applied_transactions.add(merge_id) + applications.append(merge_id) + + merge = mock.Mock(side_effect=idempotent_merge) + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + side_effect=[[], []], + ), mock.patch.object( + ramdisk, "_merge_usage", merge + ), mock.patch.object( + ramdisk, "_save_manifest", side_effect=fail_pending_removal + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "pending authority could not be removed", + ): + ramdisk.stop() + + after_failure = ramdisk._load_manifest(required=True) + self.assertEqual(len(after_failure["pending_launches"]), 1) + self.assertIn("usage_merged_at", after_failure["pending_launches"][0]) + merge.assert_called_once() + self.assertEqual(applications, [pending["usage_merge_id"]]) + + with mock.patch.object( + ramdisk, + "_managed_launch_processes", + side_effect=[[], []], + ), mock.patch.object(ramdisk, "_merge_usage", merge): + stopped = ramdisk.stop() + + self.assertEqual(merge.call_count, 2) + self.assertEqual(applications, [pending["usage_merge_id"]]) + self.assertEqual(stopped["state"], "stopped") + self.assertEqual(stopped["pending_launches"], []) + + def test_start_refuses_unresolved_unpublished_process_recovery(self): + manifest = self.manifest(state="error") + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": [ + { + "pid": 12002, + "pgid": 12002, + "node": None, + "state_dir": self.recovery_state_dir(), + "usage_baseline": {"0:1": 7}, + "usage_merge_id": "2" * 32, + "error": "group absence unproven", + } + ], + } + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with mock.patch.object(ramdisk.subprocess, "Popen") as popen: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unpublished managed-child absence is unproven", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + popen.assert_not_called() + + def test_stop_refuses_unresolved_unpublished_process_recovery(self): + manifest = self.manifest(state="error") + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": [ + { + "pid": 12003, + "pgid": 12003, + "node": None, + "state_dir": self.recovery_state_dir(), + "usage_baseline": {"0:1": 7}, + "usage_merge_id": "3" * 32, + "error": "group absence unproven", + } + ], + } + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with mock.patch.object( + ramdisk, "_terminate_verified_group" + ) as terminate, mock.patch.object( + ramdisk, "_group_alive", return_value=True + ), mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unpublished.*unproven", + ): + ramdisk.stop() + + terminate.assert_not_called() + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertEqual( + persisted["recovery"]["retained_processes"][0]["pid"], + 12003, + ) + + def test_stop_reconciles_unpublished_usage_once_with_exact_baseline(self): + merge_id = "5" * 32 + model_dir = os.path.join(self.root, "accounting-model") + state_dir = os.path.join(self.root, "accounting-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + state_support._usage_write(canonical, {"0:1": 10}) + state_support._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 12}, + ) + entry = { + "pid": 12005, + "pgid": 12005, + "node": None, + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": merge_id, + "error": "group absence unproven", + "authority_version": lifecycle_support._RETAINED_AUTHORITY_VERSION, + "nonce": "a" * 48, + "uid": 1000, + "weights_dir": os.path.join(self.root, "accounting-weights"), + "launch_not_before": 1, + "launcher_pid": 1, + "launcher_starttime": 1, + "launcher_cmdline": ["coli"], + "expected_command": ["coli", "serve"], + } + manifest = { + "state": "error", + "plan": { + "model": {"path": model_dir}, + "mounts": [], + }, + "recovery": { + "operation": "start", + "state": "attention-required", + "retained_processes": [entry], + }, + } + + def merge(record, canonical_path, plan=None): + self.assertEqual(record["usage_baseline"], {"0:1": 10}) + self.assertEqual(record["usage_merge_id"], merge_id) + return state_support._merge_usage( + record, + canonical_path, + plan=plan, + filesystem_for_path=lambda ignored: "ext4", + source_still_matches=lambda ignored: None, + ) + + saves = {"count": 0} + + def fail_first_post_merge_save(_manifest): + saves["count"] += 1 + if saves["count"] == 1: + raise OSError("manager lost manifest write after merge") + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "manager lost manifest write after merge", + ): + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=lambda ignored: False, + discover_managed_launches=mock.Mock(return_value=[]), + merge_usage=merge, + save_manifest=fail_first_post_merge_save, + ) + self.assertEqual(state_support._usage_read(canonical), {"0:1": 12}) + + # A fresh manager reloads the still-retained transaction and retries + # the same stable id. The canonical marker makes that retry a no-op. + restarted = copy.deepcopy(manifest) + lifecycle_support._reconcile_unpublished_processes( + restarted, + group_alive=lambda ignored: False, + discover_managed_launches=mock.Mock(return_value=[]), + merge_usage=merge, + save_manifest=lambda ignored: None, + ) + + self.assertEqual(state_support._usage_read(canonical), {"0:1": 12}) + self.assertEqual( + restarted["recovery"]["retained_processes"], + [], + ) + + def _retained_entry_with_authority( + self, + state_dir, + merge_id, + *, + pid=12009, + nonce="a" * 48, + ): + """A retained unpublished entry carrying full discovery authority.""" + return { + "pid": pid, + "pgid": pid, + "node": None, + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": merge_id, + "error": "group absence unproven", + "authority_version": lifecycle_support._RETAINED_AUTHORITY_VERSION, + "nonce": nonce, + "uid": 1000, + "weights_dir": os.path.join(self.root, "authority-weights"), + "launch_not_before": 1, + "launcher_pid": 1, + "launcher_starttime": 1, + "launcher_cmdline": ["coli"], + "expected_command": ["coli", "serve"], + } + + def _unpublished_usage_manifest(self, merge_id, baseline_delta=2): + model_dir = os.path.join(self.root, "unpub-model-%s" % merge_id) + state_dir = os.path.join(self.root, "unpub-state-%s" % merge_id) + os.makedirs(model_dir) + os.makedirs(state_dir) + canonical = os.path.join(model_dir, ".coli_usage") + state_support._usage_write(canonical, {"0:1": 10}) + state_support._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 10 + baseline_delta}, + ) + entry = self._retained_entry_with_authority(state_dir, merge_id) + manifest = { + "state": "error", + "plan": {"model": {"path": model_dir}, "mounts": []}, + "recovery": { + "operation": "start", + "state": "attention-required", + "retained_processes": [entry], + }, + } + return manifest, canonical, entry + + def _unpublished_merge(self, canonical): + def merge(record, canonical_path, plan=None): + return state_support._merge_usage( + record, + canonical_path, + plan=plan, + filesystem_for_path=lambda ignored: "ext4", + source_still_matches=lambda ignored: None, + ) + return merge + + def test_retained_authority_helper_preserves_discovery_metadata(self): + pending_entry = { + "nonce": "deadbeef" * 6, + "uid": 1000, + "weights_dir": "/srv/w", + "launch_not_before": 12345, + "launcher_pid": 99, + "launcher_starttime": 4321, + "launcher_cmdline": ["coli", "ramdisk"], + "expected_command": ["coli", "serve"], + } + authority = lifecycle_support._retained_process_authority(pending_entry) + self.assertEqual( + authority["authority_version"], + lifecycle_support._RETAINED_AUTHORITY_VERSION, + ) + entry = dict( + pid=12010, + pgid=12010, + state_dir="/srv/state", + usage_baseline={"0:1": 1}, + usage_merge_id="c" * 32, + **authority, + ) + validated = lifecycle_support._retained_authority(entry) + self.assertEqual(validated["nonce"], pending_entry["nonce"]) + self.assertEqual(validated["uid"], pending_entry["uid"]) + self.assertEqual( + validated["weights_dir"], pending_entry["weights_dir"] + ) + self.assertEqual( + validated["launch_not_before"], + pending_entry["launch_not_before"], + ) + self.assertEqual( + validated["launcher_pid"], pending_entry["launcher_pid"] + ) + # Legacy records (no persisted authority) cannot be validated, so + # callers fail closed instead of guessing absence. + legacy = { + "pid": 1, + "pgid": 1, + "state_dir": "/s", + "usage_baseline": {}, + "usage_merge_id": "d" * 32, + } + self.assertIsNone(lifecycle_support._retained_authority(legacy)) + + def test_unpublished_recovery_refuses_without_discovery_authority(self): + # A legacy retained record (created before authority was persisted) + # cannot be positively proven absent even when the original group is + # gone: refuse to merge, keep it retained, mutate no accounting. + merge_id = "6" * 32 + manifest, canonical, _entry = self._unpublished_usage_manifest(merge_id) + authority_keys = ( + "authority_version", + "nonce", + "uid", + "weights_dir", + "launch_not_before", + "launcher_pid", + "launcher_starttime", + "launcher_cmdline", + "expected_command", + ) + manifest["recovery"]["retained_processes"][0] = { + key: value + for key, value in ( + manifest["recovery"]["retained_processes"][0].items() + ) + if key not in authority_keys + } + merge = mock.Mock() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "lacks durable discovery authority", + ): + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=lambda ignored: False, + discover_managed_launches=mock.Mock(return_value=[]), + merge_usage=merge, + save_manifest=lambda ignored: None, + ) + merge.assert_not_called() + self.assertEqual(state_support._usage_read(canonical), {"0:1": 10}) + self.assertEqual( + len(manifest["recovery"]["retained_processes"]), 1 + ) + + def test_unpublished_recovery_refuses_when_nonce_descendant_alive(self): + # Original group gone BUT a nonce-attributed descendant survives in a + # new session: absence is unproven, so no merge and the record is kept. + merge_id = "7" * 32 + manifest, canonical, entry = self._unpublished_usage_manifest(merge_id) + escaped = { + "pid": 4242, + "uid": entry["uid"], + "starttime": 999, + "nonce": entry["nonce"], + "pgid": 4242, + "sid": 4242, + "cmdline": entry["expected_command"], + "state_dir": entry["state_dir"], + "weights_dir": entry["weights_dir"], + } + merge = mock.Mock() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "nonce-attributable live process", + ): + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=lambda ignored: False, + discover_managed_launches=mock.Mock(return_value=[escaped]), + merge_usage=merge, + save_manifest=lambda ignored: None, + ) + merge.assert_not_called() + self.assertEqual(state_support._usage_read(canonical), {"0:1": 10}) + self.assertEqual( + len(manifest["recovery"]["retained_processes"]), 1 + ) + + def test_unpublished_recovery_refuses_when_global_discovery_fails(self): + merge_id = "8" * 32 + manifest, canonical, _entry = self._unpublished_usage_manifest(merge_id) + merge = mock.Mock() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "global nonce attribution failed", + ): + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=lambda ignored: False, + discover_managed_launches=mock.Mock( + side_effect=ramdisk.RamdiskError( + "process table changed during scan" + ) + ), + merge_usage=merge, + save_manifest=lambda ignored: None, + ) + merge.assert_not_called() + self.assertEqual(state_support._usage_read(canonical), {"0:1": 10}) + + def test_unpublished_recovery_revalidates_immediately_before_merge(self): + # Preflight proves absence, but a nonce-attributed process appears + # between preflight and the per-entry merge: the immediate re-check + # must refuse and mutate no accounting. + merge_id = "9" * 32 + manifest, canonical, entry = self._unpublished_usage_manifest(merge_id) + escaped = { + "pid": 5353, + "uid": entry["uid"], + "starttime": 999, + "nonce": entry["nonce"], + "pgid": 5353, + "sid": 5353, + "cmdline": entry["expected_command"], + "state_dir": entry["state_dir"], + "weights_dir": entry["weights_dir"], + } + # First call (preflight) sees nothing; the per-entry revalidation sees + # the escaped descendant and must block the merge. + discover = mock.Mock(side_effect=[[], [escaped]]) + merge = mock.Mock() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "nonce-attributable live process", + ): + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=lambda ignored: False, + discover_managed_launches=discover, + merge_usage=merge, + save_manifest=lambda ignored: None, + ) + merge.assert_not_called() + self.assertEqual(state_support._usage_read(canonical), {"0:1": 10}) + self.assertEqual( + len(manifest["recovery"]["retained_processes"]), 1 + ) + + def test_unpublished_recovery_merges_once_when_globally_absent(self): + merge_id = "1" * 32 + manifest, canonical, _entry = self._unpublished_usage_manifest(merge_id) + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=lambda ignored: False, + discover_managed_launches=mock.Mock(return_value=[]), + merge_usage=self._unpublished_merge(canonical), + save_manifest=lambda ignored: None, + ) + self.assertEqual(state_support._usage_read(canonical), {"0:1": 12}) + self.assertEqual( + manifest["recovery"]["retained_processes"], [] + ) + + @requires_linux_operational + def test_unpublished_recovery_real_setsid_descendant_refuses(self): + # Real kernel reproduction: a managed leader forks a child that calls + # setsid() and survives in a new session while the original leader's + # process group disappears. Recovery must not mistake the empty + # original group for absence while that descendant is alive. + import subprocess + import time as _time + + def stat_fields(pid): + with open("/proc/%d/stat" % pid) as handle: + raw = handle.read() + tail = raw[raw.rfind(")") + 2:].split() + return { + "state": tail[0], + "pgid": int(tail[2]), + "sid": int(tail[3]), + "starttime": int(tail[19]), + } + + def cmdline_of(pid): + with open("/proc/%d/cmdline" % pid, "rb") as handle: + raw = handle.read() + return [ + token.decode() + for token in raw.split(b"\0") + if token + ] + + nonce = os.urandom(24).hex() + state_dir = os.path.join(self.root, "setsid-state") + weights_dir = os.path.join(self.root, "setsid-weights") + os.makedirs(state_dir) + os.makedirs(weights_dir) + child_pid_file = os.path.join(self.root, "escaped-child.pid") + leader_script = ( + "import os, time\n" + "child = os.fork()\n" + "if child == 0:\n" + " os.setsid()\n" + " time.sleep(30)\n" + " os._exit(0)\n" + "else:\n" + " open(os.environ['CHILD_PID_FILE'], 'w').write(str(child))\n" + " os._exit(0)\n" + ) + expected_command = [sys.executable, "-c", leader_script] + env = dict(os.environ) + env.update( + COLI_MANAGED_NONCE=nonce, + COLI_STATE_DIR=state_dir, + COLI_WEIGHTS_DIR=weights_dir, + CHILD_PID_FILE=child_pid_file, + ) + leader = subprocess.Popen( + expected_command, + env=env, + start_new_session=True, + ) + original_pgid = leader.pid + leader.wait() + escaped_pid = None + for _ in range(200): + if os.path.exists(child_pid_file): + with open(child_pid_file) as handle: + escaped_pid = int(handle.read().strip()) + break + _time.sleep(0.05) + self.assertIsNotNone(escaped_pid) + _time.sleep(0.3) + try: + escaped_identity = stat_fields(escaped_pid) + self.assertNotEqual(escaped_identity["pgid"], original_pgid) + self.assertEqual( + escaped_identity["pgid"], escaped_identity["sid"] + ) + # Mechanism: the original group is gone, yet the global nonce scan + # still finds the re-sessioned descendant. + self.assertFalse( + linux_ops._process_group_alive(original_pgid) + ) + launcher_cmdline = cmdline_of(os.getpid()) + launcher_starttime = stat_fields(os.getpid())["starttime"] + scan = None + for _ in range(20): + try: + scan = linux_ops._managed_launch_processes( + nonce, + host_uid(), + state_dir=state_dir, + weights_dir=weights_dir, + not_before_starttime=escaped_identity["starttime"], + launcher_pid=os.getpid(), + launcher_starttime=launcher_starttime, + launcher_cmdline=launcher_cmdline, + expected_command=expected_command, + ) + break + except ramdisk.RamdiskError: + _time.sleep(0.2) + self.assertIsNotNone(scan) + self.assertTrue( + any( + candidate.get("pid") == escaped_pid + for candidate in scan + ) + ) + + # Reconciliation with the real scan and group check must refuse to + # merge accounting while the escaped descendant is alive. + merge_id = "b" * 32 + model_dir = os.path.join(self.root, "setsid-model") + os.makedirs(model_dir) + canonical = os.path.join(model_dir, ".coli_usage") + state_support._usage_write(canonical, {"0:1": 10}) + state_support._usage_write( + os.path.join(state_dir, ".coli_usage"), + {"0:1": 13}, + ) + entry = { + "pid": original_pgid, + "pgid": original_pgid, + "node": None, + "state_dir": state_dir, + "usage_baseline": {"0:1": 10}, + "usage_merge_id": merge_id, + "error": "group absence unproven", + "authority_version": ( + lifecycle_support._RETAINED_AUTHORITY_VERSION + ), + "nonce": nonce, + "uid": host_uid(), + "weights_dir": weights_dir, + "launch_not_before": escaped_identity["starttime"], + "launcher_pid": os.getpid(), + "launcher_starttime": launcher_starttime, + "launcher_cmdline": launcher_cmdline, + "expected_command": expected_command, + } + manifest = { + "state": "error", + "plan": {"model": {"path": model_dir}, "mounts": []}, + "recovery": { + "operation": "start", + "state": "attention-required", + "retained_processes": [entry], + }, + } + merge = mock.Mock() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "incomplete", + ): + lifecycle_support._reconcile_unpublished_processes( + manifest, + group_alive=linux_ops._process_group_alive, + discover_managed_launches=( + linux_ops._managed_launch_processes + ), + merge_usage=merge, + save_manifest=lambda ignored: None, + ) + merge.assert_not_called() + self.assertEqual( + state_support._usage_read(canonical), {"0:1": 10} + ) + self.assertEqual( + len(manifest["recovery"]["retained_processes"]), 1 + ) + finally: + try: + os.kill(escaped_pid, 9) + except (ProcessLookupError, OSError): + pass + try: + os.waitpid(escaped_pid, 0) + except (ChildProcessError, OSError): + pass + + def test_destroy_refuses_unresolved_unpublished_process_recovery(self): + manifest = self.manifest(state="error") + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_processes": [ + { + "pid": 12004, + "pgid": 12004, + "node": None, + "state_dir": self.recovery_state_dir(), + "usage_baseline": {"0:1": 7}, + "usage_merge_id": "4" * 32, + "error": "group absence unproven", + } + ], + } + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with mock.patch.object( + ramdisk, "_group_alive", return_value=True + ), mock.patch.object(ramdisk, "_umount_path") as unmount, mock.patch.object( + ramdisk, "_durable_unlink" + ) as unlink: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unpublished.*unproven", + ): + ramdisk.destroy(argparse.Namespace(yes=True)) + + unmount.assert_not_called() + unlink.assert_not_called() + + @requires_linux_operational + def test_stop_validates_every_pid_before_signaling_any(self): + manifest = self.manifest( + state="running", + mount_paths=["/mnt/colibri-test/node0", "/mnt/colibri-test/node1"], + processes=[{"pid": 1}, {"pid": 2}], + ) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + def match(record): + if record["pid"] == 1: + return True, "running", {"pgid": 1} + return False, "foreign-uid", {"pgid": 2} + + with mock.patch.object(ramdisk, "_process_matches", side_effect=match), mock.patch.object( + os, "killpg" + ) as kill: + with self.assertRaisesRegex(ramdisk.RamdiskError, "unverified"): + ramdisk.stop() + kill.assert_not_called() + + def test_stop_persists_procfs_preflight_failure_before_any_signal(self): + manifest = self.manifest( + state="running", + processes=[{"pid": 12340}], + ) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with mock.patch.object( + ramdisk, + "_process_matches", + side_effect=ramdisk.RamdiskError("procfs enumeration unreadable"), + ), mock.patch.object( + ramdisk, "_terminate_verified_group" + ) as terminate, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "unverified.*procfs enumeration unreadable", + ): + ramdisk.stop() + + terminate.assert_not_called() + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertIn( + "procfs enumeration unreadable", + persisted["processes"][0]["stop_error"], + ) + self.assertNotIn("stopped_at", persisted["processes"][0]) + + def test_stop_persists_post_termination_revalidation_failure(self): + manifest = self.manifest( + state="running", + processes=[{"pid": 12341}], + ) + os.makedirs(manifest["processes"][0]["state_dir"], exist_ok=True) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + running = (True, "running", {"pid": 12341, "pgid": 12341}) + + with mock.patch.object( + ramdisk, + "_process_matches", + side_effect=( + running, + ramdisk.RamdiskError("post-termination procfs unreadable"), + ), + ), mock.patch.object( + ramdisk, + "_terminate_verified_group", + side_effect=ramdisk.RamdiskError( + "termination revalidation unreadable" + ), + ), mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "termination revalidation unreadable", + ): + ramdisk.stop() + + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertIn( + "post-termination procfs unreadable", + persisted["processes"][0]["stop_error"], + ) + self.assertNotIn("stopped_at", persisted["processes"][0]) + + def test_stop_refuses_nonmanaged_mount_recovery(self): + for ownership in ("pending", "identified"): + with self.subTest(ownership=ownership): + manifest = self.manifest(state="error") + manifest["mounts"][0]["ownership"] = ownership + if ownership == "pending": + manifest["mounts"][0].pop("identity") + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with mock.patch.object( + ramdisk, "_terminate_verified_group" + ) as terminate, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "non-managed mount ownership", + ): + ramdisk.stop() + + terminate.assert_not_called() + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertEqual( + persisted["mounts"][0]["ownership"], + ownership, + ) + + @requires_linux_operational + def test_stop_revalidates_identity_before_escalating_to_sigkill(self): + record = { + "pid": 12345, + "pgid": 12345, + "uid": host_uid(), + "starttime": 91, + "nonce": "a" * 48, + } + ops = mock.Mock() + ops.signal_verified_process_group.side_effect = ( + {"status": "signaled", "members": [12345], "signaled": [12345]}, + { + "status": "foreign", + "reason": "reused-pid", + "members": [12345], + }, + ) + with mock.patch.object( + process_support, + "get_platform_ops", + return_value=ops, + ), mock.patch.object(os, "killpg", create=True) as kill: + failure = ramdisk._terminate_verified_group( + record, + term_seconds=0, + kill_seconds=0, + ) + + kill.assert_not_called() + self.assertEqual( + [call.args[1] for call in ops.signal_verified_process_group.call_args_list], + [signal.SIGTERM, signal.SIGKILL], + ) + self.assertIn("reused-pid", failure) + + @requires_linux_operational + def test_verified_stop_reaps_a_locally_owned_zombie_before_escalation(self): + record = { + "pid": 12346, + "pgid": 12346, + "uid": host_uid(), + "starttime": 92, + "nonce": "b" * 48, + } + process = mock.Mock(pid=12346) + process.poll.return_value = 0 + ops = mock.Mock() + ops.signal_verified_process_group.side_effect = ( + {"status": "signaled", "members": [12346], "signaled": [12346]}, + {"status": "absent", "members": []}, + ) + ramdisk._track_managed_child(process) + self.addCleanup(ramdisk._forget_managed_child, process.pid) + with mock.patch.object( + process_support, + "get_platform_ops", + return_value=ops, + ), mock.patch.object(os, "killpg", create=True) as kill: + failure = ramdisk._terminate_verified_group( + record, + term_seconds=0, + kill_seconds=0, + ) + + self.assertIsNone(failure) + kill.assert_not_called() + self.assertNotIn(12346, ramdisk._managed_children) + + @requires_linux_operational + def test_verified_termination_treats_retained_live_child_as_independent_evidence(self): + record = { + "pid": 12347, + "pgid": 12347, + "uid": host_uid(), + "starttime": 93, + "nonce": "c" * 48, + } + process = mock.Mock(pid=12347, returncode=None) + process.poll.return_value = None + ramdisk._track_managed_child(process) + self.addCleanup(ramdisk._forget_managed_child, process.pid) + + ops = mock.Mock() + ops.signal_verified_process_group.return_value = { + "status": "absent", + "members": [], + } + with mock.patch.object( + process_support, + "get_platform_ops", + return_value=ops, + ), mock.patch.object(os, "killpg", create=True) as kill: + failure = ramdisk._terminate_verified_group( + record, + term_seconds=0, + kill_seconds=0, + ) + + kill.assert_not_called() + self.assertIn("retained managed child is still live", failure) + self.assertTrue(ramdisk._managed_child_liveness(process.pid)) + + def test_missing_retained_handle_preserves_not_running_result(self): + record = { + "pid": 12350, + "pgid": 12350, + "uid": host_uid(), + "starttime": 94, + "nonce": "d" * 48, + } + self.assertIsNone(ramdisk._managed_child_liveness(record["pid"])) + + ops = mock.Mock() + ops.signal_verified_process_group.return_value = { + "status": "absent", + "members": [], + } + with mock.patch.object( + process_support, + "get_platform_ops", + return_value=ops, + ), mock.patch.object(os, "killpg", create=True) as kill: + failure = ramdisk._terminate_verified_group( + record, + term_seconds=0, + kill_seconds=0, + ) + + self.assertIsNone(failure) + kill.assert_not_called() + + @requires_linux_operational + def test_stop_persists_error_when_usage_merge_fails(self): + manifest = self.manifest(state="running", processes=[{"pid": 12345}]) + os.makedirs(manifest["processes"][0]["state_dir"], exist_ok=True) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with mock.patch.object( + ramdisk, "_process_matches", return_value=(False, "not-running", None) + ), mock.patch.object( + ramdisk, "_merge_usage", side_effect=ramdisk.RamdiskError("disk unavailable") + ): + with self.assertRaisesRegex(ramdisk.RamdiskError, "cleanup is incomplete"): + ramdisk.stop() + persisted = ramdisk._read_json(ramdisk._manifest_path()) + self.assertEqual(persisted["state"], "error") + self.assertIn("disk unavailable", persisted["processes"][0]["usage_merge_error"]) + + def test_stop_retry_clears_post_merge_save_error_without_double_merge(self): + manifest = self.manifest( + state="running", + processes=[{"pid": 12351}], + ) + durable = {"manifest": None} + saves = {"count": 0} + merge_ids = [] + + def replay_merge(record, canonical_usage, *, plan): + del canonical_usage, plan + merge_ids.append(record["usage_merge_id"]) + + merge = mock.Mock(side_effect=replay_merge) + + def fail_first_completion_save(current): + saves["count"] += 1 + if saves["count"] == 2: + raise OSError("manifest write failed after usage merge") + durable["manifest"] = copy.deepcopy(current) + + common = { + "process_matches": lambda record: ( + False, + "not-running", + None, + ), + "group_alive": lambda pgid: False, + "managed_child_liveness": lambda pid: False, + "terminate_verified_group": mock.Mock(), + "merge_usage": merge, + "bind_usage_transaction": ( + lambda record, plan, reserved_ids: record.setdefault( + "usage_merge_id", + "e" * 32, + ) + ), + } + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "manifest write failed after usage merge", + ): + lifecycle_support.stop( + load_manifest=lambda required=True: manifest, + save_manifest=fail_first_completion_save, + **common, + ) + + restarted = copy.deepcopy(durable["manifest"]) + stopped = lifecycle_support.stop( + load_manifest=lambda required=True: restarted, + save_manifest=lambda current: durable.update( + manifest=copy.deepcopy(current) + ), + **common, + ) + + self.assertEqual(merge.call_count, 2) + self.assertEqual(len(set(merge_ids)), 1) + self.assertEqual(stopped["state"], "stopped") + self.assertIn("usage_merged_at", stopped["processes"][0]) + self.assertNotIn("usage_merge_error", stopped["processes"][0]) + + def test_stop_persists_legacy_journal_id_before_usage_replay(self): + manifest = self.manifest( + state="running", + processes=[{"pid": 12361}], + ) + record = manifest["processes"][0] + record.pop("usage_merge_id", None) + state_dir = record["state_dir"] + os.makedirs(state_dir, exist_ok=True) + merge_id = "a" * 32 + state_support._atomic_json( + os.path.join(state_dir, ".coli_usage.delta.json"), + {"version": 1, "id": merge_id, "delta": {"0:1": 2}}, + ) + events = [] + + def save(current): + events.append( + ("save", current["processes"][0].get("usage_merge_id")) + ) + + def merge(current, canonical, *, plan): + del canonical, plan + events.append(("merge", current.get("usage_merge_id"))) + + with mock.patch.object( + ramdisk, + "_load_manifest", + return_value=manifest, + ), mock.patch.object( + ramdisk, + "_managed_launch_processes", + return_value=[], + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(False, "not-running", None), + ), mock.patch.object( + ramdisk, + "_managed_child_liveness", + return_value=False, + ), mock.patch.object( + ramdisk, + "_group_alive", + return_value=False, + ), mock.patch.object( + ramdisk, + "_save_manifest", + side_effect=save, + ), mock.patch.object( + ramdisk, + "_merge_usage", + side_effect=merge, + ): + ramdisk.stop.__wrapped__() + + self.assertEqual(events[0], ("save", merge_id)) + self.assertIn(("merge", merge_id), events) + + def test_start_persists_legacy_journal_id_before_usage_replay(self): + manifest = self.manifest( + state="stopped", + processes=[ + { + "pid": 12362, + "stopped_at": "2026-08-01T00:00:00Z", + } + ], + ) + record = manifest["processes"][0] + record.pop("usage_merge_id", None) + state_dir = record["state_dir"] + os.makedirs(state_dir, exist_ok=True) + merge_id = "b" * 32 + state_support._atomic_json( + os.path.join(state_dir, ".coli_usage.delta.json"), + {"version": 1, "id": merge_id, "delta": {"0:1": 2}}, + ) + events = [] + + def save(current): + events.append( + ("save", current["processes"][0].get("usage_merge_id")) + ) + + def merge(current, canonical, *, plan): + del canonical, plan + events.append(("merge", current.get("usage_merge_id"))) + + with mock.patch.object( + ramdisk, + "_load_manifest", + return_value=manifest, + ), mock.patch.object( + ramdisk, + "_assert_effective_masks_unchanged", + ), mock.patch.object( + ramdisk, + "_assert_ready_mounts", + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(False, "not-running", None), + ), mock.patch.object( + ramdisk, + "_managed_child_liveness", + return_value=False, + ), mock.patch.object( + ramdisk, + "_save_manifest", + side_effect=save, + ), mock.patch.object( + ramdisk, + "_merge_usage", + side_effect=merge, + ), mock.patch.object( + ramdisk, + "_persisted_base_port", + side_effect=ramdisk.RamdiskError("stop after recovery"), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "stop after recovery", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + self.assertEqual(events[0], ("save", merge_id)) + self.assertEqual(events[1], ("merge", merge_id)) + + def test_stop_binds_all_transactions_before_save_or_signal(self): + manifest = self.manifest( + state="running", + mount_paths=[ + "/mnt/colibri-test/node0", + "/mnt/colibri-test/node1", + ], + processes=[{"pid": 12371}, {"pid": 12372}], + ) + events = [] + terminate = mock.Mock() + merge = mock.Mock() + + def bind(record, plan, reserved_ids): + del plan + merge_id = ("a" if record["pid"] == 12371 else "b") * 32 + self.assertNotIn(merge_id, reserved_ids) + events.append(("bind", record["pid"], merge_id)) + record["usage_merge_id"] = merge_id + return merge_id + + def fail_save(current): + events.append( + ( + "save", + tuple( + record.get("usage_merge_id") + for record in current["processes"] + ), + ) + ) + raise OSError("transaction authority save failed") + + with self.assertRaisesRegex( + OSError, + "transaction authority save failed", + ): + lifecycle_support.stop( + load_manifest=lambda required=True: manifest, + process_matches=lambda record: (True, "running", {}), + group_alive=lambda pgid: True, + managed_child_liveness=lambda pid: False, + save_manifest=fail_save, + terminate_verified_group=terminate, + merge_usage=merge, + bind_usage_transaction=bind, + ) + + self.assertEqual( + [event[0] for event in events], + ["bind", "bind", "save"], + ) + terminate.assert_not_called() + merge.assert_not_called() + + def test_duplicate_legacy_journals_fail_before_record_mutation(self): + manifest = self.manifest( + state="running", + mount_paths=[ + "/mnt/colibri-test/node0", + "/mnt/colibri-test/node1", + ], + processes=[{"pid": 12373}, {"pid": 12374}], + ) + merge_id = "c" * 32 + for record in manifest["processes"]: + os.makedirs(record["state_dir"], exist_ok=True) + state_support._atomic_json( + os.path.join(record["state_dir"], ".coli_usage.delta.json"), + { + "version": 1, + "id": merge_id, + "delta": {"0:1": 1}, + }, + ) + save = mock.Mock() + terminate = mock.Mock() + merge = mock.Mock() + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "duplicate usage transaction", + ): + lifecycle_support.stop( + load_manifest=lambda required=True: manifest, + process_matches=lambda record: (False, "not-running", None), + group_alive=lambda pgid: False, + managed_child_liveness=lambda pid: False, + save_manifest=save, + terminate_verified_group=terminate, + merge_usage=merge, + bind_usage_transaction=ramdisk._bind_usage_transaction, + ) + + self.assertTrue( + all( + record.get("usage_merge_id") is None + for record in manifest["processes"] + ) + ) + save.assert_not_called() + terminate.assert_not_called() + merge.assert_not_called() + + def test_start_duplicate_live_orphan_journals_fail_before_replay(self): + manifest = self.manifest( + state="stopped", + mount_paths=[ + "/mnt/colibri-test/node0", + "/mnt/colibri-test/node1", + ], + ) + model_dir = manifest["plan"]["model"]["path"] + canonical = os.path.join(model_dir, ".coli_usage") + os.makedirs(model_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + canonical_before = Path(canonical).read_bytes() + merge_id = "7" * 32 + journals = [] + journal_bytes = [] + for node, delta in enumerate((2, 5)): + state_dir = self.recovery_state_dir(node=node) + os.makedirs(state_dir, mode=0o700) + journal = os.path.join(state_dir, ".coli_usage.delta.json") + state_support._atomic_json( + journal, + {"version": 1, "id": merge_id, "delta": {"0:1": delta}}, + ) + journals.append(journal) + journal_bytes.append(Path(journal).read_bytes()) + + with mock.patch.object( + ramdisk, + "_load_manifest", + return_value=manifest, + ), mock.patch.object( + ramdisk, + "_assert_effective_masks_unchanged", + ), mock.patch.object( + ramdisk, + "_assert_ready_mounts", + ), mock.patch.object( + ramdisk, + "_save_manifest", + ) as save, mock.patch.object( + ramdisk, + "_persisted_base_port", + side_effect=ramdisk.RamdiskError("continued past journal preflight"), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "duplicate usage transaction", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + save.assert_not_called() + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + self.assertEqual( + [Path(path).read_bytes() for path in journals], + journal_bytes, + ) + + def test_start_orphan_journal_cannot_reuse_manifest_authority(self): + merge_id = "6" * 32 + manifest = self.manifest( + state="stopped", + mount_paths=[ + "/mnt/colibri-test/node0", + "/mnt/colibri-test/node1", + ], + processes=[ + { + "pid": 12375, + "stopped_at": "2026-08-01T00:00:00Z", + "usage_merge_id": merge_id, + } + ], + ) + model_dir = manifest["plan"]["model"]["path"] + canonical = os.path.join(model_dir, ".coli_usage") + os.makedirs(model_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + canonical_before = Path(canonical).read_bytes() + os.makedirs(self.recovery_state_dir(node=0), mode=0o700) + orphan_state = self.recovery_state_dir(node=1) + os.makedirs(orphan_state, mode=0o700) + journal = os.path.join(orphan_state, ".coli_usage.delta.json") + state_support._atomic_json( + journal, + {"version": 1, "id": merge_id, "delta": {"0:1": 5}}, + ) + journal_before = Path(journal).read_bytes() + + with mock.patch.object( + ramdisk, + "_load_manifest", + return_value=manifest, + ), mock.patch.object( + ramdisk, + "_assert_effective_masks_unchanged", + ), mock.patch.object( + ramdisk, + "_assert_ready_mounts", + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(False, "not-running", None), + ), mock.patch.object( + ramdisk, + "_managed_child_liveness", + return_value=False, + ), mock.patch.object( + ramdisk, + "_save_manifest", + ) as save: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "duplicate usage transaction", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + save.assert_not_called() + self.assertEqual(Path(canonical).read_bytes(), canonical_before) + self.assertEqual(Path(journal).read_bytes(), journal_before) + + def test_start_mint_reserves_preflighted_orphan_journal_id(self): + manifest = self.manifest(state="stopped") + model_dir = manifest["plan"]["model"]["path"] + os.makedirs(model_dir) + ramdisk._usage_write( + os.path.join(model_dir, ".coli_usage"), + {"0:1": 10}, + ) + state_dir = self.recovery_state_dir() + os.makedirs(state_dir, mode=0o700) + orphan_id = "8" * 32 + fresh_id = "9" * 32 + state_support._atomic_json( + os.path.join(state_dir, ".coli_usage.delta.json"), + {"version": 1, "id": orphan_id, "delta": {"0:1": 2}}, + ) + mint_attempts = [] + + def token_hex(size): + if size == 24: + return "a" * 48 + self.assertEqual(size, 16) + mint_attempts.append(len(mint_attempts)) + return orphan_id if len(mint_attempts) == 1 else fresh_id + + class FakeSocket: + def bind(self, address): + del address + + def close(self): + pass + + with mock.patch.object( + ramdisk, + "_load_manifest", + return_value=manifest, + ), mock.patch.object( + ramdisk, + "_assert_effective_masks_unchanged", + ), mock.patch.object( + ramdisk, + "_assert_ready_mounts", + ), mock.patch.object( + ramdisk, + "_save_manifest", + ), mock.patch.object( + ramdisk, + "_recover_delta", + ), mock.patch.object( + ramdisk, + "_usage_read", + return_value={"0:1": 10}, + ), mock.patch.object( + ramdisk, + "_usage_write", + ), mock.patch.object( + ramdisk, + "_admit_concurrent_runtimes", + ), mock.patch.object( + ramdisk, + "_current_process_identity", + return_value={ + "pid": 700, + "uid": host_uid(), + "starttime": 90, + "cmdline": ["coli", "ramdisk", "start"], + }, + ), mock.patch.object( + ramdisk, + "_process_start_boundary", + side_effect=ramdisk.RamdiskError("stop after transaction mint"), + ), mock.patch.object( + ramdisk.socket, + "socket", + side_effect=lambda *args, **kwargs: FakeSocket(), + ), mock.patch.object( + lifecycle_support.secrets, + "token_hex", + side_effect=token_hex, + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "stop after transaction mint", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + + self.assertEqual(len(mint_attempts), 2) + + def test_expected_orphan_journal_disappearance_fails_closed(self): + model_dir = os.path.join(self.root, "model") + canonical = os.path.join(model_dir, ".coli_usage") + state_dir = os.path.join(self.root, "orphan-state") + os.makedirs(model_dir) + os.makedirs(state_dir) + ramdisk._usage_write(canonical, {"0:1": 10}) + + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "expected usage delta journal is absent", + ): + ramdisk._recover_delta( + state_dir, + canonical, + expected_merge_id="a" * 32, + ) + + self.assertEqual(ramdisk._usage_read(canonical), {"0:1": 10}) + + def test_usage_transaction_mint_skips_reserved_collision(self): + reserved = {"d" * 32} + with mock.patch.object( + lifecycle_support.secrets, + "token_hex", + side_effect=("d" * 32, "e" * 32), + ): + merge_id = lifecycle_support._mint_usage_transaction_id(reserved) + + self.assertEqual(merge_id, "e" * 32) + self.assertEqual(reserved, {"d" * 32, "e" * 32}) + + @requires_linux_operational + def test_stop_does_not_merge_when_retained_child_is_live(self): + manifest = self.manifest(state="running", processes=[{"pid": 12348}]) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + process = mock.Mock(pid=12348, returncode=None) + process.poll.return_value = None + ramdisk._track_managed_child(process) + self.addCleanup(ramdisk._forget_managed_child, process.pid) + + with mock.patch.object( + ramdisk, + "_process_matches", + return_value=(False, "not-running", None), + ), mock.patch.object( + ramdisk, "_terminate_verified_group" + ) as terminate, mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "retained-managed-child-live", + ): + ramdisk.stop() + + terminate.assert_not_called() + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertNotIn("stopped_at", persisted["processes"][0]) + self.assertNotIn("usage_merged_at", persisted["processes"][0]) + + @requires_linux_operational + def test_stop_preserves_termination_failure_until_group_absence_is_proven(self): + manifest = self.manifest(state="running", processes=[{"pid": 12349}]) + os.makedirs(manifest["processes"][0]["state_dir"], exist_ok=True) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + running = (True, "running", {"pid": 12349, "pgid": 12349}) + inconclusive = ( + False, + "unverified-process-group", + {"pgid": 12349}, + ) + + with mock.patch.object( + ramdisk, + "_process_matches", + side_effect=(running, inconclusive), + ), mock.patch.object( + ramdisk, + "_terminate_verified_group", + return_value="identity changed after SIGTERM", + ), mock.patch.object( + ramdisk, "_merge_usage" + ) as merge: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "identity changed after SIGTERM", + ): + ramdisk.stop() + + merge.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertIn( + "identity changed after SIGTERM", + persisted["processes"][0]["stop_error"], + ) + self.assertNotIn("stopped_at", persisted["processes"][0]) + + @requires_linux_operational + def test_stop_preserves_recoverable_error_for_incomplete_mount_layout(self): + manifest = self.manifest( + state="error", + mount_paths=["/mnt/colibri-test/node0", "/mnt/colibri-test/node1"], + ) + manifest["mounts"].pop() + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + stopped = ramdisk.stop() + + self.assertEqual(stopped["state"], "error") + self.assertEqual(ramdisk._load_manifest(required=True)["state"], "error") + + def test_managed_readiness_requires_verified_health_response(self): + class Response: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return b'{"status":"ok"}' + + record = {"pid": 123, "port": 8123, "log": "/tmp/engine.log"} + with mock.patch.object( + ramdisk, "_process_matches", return_value=(True, "running", {}) + ): + ramdisk._wait_managed_ready( + record, + timeout=1, + api_key="secret", + urlopen=mock.Mock(return_value=Response()), + ) + self.assertIn("ready_at", record) + + @requires_linux_operational + def test_destroy_refuses_replaced_mount_identity(self): + mount_path = "/mnt/colibri-test" + manifest = self.manifest(mount_paths=[mount_path]) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + replacement = { + "mount_id": 5, + "device": "0:10", + "filesystem": "tmpfs", + "source": "tmpfs", + } + args = argparse.Namespace(yes=True) + with mock.patch.object(ramdisk, "_mount_at", return_value=replacement), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "foreign or replaced"): + ramdisk.destroy(args) + unmount.assert_not_called() + persisted = ramdisk._read_json(ramdisk._manifest_path()) + self.assertEqual(persisted["state"], "error") + self.assertIn(mount_path, persisted["recovery"]["retained_mounts"]) + + def test_prepare_save_ambiguity_preserves_strongest_exact_mount(self): + mount_path = os.path.join(self.root, "ramdisk-mount") + exact_identity = { + "mount_id": 41, + "device": "0:41", + "filesystem": "tmpfs", + "source": "tmpfs", + } + plan = { + "blockers": [], + "mount_root": mount_path, + "mounts": [ + { + "path": mount_path, + "node": None, + "size_bytes": 4096, + } + ], + "topology": "interleaved", + "model": { + "path": os.path.join(self.root, "model"), + "fingerprint": self.FINGERPRINT, + }, + "hardware": { + "swap": {"used_bytes": 0}, + }, + "mount_options": {}, + } + durable = {"manifest": None} + save_calls = {"count": 0} + + def ambiguous_save(current): + save_calls["count"] += 1 + durable["manifest"] = copy.deepcopy(current) + if save_calls["count"] in (3, 4): + raise OSError("post-replace directory fsync failed") + + observed = iter((None, copy.deepcopy(exact_identity))) + unmount = mock.Mock() + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "post-replace directory fsync failed", + ): + lifecycle_support.prepare( + argparse.Namespace(base_port=8000, yes=True), + display_plan=False, + load_manifest=lambda required=False: None, + build_plan=lambda args: copy.deepcopy(plan), + managed_ports_for_plan=lambda current, base: [base], + plan_confirmation_token=lambda current: "token", + render_plan=mock.Mock(), + confirm=mock.Mock(), + save_manifest=ambiguous_save, + mount_at=lambda path: next(observed), + mount_tmpfs=mock.Mock(), + umount_path=unmount, + validate_mount=mock.Mock(return_value=exact_identity), + populate_mount=mock.Mock(), + validate_namespace=mock.Mock(), + source_still_matches=mock.Mock(), + ensure_busy_mount_scan_available=mock.Mock(), + durable_unlink=mock.Mock(), + manifest_path=lambda: os.path.join(self.root, "manifest.json"), + mount_table=mock.Mock(return_value=[]), + path_is_below=lambda path, parent: False, + busy_mount_references=mock.Mock(return_value=[]), + ) + + persisted = durable["manifest"]["mounts"][0] + self.assertEqual(persisted["ownership"], "identified") + self.assertEqual(persisted["identity"], exact_identity) + self.assertEqual(persisted["cleanup"]["state"], "retained") + unmount.assert_not_called() + + def test_prepare_recovery_candidate_unions_durable_pending_records(self): + first_path = os.path.join(self.root, "mount-a") + second_path = os.path.join(self.root, "mount-b") + durable = { + "state": "preparing", + "mounts": [ + { + "path": first_path, + "operation_id": "deploy:mount:0", + "ownership": "pending", + }, + { + "path": second_path, + "operation_id": "deploy:mount:1", + "ownership": "pending", + }, + ], + } + current = { + "state": "preparing", + "mounts": [ + { + "path": first_path, + "operation_id": "deploy:mount:0", + "ownership": "identified", + "identity": {"mount_id": 42, "device": "0:42"}, + } + ], + } + + recovery = lifecycle_support._strongest_prepare_recovery_manifest( + current, + durable, + ) + + by_path = {record["path"]: record for record in recovery["mounts"]} + self.assertEqual(by_path[first_path]["ownership"], "identified") + self.assertEqual(by_path[first_path]["identity"]["mount_id"], 42) + self.assertEqual(by_path[second_path]["ownership"], "pending") + + @requires_linux_operational + def test_destroy_retains_manifest_for_unrecorded_surviving_mount(self): + mount_path = "/mnt/colibri-test" + manifest = self.manifest(state="error", mount_paths=[mount_path]) + manifest["mounts"] = [] + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + surviving = { + "mount_id": 17, + "device": "0:77", + "filesystem": "tmpfs", + "source": "tmpfs", + } + with mock.patch.object(ramdisk, "_mount_table", return_value=[]), mock.patch.object( + ramdisk, "_mount_at", return_value=surviving + ), mock.patch.object(ramdisk, "_umount_path") as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "unverified surviving mount"): + ramdisk.destroy(argparse.Namespace(yes=True)) + unmount.assert_not_called() + self.assertTrue(os.path.exists(ramdisk._manifest_path())) + + def test_destroy_retains_absent_pending_mount_that_helper_may_publish_later(self): + mount_path = "/mnt/colibri-test" + manifest = self.manifest(state="error", mount_paths=[mount_path]) + manifest["mounts"][0].pop("identity") + manifest["mounts"][0]["ownership"] = "pending" + manifest["error"] = "mount helper outcome is unknown" + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + with mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ) as mount_table, mock.patch.object( + # Even an absent-now observation cannot clear an in-flight helper. + ramdisk, "_mount_at", return_value=None + ) as mount_at, mock.patch.object( + ramdisk, "_umount_path" + ) as unmount, mock.patch.object( + ramdisk, "_durable_unlink" + ) as unlink: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "mount helper outcome is unknown.*pending", + ): + ramdisk.destroy(argparse.Namespace(yes=True)) + + mount_table.assert_not_called() + mount_at.assert_not_called() + unmount.assert_not_called() + unlink.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertEqual(persisted["mounts"][0]["ownership"], "pending") + self.assertEqual( + persisted["recovery"]["retained_mounts"], + [mount_path], + ) + + @requires_linux_operational + def test_destroy_preflights_every_busy_mount_before_unmounting(self): + paths = ["/mnt/colibri-test/node0", "/mnt/colibri-test/node1"] + manifest = self.manifest(mount_paths=paths) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + + def mounted(path): + record = next(item for item in manifest["mounts"] if item["path"] == path) + return dict(record["identity"], filesystem="tmpfs", source="tmpfs") + + with mock.patch.object(ramdisk, "_mount_at", side_effect=mounted), mock.patch.object( + ramdisk, "_validate_mount", side_effect=lambda record, plan: mounted(record["path"]) + ), mock.patch.object(ramdisk, "_validate_namespace"), mock.patch.object( + ramdisk, "_busy_mount_references", side_effect=[[], [999]] + ), mock.patch.object(ramdisk, "_umount_path") as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "busy"): + ramdisk.destroy(argparse.Namespace(yes=True)) + unmount.assert_not_called() + + @requires_linux_operational + def test_destroy_rejects_nested_child_mounts_before_any_unmount(self): + paths = ["/mnt/colibri-test/node0", "/mnt/colibri-test/node1"] + manifest = self.manifest(mount_paths=paths) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + child = { + "mount_id": 99, + "path": paths[1] + "/foreign-child", + "filesystem": "ext4", + "source": "/dev/loop0", + } + with mock.patch.object(ramdisk, "_mount_table", return_value=[child]), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex(ramdisk.RamdiskError, "nested child mounts"): + ramdisk.destroy(argparse.Namespace(yes=True)) + unmount.assert_not_called() + persisted = ramdisk._read_json(ramdisk._manifest_path()) + self.assertEqual(persisted["state"], "error") + self.assertEqual( + persisted["recovery"]["retained_mounts"], + paths, + ) + + @requires_linux_operational + def test_destroy_persists_recovery_state_when_kernel_unmount_fails(self): + mount_path = "/mnt/colibri-test" + manifest = self.manifest(state="stopped", mount_paths=[mount_path]) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + actual = dict( + manifest["mounts"][0]["identity"], + filesystem="tmpfs", + source="tmpfs", + ) + + with mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, "_mount_at", return_value=actual + ), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ), mock.patch.object( + ramdisk, "_validate_namespace" + ), mock.patch.object( + ramdisk, "_busy_mount_references", return_value=[] + ), mock.patch.object( + ramdisk, + "_umount_path", + side_effect=ramdisk.RamdiskError("kernel refused unmount"), + ): + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "kernel refused unmount", + ): + ramdisk.destroy(argparse.Namespace(yes=True)) + + persisted = ramdisk._read_json(ramdisk._manifest_path()) + self.assertEqual(persisted["state"], "error") + self.assertIn( + "kernel refused unmount", + persisted["destroy_error"], + ) + self.assertEqual( + persisted["recovery"]["retained_mounts"], + [mount_path], + ) + + def test_destroy_revalidates_mount_identity_immediately_before_unmount(self): + mount_path = "/mnt/colibri-test" + manifest = self.manifest(state="stopped", mount_paths=[mount_path]) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + actual = dict( + manifest["mounts"][0]["identity"], + filesystem="tmpfs", + source="tmpfs", + ) + replacement = dict(actual, mount_id=81, device="0:81") + + with mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, "_mount_at", side_effect=[actual, replacement] + ), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ), mock.patch.object( + ramdisk, "_validate_namespace" + ), mock.patch.object( + ramdisk, "_busy_mount_references", return_value=[] + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "foreign or replaced mount", + ): + ramdisk.destroy(argparse.Namespace(yes=True)) + + unmount.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertEqual( + persisted["recovery"]["retained_mounts"], + [mount_path], + ) + + def test_destroy_requires_post_unmount_absence(self): + mount_path = "/mnt/colibri-test" + for replacement_id in (None, 91): + with self.subTest(replacement_id=replacement_id): + manifest = self.manifest( + state="stopped", + mount_paths=[mount_path], + ) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + actual = dict( + manifest["mounts"][0]["identity"], + filesystem="tmpfs", + source="tmpfs", + ) + after = ( + actual + if replacement_id is None + else dict( + actual, + mount_id=replacement_id, + device="0:%d" % replacement_id, + ) + ) + + with mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, + "_mount_at", + side_effect=[actual, actual, actual, after], + ), mock.patch.object( + ramdisk, "_validate_mount", return_value=actual + ), mock.patch.object( + ramdisk, "_validate_namespace" + ), mock.patch.object( + ramdisk, "_busy_mount_references", return_value=[] + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount, mock.patch.object( + ramdisk, "_durable_unlink" + ) as unlink: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "remains or was replaced", + ): + ramdisk.destroy(argparse.Namespace(yes=True)) + + unmount.assert_called_once() + unlink.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertEqual( + persisted["recovery"]["retained_mounts"], + [mount_path], + ) + + def test_destroy_rechecks_identity_after_busy_scan(self): + mount_path = "/mnt/colibri-test" + manifest = self.manifest(state="stopped", mount_paths=[mount_path]) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + actual = dict( + manifest["mounts"][0]["identity"], + filesystem="tmpfs", + source="tmpfs", + ) + replacement = dict(actual, mount_id=92, device="0:92") + current = {"identity": actual} + busy_calls = {"count": 0} + + def busy(_path, hardware=None): + busy_calls["count"] += 1 + if busy_calls["count"] == 2: + current["identity"] = replacement + return [] + + with mock.patch.object( + ramdisk, "_mount_table", return_value=[] + ), mock.patch.object( + ramdisk, + "_mount_at", + side_effect=lambda ignored: current["identity"], + ), mock.patch.object( + ramdisk, + "_validate_mount", + side_effect=lambda ignored, plan: current["identity"], + ), mock.patch.object( + ramdisk, "_validate_namespace" + ), mock.patch.object( + ramdisk, "_busy_mount_references", side_effect=busy + ), mock.patch.object( + ramdisk, "_umount_path" + ) as unmount: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "after busy scan", + ): + ramdisk.destroy(argparse.Namespace(yes=True)) + + unmount.assert_not_called() + persisted = ramdisk._load_manifest(required=True) + self.assertEqual(persisted["state"], "error") + self.assertEqual( + persisted["recovery"]["retained_mounts"], + [mount_path], + ) + + @requires_linux_operational + def test_busy_mount_scan_includes_the_manager_process(self): + held = os.path.join(self.root, "held-mount") + child = os.path.join(held, "inside") + os.makedirs(child) + previous = os.getcwd() + try: + os.chdir(child) + # Exercise the root-only procfs implementation without enumerating + # unrelated host processes. The unprivileged fuser command and + # parser contracts are covered independently in platform tests. + with mock.patch.object( + linux_ops.os, + "listdir", + return_value=[str(os.getpid())], + ): + self.assertIn( + os.getpid(), + linux_ops._busy_mount_references_proc(held), + ) + finally: + os.chdir(previous) + + @requires_linux_operational + def test_dashboard_rss_sums_verified_wrapper_and_engine_group(self): + record = { + "pid": 101, + "pgid": 101, + "uid": host_uid(), + "nonce": "a" * 48, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + } + members = [ + { + "pid": 101, + "uid": host_uid(), + "inert": False, + "nonce": "a" * 48, + "pgid": 101, + "sid": 101, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + }, + { + "pid": 102, + "uid": host_uid(), + "inert": False, + "nonce": "a" * 48, + "pgid": 101, + "sid": 101, + "state_dir": "/state/node-0", + "weights_dir": "/mnt/weights", + }, + { + "pid": 103, + "uid": host_uid(), + "inert": True, + "starttime": 203, + "nonce": None, + "pgid": 101, + "sid": 101, + "state_dir": None, + "weights_dir": None, + }, + ] + + def proc_text(path, default=""): + if path == "/proc/101/status": + return "VmRSS:\t100 kB\n" + if path == "/proc/102/status": + return "VmRSS:\t900 kB\n" + if path == "/proc/103/status": + self.fail("inert zombie must be excluded from RSS metrics") + return default + + with mock.patch.object( + ramdisk, "_process_matches", return_value=(True, "running", {}) + ), mock.patch.object( + ramdisk, "_process_group_members", return_value=(members, []) + ), mock.patch.object(ramdisk, "_read_text", side_effect=proc_text): + metrics = ramdisk._managed_process_metrics(record) + self.assertEqual(metrics["rss_bytes"], 1000 * 1024) + self.assertEqual(metrics["rss_processes"], 2) + + def test_status_absent_is_versioned(self): + report = ramdisk.status() + self.assertEqual(report["schema"], ramdisk.STATUS_SCHEMA) + self.assertEqual(report["state"], "absent") + + def test_status_distinguishes_absent_mount_from_unknown_observation(self): + manifest = self.manifest(state="ready") + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object(ramdisk, "_mount_at", return_value=None): + absent = ramdisk.status(deep=False)["mounts"][0] + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, + "_mount_at", + side_effect=OSError("mount table unreadable"), + ): + unknown = ramdisk.status(deep=False)["mounts"][0] + + self.assertIs(absent["mounted"], False) + self.assertIsNone(absent["option_error"]) + self.assertIsNone(unknown["mounted"]) + self.assertIn("mount table unreadable", unknown["option_error"]) + self.assertFalse(unknown["verified"]) + + def test_status_exposes_sanitized_actionable_recovery(self): + manifest = self.manifest(state="error") + secret_nonce = "9" * 48 + secret_merge_id = "8" * 32 + manifest["launch_error"] = "engine readiness failed" + manifest["cleanup_errors"] = ["group absence unproven"] + manifest["recovery"] = { + "operation": "start", + "state": "attention-required", + "retained_mounts": [ + "/mnt/colibri-test", + {"private_nonce": secret_nonce}, + ], + "retained_processes": [ + { + "pid": 14001, + "pgid": 14001, + "node": None, + "state_dir": os.path.join(self.root, "retained-state"), + "usage_baseline": {"0:1": 19}, + "usage_merge_id": "7" * 32, + "error": "process group still live", + } + ], + } + manifest["pending_launches"] = [ + { + "operation_id": "start:" + secret_merge_id, + "nonce": secret_nonce, + "uid": host_uid(), + "port": 8000, + "node": None, + "state_dir": os.path.join(self.root, "pending-state"), + "weights_dir": "/mnt/colibri-test", + "launch_not_before": 100, + "launcher_pid": 700, + "launcher_starttime": 90, + "launcher_cmdline": ["coli", "ramdisk", "start"], + "expected_command": ["coli", "serve"], + "usage_baseline": {"0:1": 23}, + "usage_merge_id": secret_merge_id, + } + ] + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ): + report = ramdisk.status(deep=False) + + self.assertEqual(report["recovery"]["operation"], "start") + self.assertEqual( + report["recovery"]["retained_processes"][0]["pid"], + 14001, + ) + self.assertEqual( + report["recovery"]["pending_launches"][0]["port"], + 8000, + ) + self.assertIn( + "engine readiness failed", + report["recovery"]["errors"]["launch_error"], + ) + serialized = json.dumps(report, sort_keys=True) + self.assertNotIn(secret_nonce, serialized) + self.assertNotIn(secret_merge_id, serialized) + self.assertNotIn("usage_baseline", serialized) + self.assertNotIn("usage_merge_id", serialized) + self.assertNotIn("launch_not_before", serialized) + self.assertNotIn("launcher_pid", serialized) + self.assertNotIn("launcher_starttime", serialized) + self.assertNotIn("launcher_cmdline", serialized) + self.assertNotIn("expected_command", serialized) + self.assertNotIn("weights_dir", serialized) + + def test_deep_status_preserves_recovery_when_source_scan_raises_oserror(self): + manifest = self.manifest(state="error") + manifest["recovery"] = { + "operation": "destroy", + "state": "attention-required", + "retained_mounts": [manifest["mounts"][0]["path"]], + "released_mounts": [], + } + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, + "_source_still_matches", + side_effect=OSError("source shard became unreadable"), + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ): + report = ramdisk.status(deep=True) + + self.assertFalse(report["source_fingerprint_verified"]) + self.assertIn( + "source shard became unreadable", + report["source_fingerprint_error"], + ) + self.assertEqual( + report["recovery"]["retained_mounts"], + [manifest["mounts"][0]["path"]], + ) + self.assertIn("`coli ramdisk destroy`", report["recovery"]["action"]) + + def test_status_propagates_control_flow_from_probe_seams(self): + mount_interrupt = KeyboardInterrupt("mount probe interrupted") + manifest = self.manifest(state="ready") + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_mount_at", side_effect=mount_interrupt + ): + with self.assertRaises(KeyboardInterrupt): + ramdisk.status(deep=False) + + process_manifest = self.manifest( + state="running", + processes=[{"pid": 14003}], + ) + identity_interrupt = ramdisk._TuiTerminationSignal(signal.SIGTERM) + with mock.patch.object( + ramdisk, "_load_manifest", return_value=process_manifest + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ), mock.patch.object( + ramdisk, "_process_matches", side_effect=identity_interrupt + ): + with self.assertRaises(ramdisk._TuiTerminationSignal): + ramdisk.status(deep=False) + + liveness_interrupt = ramdisk._TuiTerminationSignal(signal.SIGINT) + with mock.patch.object( + ramdisk, "_load_manifest", return_value=process_manifest + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ), mock.patch.object( + ramdisk, + "_process_matches", + return_value=(False, "not-running", None), + ), mock.patch.object( + ramdisk, + "_managed_child_liveness", + side_effect=liveness_interrupt, + ): + with self.assertRaises(ramdisk._TuiTerminationSignal): + ramdisk.status(deep=False) + + def test_status_gives_conservative_mount_only_recovery_action(self): + manifest = self.manifest(state="error") + manifest["recovery"] = { + "operation": "destroy", + "state": "attention-required", + "retained_mounts": ["/mnt/colibri-test"], + "released_mounts": [], + } + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ): + report = ramdisk.status(deep=False) + + action = report["recovery"]["action"] + self.assertIn("mount identity", action) + self.assertIn("nested mounts", action) + self.assertIn("busy references", action) + self.assertIn("`coli ramdisk destroy`", action) + self.assertIn("only after confirming it is safe", action) + self.assertNotIn("--yes", action) + + def test_status_synthesizes_recovery_for_hard_crash_pending_mount(self): + manifest = self.manifest(state="preparing") + manifest["mounts"][0].pop("identity") + manifest["mounts"][0]["ownership"] = "pending" + mount_path = manifest["mounts"][0]["path"] + + with mock.patch.object( + ramdisk, "_load_manifest", return_value=manifest + ), mock.patch.object( + ramdisk, "_mount_at", return_value=None + ): + report = ramdisk.status(deep=False) + + recovery = report["recovery"] + self.assertEqual(recovery["operation"], "prepare") + self.assertEqual(recovery["state"], "attention-required") + self.assertEqual(recovery["retained_mounts"], [mount_path]) + self.assertIn("pending ownership", recovery["action"]) + self.assertIn("`coli ramdisk destroy`", recovery["action"]) + + def test_stopped_process_group_is_revalidated_by_start_stop_and_status(self): + manifest = self.manifest( + state="stopped", + processes=[ + { + "pid": 14002, + "stopped_at": "2026-08-01T00:00:00Z", + "usage_merged_at": "2026-08-01T00:00:00Z", + "usage_merge_id": "d" * 32, + } + ], + ) + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + running = (True, "running-group", {"pgid": 14002}) + + with mock.patch.object( + ramdisk, "_process_matches", return_value=running + ), mock.patch.object( + ramdisk, "_managed_child_liveness", return_value=None + ), mock.patch.object( + ramdisk, "_assert_effective_masks_unchanged" + ), mock.patch.object( + ramdisk, "_assert_ready_mounts" + ), mock.patch.object( + ramdisk.subprocess, "Popen" + ) as popen: + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "stopped-record-process-group-live", + ): + ramdisk.start.__wrapped__( + argparse.Namespace(base_port=None), + cli_path=sys.executable, + ) + with self.assertRaisesRegex( + ramdisk.RamdiskError, + "stopped-record-process-group-live", + ): + ramdisk.stop() + report = ramdisk.status(deep=False) + + popen.assert_not_called() + self.assertTrue(report["processes"][0]["running"]) + self.assertTrue(report["processes"][0]["attention_required"]) + self.assertEqual( + report["processes"][0]["reason"], + "stopped-record-process-group-live", + ) + + @requires_linux_operational + def test_manifest_rejects_volatile_durable_state(self): + manifest = self.manifest() + ramdisk._atomic_json(ramdisk._manifest_path(), manifest) + with mock.patch.object(ramdisk, "_filesystem_for_path", return_value="tmpfs"): + with self.assertRaisesRegex(ramdisk.RamdiskError, "volatile"): + ramdisk._load_manifest(required=True) diff --git a/c/tools/clean.py b/c/tools/clean.py index 463618fb6..438bf6c03 100644 --- a/c/tools/clean.py +++ b/c/tools/clean.py @@ -10,20 +10,54 @@ # Files (relative to c/) to remove if present. FILES = [ + ".build-config", "olmoe", "olmoe.exe", + "inkling", "inkling.exe", + "kimi_k3", "kimi_k3.exe", + "colibri", "colibri.exe", "glm", "glm.exe", "iobench", "iobench.exe", - "backend_cuda.o", "backend_loader.o", + "backend_cuda.o", "backend_cuda_ink.o", "backend_loader.o", "backend_vulkan.o", "backend_cuda_test", "backend_cuda_test.exe", + "ragged_attention_test", "ragged_attention_test.exe", "backend_cuda_bench", "backend_cuda_bench.exe", - "backend_metal.o", "backend_metal_test", + "backend_metal.o", "backend_metal_test", "backend_metal_test.exe", + "gemm_largebatch_test", "gemm_largebatch_test.exe", "coli_cuda.dll", "coli_cuda.lib", "coli_cuda.exp", + "tools/libiq3.so", "tools/libiq3.dylib", "tools/iq3.dll", + "tools/librans_c.so", "tools/librans_c.dylib", "tools/rans_c.dll", + # qmatmul.spv is checked in; only the other Vulkan shaders are generated. + "shaders/qmatmul_gate_up.spv", "shaders/attention_absorb.spv", + "shaders/rmsnorm.spv", +] +# Test binaries are extensionless on Unix and `.exe` on Windows. Keep the +# basenames explicit so clean can never mistake a source/fixture for an output. +TEST_BASENAMES = [ + "test_serve_sentinel", "test_ue8m0", + "test_json", "test_st", "test_st_pread", "test_st_mirror", "test_tier", "test_grammar", + "test_ablate", "test_schema_gbnf", "test_decode_batch", "test_idot", + "test_i4_grouped", "test_stops", "test_topp", "test_temp_env", "test_kv_alloc", + "test_rans", "test_fp8_passthrough", "test_fp8_load", "test_qt_addrow", + "test_i4_acc512", "test_compat_direct", "test_dsa_select", + "test_int3", "test_int3_load", "test_logit_nan", "test_router_nan", "test_pipe_block", + "test_sample_nan", "test_tok_o200k", "test_route_trace", "test_corpus_draft", + "test_cap_precedence", "test_ssd_probe", "test_pilot_ring", + "test_uring", "test_rammap", "test_resource_masks", "test_e8_kernel", +] +ON_DEMAND_BASENAMES = [ + "bench_topp", "bench_dsa_select", "bench_idot", "bench_mla_simd", "fuzz_rans", +] +TEST_GLOBS = [ + "tests/%s%s" % (name, suffix) + for name in TEST_BASENAMES + ON_DEMAND_BASENAMES + for suffix in ("", ".exe") ] -# Test binaries match this pattern. Only remove executables (.exe on Windows, -# no extension on Unix) — never .c or .py source files. -TEST_GLOBS = ["tests/test_*.exe"] # Directories to remove. -DIRS = ["tests/__pycache__"] +DIRS = [ + "__pycache__", + "ramdisk_support/__pycache__", + "tests/__pycache__", +] removed = 0 for f in FILES: diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 1d1484f20..13607898f 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -2,11 +2,14 @@ Reference for the environment variables read by the colibrì engine. -**Generated from `dev @ d5327e2`** by scanning every `getenv()` site in `c/glm.c` and the other C sources (`c/olmoe.c`, `c/backend_cuda.cu`, `c/backend_metal.mm`). Defaults and behavior are taken from the source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate this after the code changes. +The base inventory was generated from `dev @ d5327e2`. The RAM-workspace +entries are verified against this source tree. Defaults and behavior are taken +from source; see [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) for the refresh +procedure. ## Which program reads these? -The C engine binary (`c/glm`, built from `c/glm.c`) reads **all** of these. You rarely export them by hand — the `coli` CLI and `openai_server.py` translate most of their flags into these variables before launching `glm` (e.g. `--temp` → `TEMP`, `--ctx` → `CTX`). See [SETTINGS.md](SETTINGS.md) for the flag → variable mapping. Export a variable directly only to reach a knob the CLI doesn't surface, or to override what the CLI would set. +The C engine binary (`c/colibri`, built from `c/colibri.c`) reads **all** of these. You rarely export them by hand — the `coli` CLI and `openai_server.py` translate most of their flags into these variables before launching Colibri (e.g. `--temp` → `TEMP`, `--ctx` → `CTX`). See [SETTINGS.md](SETTINGS.md) for the flag → variable mapping. Export a variable directly only to reach a knob the CLI doesn't surface, or to override what the CLI would set. Format: `VAR` — default — effect. @@ -25,7 +28,7 @@ Format: `VAR` — default — effect. | `TOPK` | `0` (off) | Top-k filter on the sampling distribution (`0` = no limit). | | `TOPP` | `0` (off) | Top-p filter (`0` = use `NUCLEUS`). | | `SEED` | unset → seeded from clock + PID | RNG seed for sampling. **Unset = different every run.** Set a fixed value for reproducible sampling. | -| `KVSAVE` | `1` (on) | Persist the KV cache to `/.coli_kv` so a conversation reopens warm. `KVSAVE=0` disables save+load (lossless round-trip; does not change output). | +| `KVSAVE` | `1` (on) | Persist the KV cache to `/.coli_kv` so a conversation reopens warm. `KVSAVE=0` disables save+load (lossless round-trip; does not change output). | | `KV_SLOTS` | `1` | Number of independent KV conversation slots (1–16), used in serve mode. | | `THINK` | `0` (off) | Emit a `` reasoning block. `THINK=1` turns on visible reasoning. | | `MTP` | on | Multi-Token Prediction (speculative draft head). `MTP=0` disables it. | @@ -44,14 +47,20 @@ Format: `VAR` — default — effect. | `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. | | `DIRECT` | `0` (off) | Use `O_DIRECT`/unbuffered reads for expert slabs. **Drive-dependent — measure it on your hardware.** On real NVMe with DRAM cache and headroom it is often a large win (measured +34% decode with `PIPE=1` on a Blackwell/Windows box, and 4.25→9.69 GB/s in iobench on a GB10); on QLC/DRAM-less drives or slow/virtualised disks it can be neutral to negative. Helps sustained NVMe; keeps the zero-copy GPU path. | | `COLI_NO_OMP_TUNE` | off | **Kill-switch** for the OpenMP hot-thread tuning (`OMP_WAIT_POLICY=active` spin + proc-bind). Set `=1` when the CPU is mostly waiting on the GPU (Metal) so spin doesn't steal the shared power budget. | -| `COLI_NUMA` | auto in generated plans on multi-socket Linux; otherwise off | `COLI_NUMA=1` selectively interleaves large expert and dense slabs across NUMA nodes via `mbind` (raw syscall, no libnuma). Helps multi-socket hosts (+7–40% expert matmul); silent no-op on single-node or non-Linux. Explicit `COLI_NUMA=0` overrides the generated plan. | +| `COLI_NUMA` | on for generated shared plans; off for node-local replicas | `COLI_NUMA=1` applies a NUMA policy to large expert and dense slabs via `mbind` (raw syscall, no libnuma). Multiple selected nodes interleave; an explicit one-node `COLI_NUMA_NODES` mask binds to that node. Helps multi-socket hosts (+7–40% expert matmul); an implicit single-node host remains a no-op. Explicit `COLI_NUMA=0` overrides the generated plan. | +| `COLI_NUMA_NODES` | all online nodes | Exact Linux NUMA range list for `COLI_NUMA=1`, including sparse IDs such as `0,2,8`. Managed values must be online and inside `Mems_allowed_list`; invalid or inapplicable masks fail closed. `coli ramdisk` sets this from the reviewed memory-node placement and uses a static bind when exactly one node is selected. | +| `COLI_CPU_AFFINITY` | unset | Exact Linux CPU range list restored before the OpenMP self-reexec. The engine applies and reads back the mask, failing closed if offline CPUs or a cpuset narrow it. Managed RAM-disk engines set this automatically; unmanaged launches retain the historical all-online reset. | | `MLOCK` | `-1` (auto: on for macOS) | Wire the streamed expert cache into physical RAM (`mlock`) to dodge the memory compressor. `0` off, `1` force. | | `CAP` | unset | Expert-cache cap (slots/layer) when no CLI positional was given. Precedence: explicit `--cap`/positional > `CAP` > platform default > historic default (#379). Mainly for direct `./glm` use — `coli` users should prefer `--cap`. | | `CAP_RAISE` | `1` (on); `0` on Metal + macOS + fast model volume (#379) | Let the engine raise the expert-cache cap above `topk` when RAM allows (bigger batches). `0` fixes the cap. When the platform-aware Metal cache default engages (F_NOCACHE probe measured the model volume fast), the *default* flips to `0` — auto-raise re-creates the Metal residency churn the minimal cache avoids. An explicit `CAP_RAISE` always wins. | | `COLI_SSD_FAST_GBS` | `4.0` | Threshold (GB/s, measured F_NOCACHE, cached in `/.coli_ssd` — see [The `.coli_ssd` probe cache](#the-coli_ssd-probe-cache) below) at or above which the model volume counts as "fast" for the platform-aware Metal cache defaults (#379). | | `PREFETCH` | `0` | Prefetch depth for streamed experts. | -| `COLI_MMAP` | `0` | `mmap` the weights instead of read()-ing into slabs. | -| `PIN` | unset | Path to a `.coli_usage`/stats file; pins the hottest experts into a resident "hot store" at startup. **`PIN=auto`** seeds from the model dir's live `.coli_usage` (appended after every turn, so each restart's pin placement follows the accumulated real workload) with `stats.txt` as the fallback for a virgin model dir; neither present → no pin this run. | +| `COLI_WEIGHTS_DIR` | `SNAP` | Complete safetensors namespace used for weight loading. `coli ramdisk` points this at a tmpfs namespace containing staged shards and validated symlinks to unstaged canonical shards; configuration and tokenizer identity remain rooted at `SNAP`. | +| `COLI_STATE_DIR` | `SNAP` | Durable directory for `.coli_usage` and `.coli_kv*`. Managed RAM-disk engines derive stable per-model/per-node directories from an absolute SSD-backed `XDG_STATE_HOME`; tmpfs/ramfs and paths beneath the volatile weight mount are rejected, so volatile weight mounts never hold runtime state. | +| `COLI_RAMMAP` | `0` | Linux-only direct mapping of complete experts whose six weight/scale tensors are tmpfs-backed. Mapped experts bypass slabs and LRU I/O; unstaged experts keep the existing SSD path. Incompatible with `COLI_MMAP=1`. | +| `COLI_RAM_PREFAULT` | `0` | Prefault direct RAM mappings at startup. Managed full-model mode defaults this to `1` for benchmarking; `0` leaves pages demand-faulted. | +| `COLI_MMAP` | `0` | Legacy page-cache-backed `mmap` path. Do not combine it with `COLI_RAMMAP=1`; the engine rejects that configuration. | +| `PIN` | unset | Path to a `.coli_usage`/stats file; pins the hottest experts into a resident "hot store" at startup. **`PIN=auto`** seeds from `COLI_STATE_DIR/.coli_usage` (updated after every turn) with `SNAP/stats.txt` as the fallback for a virgin state directory; neither present → no pin this run. RAM-mapped experts are already resident and are excluded from pin/LRU allocation. | | `PIN_GB` | `10.0` | Size budget (GB) for the pinned hot store when `PIN` is set. | | `AUTOPIN` | `1` (on) | Auto-pin the hot store from usage history once ≥5000 selections are recorded. | | `REPIN` | `0` (off) | Live re-pin the hot store every N emitted tokens (RFC). | @@ -72,7 +81,7 @@ Format: `VAR` — default — effect. | `ABSORB` | `-1` (auto: absorbed for S≤4) | MLA attention absorption mode. | | `IDOT` | `1` | Integer dot-product kernel. `IDOT=0` uses exact f32 kernels (for A/B numerical checks). | | `COLI_POLICY` | `quality` | Resource policy: `quality`, `balanced`, or `experimental-fast`. | -| `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), expert-I/O totals and cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. | +| `PROF` | `0` (off) | Performance profile: a startup header (machine + effective config), then per run — or per turn in serve mode, on stderr — forward-latency percentiles (p50/p90/p99/max), SSD-backed expert bytes requested, Linux block-layer `read_bytes`, cache-tier fill, phase shares of wall time, and a verdict naming the knob most likely to help on this machine. Output is additive; `PROF` unset changes nothing. | | `COLI_NO_FUSED_PAIR` | `0` (off) | `=1` disables the fused-pair matmul kernel. | | `DISK_SPLIT` | `0` (off) | `=1` splits the reported disk-load time across the draft/absorb/forward phases in stats. | | `I4S` | unset | Engage the int4 `IDOT` kernel only for batch `S>=` (testing). | @@ -233,7 +242,7 @@ These are for testing, benchmarking, or internal use — not part of the everyda ## Server / CLI (`openai_server.py`, `coli`) -These are read by the Python programs (not the `glm` engine), so they don't appear in `glm.c`. They cover the OpenAI-compatible server, tool calling, and the debug view. +These are read by the Python programs (not the C engine), so they don't appear in `colibri.c`. They cover the OpenAI-compatible server, tool calling, and the debug view. | Variable | Default | Effect | |---|---|---| @@ -246,6 +255,7 @@ These are read by the Python programs (not the `glm` engine), so they don't appe | `COLI_ALLOWED_HOSTS` | unset | Comma-separated hostnames or IP addresses accepted by the DNS-rebinding guard in addition to loopback and the bind address. Equivalent to repeating `--allowed-host`. | | `COLI_MAX_QUEUE` | `8` | Max queued requests. | | `COLI_QUEUE_TIMEOUT` | `300` | Seconds a request may wait in the queue. | +| `COLI_ENGINE_READY_TIMEOUT` | `7200` | Seconds the Python server waits for the C engine/model to become ready. Invalid values and timeouts terminate and reap the child process. | | `COLI_KV_SLOTS` | `1` | Independent KV conversation slots (→ engine `KV_SLOTS`). | | `COLI_POLICY` | `quality` | Resource policy (shared with the engine): `quality` \| `balanced` \| `experimental-fast`. | | `COLI_COLOR` | auto (TTY) | `COLI_COLOR=1` forces colored `coli` output when not a TTY. | @@ -253,11 +263,24 @@ These are read by the Python programs (not the `glm` engine), so they don't appe > **Debugging an OpenCode session:** `COLI_DEBUG=1` watches the model's output stream; `COLI_DEBUG=2` shows both sides (prompt + output) as a transcript. Add `COLI_TOOL_SALVAGE=1` on int4 to catch mangled tool calls. +### RAM-workspace control plane + +These variables configure the Linux `coli ramdisk` interface and its durable +control state: + +| Variable | Default | Effect | +|---|---|---| +| `COLI_RAMDISK_UI` | `auto` | TUI frontend: `auto` uses Textual when installed and otherwise curses; `textual` or `curses` requires that frontend. | +| `XDG_STATE_HOME` | `~/.local/state` | Durable state base. RAM-workspace state lives below `colibri/ramdisk`; the value must resolve to an absolute non-volatile path. | +| `COLI_RAMDISK_MANIFEST` | `$XDG_STATE_HOME/colibri/ramdisk/manifest.json` | Absolute durable override for the managed deployment manifest. | +| `COLI_RAMDISK_START_TIMEOUT` | `7200` | Managed-engine startup timeout in seconds; accepted range is 1–86400. | +| `COLI_BUILD_COMMIT` | auto-detected | Optional revision label stored in reproducible benchmark reports when Git metadata is unavailable or should be overridden. | + ## Set by the CLI (don't usually set by hand) `coli` / `openai_server.py` set these internally to select a run mode or pass through a flag: -- `SNAP` — model snapshot directory (required by `glm`; set from `--model`). +- `SNAP` — model snapshot directory (required by Colibri; set from `--model`). - `SERVE`, `SERVE_BATCH` — select serve / batched-serve mode. - `PROMPT` — one-shot text mode (the engine also honors `COLI_PROMPT`, preferred cross-platform; `PROMPT` is ignored on Windows if it contains cmd.exe `$`-metacharacters). - `COLI_OMP_TUNED` — internal sentinel guarding the OMP re-exec (see `COLI_NO_OMP_TUNE`); not user-facing. diff --git a/docs/SETTINGS.md b/docs/SETTINGS.md index 1bbe1392a..89b56c4f1 100644 --- a/docs/SETTINGS.md +++ b/docs/SETTINGS.md @@ -1,8 +1,10 @@ # CLI & Settings Reference -Command-line settings for the two user-facing programs: the **`coli`** CLI and the **`openai_server.py`** server. The underlying `glm` engine is driven by environment variables — see [ENVIRONMENT.md](ENVIRONMENT.md). +Command-line settings for the two user-facing programs: the **`coli`** CLI and the **`openai_server.py`** server. The underlying Colibri engine is driven by environment variables — see [ENVIRONMENT.md](ENVIRONMENT.md). -**Updated for the contribution based on `upstream/dev @ 21e7a35`** (argparse definitions in `c/coli` and `c/openai_server.py`). See [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) to regenerate. +The base CLI inventory tracks `upstream/dev @ 21e7a35`. The `ramdisk` section +is verified against `c/coli` and `c/ramdisk_support/cli.py` in this source tree. +See [MAINTAINING-DOCS.md](MAINTAINING-DOCS.md) for the refresh procedure. --- @@ -12,7 +14,7 @@ Command-line settings for the two user-facing programs: the **`coli`** CLI and t coli [flags] ``` -Flags may also be given **after** the subcommand. Most flags map onto an engine environment variable before `glm` is launched (see the mapping table at the bottom). +Flags may also be given **after** the subcommand. Most flags map onto an engine environment variable before Colibri is launched (see the mapping table at the bottom). ### Subcommands @@ -26,6 +28,8 @@ Flags may also be given **after** the subcommand. Most flags map onto an engine | `run ""` | One-shot generation for the given prompt (positional, may be multi-word). | | `chat` | Interactive REPL chat. | | `serve` | Start the OpenAI-compatible HTTP server. | +| `stop` | Stop a server on the selected port. | +| `ramdisk` | Open the Linux NUMA-aware RAM-disk TUI or run a scriptable lifecycle action. | | `bench [tasks]` | Run benchmark tasks (`--limit`, `--data`). | | `convert` | Convert an FP8 repo to a colibrì int4 snapshot. | @@ -74,6 +78,70 @@ Flags may also be given **after** the subcommand. Most flags map onto an engine | `--xbits` | `0` | Extra/override bit width. | | `--no-mtp` | off | Skip the MTP speculative-draft head. | +**`ramdisk`** (Linux only) + +Omit an action to open the guided terminal interface. New workspaces offer four +presets; each one creates a reviewable draft and performs no lifecycle action +by itself. + +| Preset | Result | +|---|---| +| **Fastest GPU staging** (default) | One shared copy and engine on usable GPU-local NUMA nodes. It tries full staging, then a compatible profile-guided partial plan. Unsafe or unproven CUDA/NUMA discovery falls back visibly to **Single RAM copy**. | +| **Single RAM copy** | One full shared copy and one engine using the normal effective NUMA placement. | +| **Minimal RAM** | The largest safely admitted profile-guided partial staging set. A missing or incompatible profile produces a blocker. | +| **Multiple NUMA replicas** | One complete copy and independent engine per selected NUMA node. This is the only preset that selects replicas. | + +Prepared workspaces skip the preset question and load their persisted +placement. Editing a preset-populated draft marks it **Custom**. See the +[operator reference](ramdisk-tui.md) for controls and lifecycle semantics, or +the [shared full-model how-to](ramdisk-tui-howto.md) for a complete walkthrough. + +The scriptable actions use the same planner and lifecycle: + +| Action | Purpose | +|---|---| +| `plan` | Print the exact staging and reserve plan. | +| `prepare` | Mount, stage, and validate the reviewed weights. | +| `status` | Report managed mounts and processes. | +| `benchmark` | Compare eligible SSD, tmpfs/slab, and direct-map paths. | +| `start` | Start the persisted managed deployment. | +| `stop` | Stop only identity-verified managed processes. | +| `destroy` | Unmount the verified volatile workspace. | + +```sh +coli ramdisk plan --model /models/glm --memory-nodes 0,2 --cpu-list 0-31,64-95 --json +coli ramdisk prepare --model /models/glm --yes +coli ramdisk start --base-port 8000 +coli ramdisk status --json +coli ramdisk benchmark --json +coli ramdisk stop +coli ramdisk destroy --yes +``` + +| Flag | Default | Meaning | +|---|---|---| +| `--mode` | `full` | `full` copies every shard; `partial` requires a compatible usage profile and stages complete shard closures. | +| `--topology` | `interleaved` | One shared copy/engine, or `per-node`: one complete staged copy and independent engine per NUMA node (replication, not sharding). | +| `--memory-nodes` | effective CPU-bearing NUMA nodes | Linux NUMA range list such as `0-3,8`. Shared mode requests equal interleave over a multi-node mask and a strict bind for one selected node, then verifies initial page placement; per-node mode creates replicas only on these nodes. | +| `--cpu-list` | effective CPUs on selected nodes | Linux CPU range list for managed engines. Selections must contain whole effective physical-core sibling groups. Shared plans flag CPUs outside the memory-node mask as intentional remote access; replica plans reject them. | +| `--capacity-gb` | full model size | Staging budget; required and strictly enforced in partial mode. | +| `--profile` | `/.coli_usage` | Explicit compatible text/JSON expert-usage profile for partial mode. | +| `--mount-root` | `/mnt/colibri-ram` | Managed tmpfs mount root. V1 accepts only non-symlink paths below `/mnt`; existing paths must be empty and not writable by the invoking user. | +| `--allow-swappable` | off | Explicitly permit tmpfs without `noswap` on older kernels; Colibri never runs `swapoff`. | +| `--thp` | `auto` | tmpfs THP policy: `auto` prefers `within_size`, or select `within_size`/`advise` explicitly. Unsupported `within_size` mounts retry with `advise`. | +| `--prefault` | `1` for managed full mode, otherwise `0` | Prefault direct mappings at engine startup. | +| `--parallel` | `2` | Bounded shard-copy worker count. | +| `--ctx` | `0` (`4096`) | Context length reserved for each managed engine. | +| `plan/status/benchmark --json` | off | Emit a versioned machine-readable report. | +| `prepare/destroy --yes` | off | Confirm reviewed mount or cleanup work in non-interactive scripts. | +| `start --base-port` | prepared value (`8000` initially) | Interleaved port, or base plus NUMA node id for replicas; omitted restarts preserve the previous value. | + +Options may appear before or after the action. Planning and status are +unprivileged. Prepare and Destroy require reusable foreground sudo +authorization for verified `mount`/`umount` operations. Model files and durable +`.coli_usage`, `.coli_kv*`, and benchmark state remain outside the volatile +workspace. `XDG_STATE_HOME`, when set, must be an absolute durable path. + **`bench`**: `[tasks...]` (positional), `--limit 40`, `--data `. **`plan` / `doctor`**: `--json`. @@ -95,7 +163,7 @@ Run directly (or via `coli serve`). OpenAI-compatible `/v1/chat/completions`. | Flag | Default | Meaning | |---|---|---| | `--model` | `$COLI_MODEL` (required if unset) | Model snapshot directory. | -| `--engine` | `./glm` | Path to the engine binary. | +| `--engine` | `./colibri` | Path to the engine binary. | | `--host` | `127.0.0.1` | Bind address. | | `--port` | `8000` | Port. | | `--model-id` | `$COLI_MODEL_ID` or `glm-5.2-colibri` | Model id in API responses. | @@ -108,7 +176,7 @@ Run directly (or via `coli serve`). OpenAI-compatible `/v1/chat/completions`. | `--queue-timeout` | `$COLI_QUEUE_TIMEOUT` or `300` | Request queue timeout (s). | | `--kv-slots` | `$COLI_KV_SLOTS` or `1` | KV conversation slots. | -Tool calling (`tools` in the request) is supported; the opt-in `COLI_TOOL_SALVAGE=1` env var recovers malformed int4 tool calls. Server-relevant env vars: `COLI_METAL`, `PIPE`, `DIRECT`, `COLI_NO_OMP_TUNE`, `RAM_GB`, `CTX`, `KVSAVE` (all from [ENVIRONMENT.md](ENVIRONMENT.md)) apply because the server launches the same `glm` engine. +Tool calling (`tools` in the request) is supported; the opt-in `COLI_TOOL_SALVAGE=1` env var recovers malformed int4 tool calls. Server-relevant env vars: `COLI_METAL`, `PIPE`, `DIRECT`, `COLI_NO_OMP_TUNE`, `RAM_GB`, `CTX`, `KVSAVE` (all from [ENVIRONMENT.md](ENVIRONMENT.md)) apply because the server launches the same Colibri engine. --- @@ -118,4 +186,4 @@ A flag and its mapped environment variable are two routes to the same engine kno - For knobs with a flag (`--temp`, `--ctx`, `--ram`, `--topk`, `--topp`, `--repin`, `--cap`, `--ngen`, `--policy`), prefer the flag — it's the supported surface. - For knobs with **no** flag (`COLI_METAL`, `PIPE`, `DIRECT`, `COLI_NO_OMP_TUNE`, `MLOCK`, `CAP_RAISE`, `KVSAVE`, `SEED`, `NUCLEUS`, …), export the environment variable. -- The CLI copies your whole environment through to `glm`, so any variable you export is honored unless a flag explicitly overrides it. +- The CLI copies your whole environment through to Colibri, so any variable you export is honored unless a flag explicitly overrides it. diff --git a/docs/api.md b/docs/api.md index 7aac3f5c3..16b73be4e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -75,6 +75,73 @@ errors before streaming headers are sent. `GET /health` exposes active/queued/completed/rejected counters, and successful generation responses include `x-colibri-queue-wait-ms`. +## Operational telemetry + +`GET /health` is the serving liveness probe. Its public response is: + +```json +{"status": "ok"} +``` + +The server returns HTTP 503 with `status: error` when the engine is absent, +its dispatcher failed, polling the child fails, or the child has exited. A +live Python wrapper around a dead engine is therefore not reported as serving. + +When no `COLI_API_KEY` is configured, the same response also includes detailed +runtime telemetry. When a key is configured, send a valid +`Authorization: Bearer` or `x-api-key` credential to receive those fields; an +unauthenticated health probe still receives public liveness only. + +| Field | Meaning | +|---|---| +| `scheduler` | Active, queued, completed, and rejected request counters. | +| `kv_slots` | Configured engine KV-context count. | +| `tiers` | Aggregate expert counts in VRAM, RAM, and disk plus resident VRAM/RAM GB. | +| `hwinfo` | Latest engine CPU, RAM, GPU-count, and aggregate VRAM snapshot. | +| `gpus` | Latest accepted per-device engine telemetry. | +| `gpus_seq` | Monotonic change counter for accepted `GPUS` or `GPUDETAIL` records. | + +A `gpus` item produced from `GPUDETAIL` has: + +| Field | Meaning | +|---|---| +| `device` | Logical CUDA ordinal inside the engine's visible-device list. | +| `identity` | Backend UUID/PCI identity, or `null` when the engine emitted `-`. | +| `total_bytes` / `free_bytes` / `used_bytes` | Integer-byte device-memory snapshot taken by the engine. | +| `model_bytes` | Exact bytes in model tensor allocations on this device. | +| `expert_bytes` / `expert_count` | Resident hot-expert bytes and complete expert count. | +| `nonexpert_bytes` | Other model tensor bytes; `expert_bytes + nonexpert_bytes = model_bytes`. | + +The server accepts a `GPUDETAIL` snapshot atomically only when device ordinals +are unique, every byte count is nonnegative, free/model bytes do not exceed +total bytes, and the expert/non-expert sum equals model bytes. A device whose +engine-side memory query fails is omitted rather than represented by invented +capacity; clients should treat an incomplete selected-device set as stale. + +For compatibility with older engines, a legacy `GPUS` item instead contains +`device`, `identity`, `used_gb`, `total_gb`, and `expert_count`. Clients should +inspect the available keys rather than infer model bytes from legacy +card-wide usage. + +`GET /profile` returns: + +```json +{"seq": 1, "turns": []} +``` + +`turns` is an in-memory rolling window of at most 120 completed requests. +Each modern record contains `wall_s`, `prompt_tokens`, `completion_tokens`, +`expert_disk_s`, `expert_wait_s`, `expert_matmul_s`, `attention_s`, +`lm_head_s`, and `forwards`. The matching `DONE` frame enriches it with +`tokens_per_second`, `cache_hit_percent`, `rss_gb`, and `length_limited`. +Extended producers can also supply `forward_p50_ms`, `forward_p99_ms`, +`physical_ssd_bytes`, `physical_ssd_valid`, `rammap_experts`, `rammap_bytes`, +`ttft_ms`, and `prefault_seconds`. + +Profile throughput and TTFT describe the completed request/engine as a whole. +They are not per-GPU measurements, and no profile row is published for an +in-flight or aborted request without a matching completed `PROF`/`DONE` pair. + ## Anthropic-protocol endpoint (`/v1/messages`) The same server also speaks the **Anthropic Messages API**, so clients that only talk diff --git a/docs/quickstart.md b/docs/quickstart.md index d0ea21d70..a816a62e8 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -86,7 +86,7 @@ One setup step: **install Python 3** from API gateway are Python scripts (the engine itself is pure C and needs nothing). No renaming, no configuration: the launcher finds `colibri.exe` next to itself. -For better understanding, from powershell prompt, a complete invocation line +For better understanding, from powershell prompt, a complete invocation line (relying on py launcher, to be launched from the folder where colibri.exe is) is: PS1> $env:COLI_MODEL="drive:/path/to/1st_copy/model/folder/"; $env:COLI_MODEL_MIRROR="/2nd_copy/model/folder/"; & py ./coli chat diff --git a/docs/superpowers/specs/2026-08-01-pr-377-stabilization-design.md b/docs/superpowers/specs/2026-08-01-pr-377-stabilization-design.md new file mode 100644 index 000000000..306cf9061 --- /dev/null +++ b/docs/superpowers/specs/2026-08-01-pr-377-stabilization-design.md @@ -0,0 +1,140 @@ +# PR 377 stabilization and reconstruction design + +Date: 2026-08-01 + +## Decision + +PR #377 is not a merge candidate in its current 38-commit, 87-file form. The +current branch will receive only forward fixes until the complete CI matrix is +green. That green commit becomes an immutable donor for three smaller, +dependent pull requests reconstructed from reviewed hunks. + +The audited range is pinned to base +`06f31b243792e02174a0c343021e5543d5072be4` and remote head +`3f795a7aa7af31a74abc6630664f2c1de8d96cb3`. Moving the reconstruction base +requires a separate compatibility review. + +## Stabilization scope + +The donor branch receives five focused change clusters: + +1. Replace synthetic global platform mutation with injected platform services; + make optional facade, frontend, benchmark, and HTTP-monitoring dependencies + lazy. +2. Make wheel builds isolated and diagnostic, then apply the packaging fix + supported by the exposed backend error. Test both the declared minimum and a + current supported setuptools backend. +3. Separate portable tests from Linux operational tests, repair Windows and + Darwin fixtures, use binary file descriptors on Windows, and make POSIX-only + process control explicitly unsupported elsewhere. +4. Make mount and managed-process rollback fail closed and cover every recovery + transition with fault injection. +5. Fix the source-level FP8 compiler diagnostic, align the CI matrix and skip + inventory, and capture expected negative-test diagnostics. + +No new product behavior, frontend features, benchmark policies, or unrelated +refactors belong in stabilization. + +## Cleanup safety model + +Mount ownership is persisted as `pending` before the mount helper runs. A mount +becomes managed only after its identity is recorded. Unreadable, replaced, +foreign, or nested mounts are never unmounted solely by pathname; they remain in +durable error state for explicit recovery. + +Process absence must be positively established. An inconclusive identity check +cannot erase a termination failure when a retained direct-child handle still +reports the process alive. No usage merge, `stopped_at` publication, `ready` +restoration, or process-record removal occurs while absence is unproven. An +unverified process group is never signalled. + +Cleanup may intentionally retain a resource after kernel refusal or unverifiable +ownership. The contract is no unsafe signal or unmount, no false clean result, +no premature accounting merge, and durable recovery metadata for every retained +resource. + +## Supported headless contract + +The replacement core is CLI-first: + +- `plan --json` +- `stage --plan-token TOKEN --yes --json` (`prepare` may remain an alias) +- read-only `verify --json` +- `status --json` +- `destroy --deployment-token TOKEN --yes --json` + +Plan and deployment tokens bind mutations to reviewed state. Result and error +schemas are versioned. Public managed `start` and `stop` remain deferred until +managed-runner recovery tests pass. + +Neither current TUI is part of the mergeable replacement stack. If later demand +justifies a UI, one Textual companion may consume only the stable subprocess/JSON +contract; curses will not be maintained alongside it. + +## Replacement pull requests + +### PR1: engine RAMMAP, NUMA, and telemetry + +Contains the engine C changes, CUDA accounting, strict telemetry protocol +parsing, focused documentation, and their tests. It excludes planning, mounts, +lifecycle, benchmark, and UI code. + +### PR2: headless planning, staging, mounts, and recovery + +Contains planning and discovery, platform operations, durable state, safe mount +recovery, the reduced facade and CLI, packaging, and portable tests. It depends +on PR1's engine/environment contract and must build with benchmark and UI modules +physically absent. + +### PR3: managed runner, benchmark, and evidence + +Starts as a draft and contains safe process supervision, a narrow deterministic +benchmark protocol, raw evidence production, and no frontend. A benchmark-only +commit boundary is preserved so optional UI work can never block or contaminate +the benchmark deliverable. + +Shared files are assigned hunk by hunk to their earliest consumer. The donor's +cross-cutting commits are not cherry-picked wholesale. + +## Measurement design + +The causal CPU matrix holds the expert set and numeric residency budget fixed: + +- anonymous PIN storage, interleaved across nodes 0 and 1; +- anonymous PIN storage, local to node 0; +- tmpfs RAMMAP storage, interleaved across nodes 0 and 1; +- tmpfs RAMMAP storage, local to node 0. + +SSD-slab and tmpfs-slab controls separate media effects from direct mapping. +CUDA is evaluated in a separate block with fixed host and GPU budgets because +the production managed-CUDA policy changes RAMMAP and pinning simultaneously. +`PIN_GB=all` and `CUDA_EXPERT_GB=auto` are production-policy validation, not +causal evidence. + +Each measured cell uses at least seven randomized fresh processes and records +the actual applied policy, topology, binary/model fingerprints, output hashes, +physical reads, swap, file/anonymous/shmem accounting, NUMA placement, DRAM +traffic, throughput, and latency. Staging is excluded from the measured engine +interval. + +Correctness requires exact output parity, zero swap growth, zero physical SSD +reads for full RAMMAP runs, and verified requested placement. Performance claims +require predeclared practical thresholds and paired confidence intervals; +otherwise the result is reported as neutral. + +## Verification and delivery gates + +The donor is green only when Python, Linux, Windows/UCRT64, macOS, Nix Linux, +and Nix macOS pass at one commit, while real-tmpfs and zero-SSD-read checks remain +green. Wheel failures expose backend diagnostics, the platform skip inventory is +stable, and `git diff --check` passes. + +Required lifecycle tests cover mount success followed by identity failure, +single- and multi-mount rollback, forced unmount failure, manifest-write failure, +live-child termination failure with inconclusive identity, proven-not-running +control, repeated interruption, forked-child readiness failure, and +foreign/replaced/nested mounts. + +After the donor is green it is frozen. The three successor PRs are opened and +linked before #377 is closed as superseded. The existing branch is never rebased +or force-pushed. diff --git a/flake.nix b/flake.nix index fa8c4b62d..fb00a6985 100644 --- a/flake.nix +++ b/flake.nix @@ -1,9 +1,8 @@ { description = "colibrì — run GLM-5.2 (744B MoE) on a consumer machine with ~25 GB RAM"; - # Reproducibility: these inputs track a branch, so torch/numpy/etc. float across - # rebuilds. For deterministic builds run `nix flake lock` once and COMMIT the - # generated flake.lock (it pins each input to a commit SHA). (#D2) + # Reproducibility: flake.lock pins each branch input to an exact commit. + # Update it intentionally with `nix flake update` and review the lockfile diff. inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; flake-utils.url = "github:numtide/flake-utils"; @@ -28,6 +27,7 @@ numpy tokenizers datasets + textual ] ); @@ -38,14 +38,18 @@ nativeBuildInputs = with pkgs; [makeWrapper]; - buildInputs = with pkgs; [ - gcc - gmp - ]; + buildInputs = + (with pkgs; [ + gcc + gmp + ]) + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isLinux [ + pkgs.psmisc + pkgs.util-linux + ]; - # python3 is needed by checkPhase: `make test-c` shells out to - # `python3 tools/run_tests.py` (see c/Makefile, PYTHON ?= python3). - nativeCheckInputs = with pkgs; [python3]; + # `make test` runs both the C harness and Python converter tests. + nativeCheckInputs = [pythonEnv]; # Use x86-64-v3 (AVX2) for a portable binary; override with ARCH=native for local builds ARCH = @@ -66,24 +70,29 @@ # source tree `coli` runs in (see the path-resolution logic at the # top of c/coli): the engine, the coli CLI script, the support # modules it imports (openai_server.py, resource_plan.py, - # doctor.py), and tools/ all sit next to each other. + # doctor.py, ramdisk.py, ramdisk_ui.py, ramdisk_textual.py, + # ramdisk_support/), and tools/ all sit next to each other. mkdir -p $out/lib/colibri/tools $out/bin cp c/colibri $out/lib/colibri/colibri cp c/coli $out/lib/colibri/coli chmod +x $out/lib/colibri/coli - cp c/openai_server.py c/resource_plan.py c/doctor.py c/autotune.py c/version.py \ + cp c/openai_server.py c/resource_plan.py c/doctor.py c/autotune.py c/version.py c/ramdisk.py c/ramdisk_ui.py c/ramdisk_textual.py c/requirements-tui.txt \ $out/lib/colibri/ + install -d -m 755 $out/lib/colibri/ramdisk_support + install -m 644 c/ramdisk_support/*.py $out/lib/colibri/ramdisk_support/ cp -r c/tools/* $out/lib/colibri/tools/ # $out/bin holds the user-facing entry points. ln -s ../lib/colibri/colibri $out/bin/colibri + ln -s colibri $out/bin/glm # Wrap coli: point it at the bundled engine (COLI_ENGINE) so it is # found by default, and at the module dir (PYTHONPATH) so - # `import openai_server` / `resource_plan` / `doctor` resolve. + # `import openai_server` / `resource_plan` / `doctor` / `ramdisk` resolve. makeWrapper ${pythonEnv}/bin/python $out/bin/coli \ --add-flags "$out/lib/colibri/coli" \ --set-default COLI_ENGINE "$out/lib/colibri/colibri" \ + ${pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isLinux "--prefix PATH : ${pkgs.lib.makeBinPath [pkgs.psmisc pkgs.util-linux]}"} \ --set PYTHONPATH "$out/lib/colibri:${pythonEnv}/${pkgs.python3.sitePackages}" runHook postInstall ''; @@ -91,7 +100,8 @@ checkPhase = '' runHook preCheck cd c - make test-c + export PYTHONDONTWRITEBYTECODE=1 + make test cd .. runHook postCheck ''; diff --git a/pyproject.toml b/pyproject.toml index 18f8f858e..dddf94324 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68.0"] +requires = ["setuptools>=77.0.1"] build-backend = "setuptools.build_meta" [project] @@ -9,6 +9,9 @@ description = "Tiny engine, immense model — run GLM-5.2 (744B MoE) locally" readme = "README.md" license = "Apache-2.0" requires-python = ">=3.10" +dependencies = [ + "textual>=8.2.8,<9", +] authors = [ {name = "JustVugg"}, ] @@ -50,4 +53,17 @@ version = {attr = "colibri._version.__version__"} [tool.setuptools.packages.find] where = ["."] -include = ["colibri*"] +include = ["colibri*", "c", "c.tools", "c.ramdisk_support*"] +namespaces = false + +[tool.setuptools.package-data] +c = [ + "coli", + "requirements-tui.txt", +] +"c.tools" = [ + "*.json", + "*.md", + "*.sh", + "*.txt", +]