diff --git a/.gitignore b/.gitignore index d6fb8e5..e619982 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,10 @@ __pycache__/ .ruff_cache/ frontend/node_modules/ frontend/dist/ +frontend/artifacts/ +docs/coordination.defaults.toml +docs/coordination/current_gap_slice.md +docs/coordination/focus.txt +docs/coordination/inbox/ +docs/coordination/state.json +docs/coordination/verification_target.md diff --git a/AGENTS.md b/AGENTS.md index 84e5dde..af26f35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,8 @@ Primary goals: ## Conventions - Treat the runner repo as the telemetry producer. - Consume telemetry through external interfaces; do not rely on runner internal imports as the primary path. +- This repo owns the telemetry gap ledger and the operator question checklist. +- Use dashboard-facing failures to define upstream telemetry work; do not silently normalize away telemetry truth gaps. - Prefer clear separation between: - decisions in `docs/decisions/` - implementation specs in `docs/specs/` diff --git a/README.md b/README.md index edcd88b..56e1e85 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,21 @@ That means: The current architecture notes live in: - `docs/dashboard_repo_architecture.md` +- `docs/gap_ledger.md` +- `docs/current_state_audit.md` +- `docs/operator_questions.md` +- `docs/coordination_protocol.md` - `docs/decisions/` - `docs/specs/` +## Product accountability +This repo owns the operator-facing telemetry gap ledger. + +That means: +- this repo defines what the dashboard still cannot answer +- this repo records which upstream telemetry gaps are blocking usefulness +- `model-runner` remains accountable for closing upstream telemetry truth gaps + ## Setup ```bash python -m venv .venv @@ -60,3 +72,72 @@ Integrated build served by Python: cd frontend && npm run build && cd .. ./scripts/dev-backend.sh --no-replay-fixtures ``` + +## Coordination +Push the current 3-5 upstream gap slice into `model-runner`: +```bash +./scripts/push-gap-slice.sh 5 +``` + +Pull the latest upstream response back into this repo: +```bash +./scripts/pull-model-runner-response.sh +``` + +Run one upstream implementation pass automatically: +```bash +./scripts/run-model-runner-gap-pass.sh +``` + +Run the recursive loop: +```bash +./scripts/run-gap-loop.sh +``` + +Loop defaults live in: +```bash +docs/coordination.defaults.toml +``` + +Start from: +```bash +cp docs/coordination.defaults.example.toml docs/coordination.defaults.toml +``` + +Loop state lives in: +```bash +docs/coordination/state.json +``` + +Optional slice focus lives in: +```bash +docs/coordination/focus.txt +``` + +Start from: +```bash +cp docs/coordination/focus.example.txt docs/coordination/focus.txt +``` + +Add one `GAP-...` id per line to steer the next slice toward a narrow issue. + +The upstream mirror also includes a generated verification target so `model-runner` can run a real telemetry-producing command after emission changes instead of claiming fixes without evidence. + +## Dashboard Audit +Single-command audit bundle: +```bash +./scripts/audit-dashboard.sh +``` + +If the backend is not already running: +```bash +./scripts/audit-dashboard.sh --start-backend --telemetry-jsonl /path/to/model-runner-telemetry.jsonl +``` + +Artifacts are written under `frontend/artifacts/`: +- `latest.png` +- `history-.png` +- `api-status-.json` +- `api-sessions-.json` +- `api-session-detail-.json` +- `audit-meta-.txt` diff --git a/docs/coordination.defaults.example.toml b/docs/coordination.defaults.example.toml new file mode 100644 index 0000000..578c205 --- /dev/null +++ b/docs/coordination.defaults.example.toml @@ -0,0 +1,8 @@ +model_runner_root = "/path/to/model-runner" +telemetry_jsonl = "/path/to/model-runner/models/YourModel/backend/config/telemetry/session.jsonl" +api_base = "http://127.0.0.1:8001" +dashboard_url = "http://127.0.0.1:8001/" +gap_slice_count = 5 +loop_iterations = 3 +codex_exec_mode = "danger-full-access" +verification_policy = "auto" diff --git a/docs/coordination/focus.example.txt b/docs/coordination/focus.example.txt new file mode 100644 index 0000000..0442fcc --- /dev/null +++ b/docs/coordination/focus.example.txt @@ -0,0 +1,8 @@ +# Optional focus steering for the gap slice generator. +# +# Copy to `docs/coordination/focus.txt` to activate. +# Add one GAP id per line. +# +# Example: +# GAP-001 +# GAP-015 diff --git a/docs/coordination_protocol.md b/docs/coordination_protocol.md new file mode 100644 index 0000000..f7be446 --- /dev/null +++ b/docs/coordination_protocol.md @@ -0,0 +1,105 @@ +# Coordination Protocol + +This repo owns the gap ledger. + +`model-runner` owns upstream telemetry implementation. + +The file-based loop is: +1. Audit the dashboard in `lab-llm`. +2. Update `docs/gap_ledger.md`. +3. Generate a 3-5 gap slice. +4. Push the mirrored bundle into `model-runner`. +5. `model-runner` implements the slice, runs a real telemetry-producing verification path when telemetry behavior changed, and writes `docs/downstream/lab_llm/response.md`. +6. Pull the response back into `lab-llm`. +7. Re-run the dashboard audit, preferring the verified session id or evidence target from the upstream response when present. +8. Only if the audit shows a genuine local projection/wiring issue, let `lab-llm` make the minimal read-model/API/UI change needed to expose the new telemetry truth. +9. Audit again, update the ledger, and repeat. + +## Commands +Generate and push the current slice to `model-runner`: +```bash +./scripts/push-gap-slice.sh +``` + +Pull the latest `model-runner` response back here: +```bash +./scripts/pull-model-runner-response.sh +``` + +Run one upstream implementation pass automatically: +```bash +./scripts/run-model-runner-gap-pass.sh +``` + +Pull response, rerun audit, and update the ledger here: +```bash +./scripts/reconcile-gap-ledger.sh +``` + +Run a multi-iteration loop: +```bash +./scripts/run-gap-loop.sh +``` + +All defaults come from: +```bash +docs/coordination.defaults.toml +``` + +Bootstrap from: +```bash +cp docs/coordination.defaults.example.toml docs/coordination.defaults.toml +``` + +Optional manual steering lives in: +```bash +docs/coordination/focus.txt +``` + +Bootstrap from: +```bash +cp docs/coordination/focus.example.txt docs/coordination/focus.txt +``` + +If that file contains one or more gap ids, the slice generator will prefer those gaps before normal ranking. +This is the intended way to force a narrow proof pass without disabling automation. + +That file currently defines: +- `model_runner_root` +- `telemetry_jsonl` +- `api_base` +- `dashboard_url` +- `gap_slice_count` +- `loop_iterations` +- `verification_policy` + +## Practical limit +This loop can automate most coordination work, but it still depends on: +- `model-runner` being able to make real upstream fixes without interactive blockers +- a live or restartable dashboard/backend target for audit +- telemetry changes being observable in audit artifacts + +It should reduce manual passes heavily, but “wake up with no gaps” is only realistic when the remaining gaps are actually automatable and verifiable from live artifacts. + +## Unattended mode +`run-gap-loop.sh` uses `codex exec --dangerously-bypass-approvals-and-sandbox` for the outer loop so it does not stop for approvals overnight. + +That is intentional for unattended local automation. +Use it only in the repo/workspace you are comfortable granting that level of autonomy. + +## Canonical locations +Source of truth in this repo: +- `docs/gap_ledger.md` +- `docs/operator_questions.md` +- `docs/current_state_audit.md` +- `docs/coordination.defaults.example.toml` +- `docs/coordination/focus.example.txt` + +Mirrored bundle in `model-runner`: +- `docs/downstream/lab_llm/` + +Response file expected from `model-runner`: +- `docs/downstream/lab_llm/response.md` + +Verification target mirrored into `model-runner`: +- `docs/downstream/lab_llm/verification_target.md` diff --git a/docs/current_state_audit.md b/docs/current_state_audit.md new file mode 100644 index 0000000..a7e5489 --- /dev/null +++ b/docs/current_state_audit.md @@ -0,0 +1,59 @@ +# Current State Audit + +Date: 2026-03-16 + +This file is a blunt assessment of the current dashboard against `docs/successful_lab.md`. + +## Latest evidence reviewed +- `frontend/artifacts/latest.png` +- `frontend/artifacts/history-2026-03-16T08-06-19-037Z.png` +- `frontend/artifacts/api-status-20260316T080618Z.json` +- `frontend/artifacts/api-sessions-20260316T080618Z.json` +- `frontend/artifacts/api-session-detail-20260316T080618Z.json` +- `docs/coordination/inbox/response.md` +- `docs/coordination/inbox/artifacts/hf-qwen35-9b-sess_d8f3abe025ee4201abab2d30fa06188f.jsonl` + +## What already exists +- fixture and JSONL ingestion +- local REST/SSE service +- session list/detail shell +- basic throughput/GPU/KV charts +- load/runtime/turn/log/inspect surfaces + +## What is already directionally correct +- the repo split is correct +- the dashboard is built around operator-facing sessions, not generic metrics alone +- projected truth is available in-place +- upstream canonical telemetry has started to improve for newly verified sessions: + - model display name is split from model path + - requested runtime intent is split from confirmed runtime truth, with mismatches called out explicitly + - activity state is split from session lifecycle status + - KV cache unavailability is explicit instead of silent omission + - throughput now carries trust/measurement-state semantics + - completed-turn throughput now separates raw completion speed from effective request speed + - turn-finished timing can include TTFT plus honest decode latency + - turn stop reason can include source attribution + - log messages can arrive without duplicated timestamp/source prefixes + - GPU samples can include utilization/power/temperature plus explicit diagnosis incompleteness + +## What is still failing usefulness +- the audited frontend/API is still serving older path-only and numeric-only projections even though the newer verified finished session `sess_d8f3abe025ee4201abab2d30fa06188f` is already present in the sessions list +- the selected audited session `sess_c7acaf69b1cc451081d7038203fd9c1e` started at `2026-03-16T06:01:57.731Z`, before the upstream verification run at `2026-03-16T08:03:29.432Z`, so the dashboard is still showing a pre-fix projection for the main detail pane +- too many summaries are still thin wrappers around incomplete telemetry +- model identity is still not trustworthy on the rendered surface +- the top summary still says `Latest event Not yet` even though the audited detail payload has `session.last_event_at: 2026-03-16T08:06:17.491Z` and `latest_runtime.ts: 2026-03-16T08:06:17.488Z` +- tok/s is still visible before its trustworthiness is clear +- session lifecycle and current activity are still collapsed into one `running/live` story +- GPU and KV cards still spend primary space on unavailable signals +- the recent-turn strip still shows `TTFT Unavailable` and does not surface stop reason or plain-language timing +- the rendered log rows are still duplicated/noisy, and the audited detail payload does not expose log entries for cross-checking +- the inspect path stops at projected truth instead of exposing the canonical event fields that now matter +- source/path status is still unavailable on the audited backend +- the session list is still too ambiguous to help an operator pick the right run quickly, even though the sessions API already has timestamps and error state +- bottleneck diagnosis is not possible yet +- several important question areas from `docs/successful_lab.md` have no meaningful surface at all + +## Main conclusion +The current app is now split between better upstream telemetry and an older dashboard projection. The newest verified runner artifact proves real improvement in canonical telemetry, but the newest audited `lab-llm` surface still does not carry forward most of that truth. + +That means the next useful slices should still be chosen from the gap ledger, with emphasis on read-model/API adoption of canonical identity, runtime-truth mismatch semantics, activity state, KV/throughput measurement semantics, richer turn timing, log normalization, GPU live signals, honest freshness/source states, and a less ambiguous session list. diff --git a/docs/dashboard_repo_architecture.md b/docs/dashboard_repo_architecture.md index c7d6f84..ed6e16f 100644 --- a/docs/dashboard_repo_architecture.md +++ b/docs/dashboard_repo_architecture.md @@ -8,6 +8,7 @@ Purpose: - keep repo-facing notes about the downstream dashboard architecture - document the current expected boundary from `model-runner` - avoid turning this repo into the dashboard implementation home +- reinforce that this repo owns the operator-facing gap ledger This file is intentionally downstream-facing and non-authoritative for the dashboard repo. It exists so this repo can remember what the downstream architecture expects today. @@ -129,6 +130,14 @@ The downstream UI should be: - explicit about live vs completed state - explicit about unknown/unavailable metrics +## Gap ownership +This repo owns the product-facing telemetry gap list. + +That means: +- `lab-llm` records which operator questions are not answerable +- `lab-llm` records which telemetry gaps cause those failures +- `model-runner` remains accountable for upstream telemetry truth improvements + ## Open notes - If the dashboard repo needs long-retention storage, that should be solved there, not here. - If the dashboard repo wants richer compare workflows or annotations, that should be solved there, not here. diff --git a/docs/decisions/0003-dashboard-owns-gap-ledger.md b/docs/decisions/0003-dashboard-owns-gap-ledger.md new file mode 100644 index 0000000..69684cb --- /dev/null +++ b/docs/decisions/0003-dashboard-owns-gap-ledger.md @@ -0,0 +1,33 @@ +# 0003 - The dashboard repo owns the telemetry gap ledger + +Date: 2026-03-15 + +## Context +This repo cannot become useful by polishing around weak or incomplete telemetry. + +The runner repo emits telemetry, but the dashboard repo is where usability failures become obvious: +- duplicated or misleading labels +- unknown values shown as if they were facts +- noisy or untrustworthy metrics +- missing model/load/runtime truth +- views that fail to answer operator questions + +If gap ownership stays split or informal, the dashboard will drift into generic UI work while telemetry truth problems remain unresolved. + +## Decision +`lab-llm` owns the telemetry gap ledger. + +That means: +- this repo records the operator questions the product must answer +- this repo records the telemetry gaps preventing those answers +- this repo defines the required upstream contract improvements +- `model-runner` is accountable to that ledger for telemetry truth and completeness + +The runner repo still owns telemetry implementation. +This repo owns the product-facing definition of what is missing and why it matters. + +## Consequences +- Dashboard work should be driven by question-answering gaps, not generic UI iteration. +- Every important dashboard slice should tie back to one or more operator questions. +- Telemetry issues found here should be written as upstream contract gaps rather than silently worked around. +- Raw JSON/debug surfaces remain important, but they are not the acceptance surface for usefulness. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 8c5f8d1..0b2926f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -8,6 +8,7 @@ Architectural decisions for `lab-llm`. |---|---|---| | 0001 | Consume runner telemetry through external interfaces | This repo consumes canonical telemetry via external app/metrics boundaries rather than importing runner internals. | | 0002 | Build an observability-first web product with room for chat later | The primary UX is runtime observability now, with a shared browser shell that can host chat later. | +| 0003 | The dashboard repo owns the telemetry gap ledger | `lab-llm` defines operator-facing telemetry gaps and holds `model-runner` accountable for closing them. | ## Format diff --git a/docs/gap_ledger.md b/docs/gap_ledger.md new file mode 100644 index 0000000..0ba6c0d --- /dev/null +++ b/docs/gap_ledger.md @@ -0,0 +1,338 @@ +# Telemetry Gap Ledger + +Date: 2026-03-16 + +Purpose: +- make `lab-llm` the accountability surface for telemetry quality +- turn `docs/successful_lab.md` into a working gap list +- record why the dashboard is not yet answering core operator questions +- define the upstream facts `model-runner` must provide or clarify + +## How to use this file +Each gap is phrased in product terms, not schema terms. + +Template: +- `Question`: what the operator is trying to answer +- `Expected answer`: what a useful dashboard should say +- `Current behavior`: what the dashboard says now +- `Telemetry gap`: what is missing, ambiguous, noisy, or wrong upstream +- `Dashboard gap`: what `lab-llm` still fails to interpret or present clearly +- `Fallback`: how the dashboard should behave until the gap is fixed +- `Upstream owner`: usually `model-runner` +- `Status`: `open`, `partial`, or `closed` + +## Current audit summary +Relative to `docs/successful_lab.md`, the current dashboard is: +- **partially useful** for a narrow slice of session/turn inspection +- **not yet useful** for trustworthy diagnosis +- **not yet understandable** enough for a non-expert operator + +The dominant failure modes are: +- the audited dashboard/API is lagging newer canonical telemetry truth +- model/load/runtime facts are still too incomplete or too raw on the rendered surface +- unknown vs noisy vs trustworthy signals are not clear enough +- performance plots exist without enough explanation context +- freshness and source-state cards still misstate or hide available evidence +- the UI still exposes telemetry quality problems more than it resolves them + +## Audit notes +Latest direct audit inputs: +- screenshot: `frontend/artifacts/latest.png` +- screenshot history: `frontend/artifacts/history-2026-03-16T08-06-19-037Z.png` +- API captures: + - `frontend/artifacts/api-status-20260316T080618Z.json` + - `frontend/artifacts/api-sessions-20260316T080618Z.json` + - `frontend/artifacts/api-session-detail-20260316T080618Z.json` +- upstream response and verification evidence: + - `docs/coordination/inbox/response.md` + - `docs/coordination/inbox/artifacts/hf-qwen35-9b-sess_d8f3abe025ee4201abab2d30fa06188f.jsonl` + +The findings below are based on the rendered dashboard and those live artifacts, not on hypothetical UI issues. +Upstream-only improvements are recorded as `partial` unless the audited dashboard/API also reflects them. + +## Model reality gaps +### GAP-001: What model actually loaded? +- `Question`: What model did the runtime actually load? +- `Expected answer`: A human-readable model identity, not just a filesystem path unless no better identity exists. +- `Current behavior`: In the newest audited screenshot and API captures, the operator still gets path-like identity on the main surface: session rows use `resolved_model_id`, the selected title is `/home/poop/ml/models/Qwen3.5-9B`, and no separate display-label/path fields appear. The sessions API at `frontend/artifacts/api-sessions-20260316T080618Z.json` still lists the newly verified finished session `sess_d8f3abe025ee4201abab2d30fa06188f` with only path-like `resolved_model_id`, even though the canonical verification artifact now carries `model_display_name: "Qwen3.5-9B"` plus explicit `model_path`. +- `Telemetry gap`: canonical runner telemetry is improved for the verified 2026-03-16 HF session, but the audited dashboard capture is still backed by older path-only projections/data for the operator surface. +- `Dashboard gap`: the dashboard still treats path-like identifiers as if they were normal model names. +- `Fallback`: show path only when no better identity exists, label it explicitly as a path, and avoid repeating it across multiple cards. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-015: The same path is repeated as session title, page title, and model value +- `Question`: What model loaded, without making the operator parse a filesystem path repeatedly? +- `Expected answer`: One clear primary model label, with filesystem path available separately when needed. +- `Current behavior`: `frontend/artifacts/latest.png` still shows `/home/poop/ml/models/Qwen3.5-9B` as the selected session title, page title, and model value. `frontend/artifacts/api-sessions-20260316T080618Z.json` still exposes only path-like `resolved_model_id` rows, including the newer verified finished session `sess_d8f3abe025ee4201abab2d30fa06188f`. +- `Telemetry gap`: upstream canonical events now separate `model_display_name` and `model_path` for the verified 2026-03-16 session, but the audited sessions/detail API still does not project those fields. +- `Dashboard gap`: the dashboard repeats the path in multiple “identity” positions instead of collapsing it into one labeled path field and one human-facing model label. +- `Fallback`: derive a short display name from the basename when only a path is available, and label the full path explicitly as `Model path`. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-002: Which runtime engine and attention path are actually active? +- `Question`: Which engine loaded the model, and what attention implementation is really active? +- `Expected answer`: Engine and attention backend are visible in a clear load summary. +- `Current behavior`: some load truth exists, but it is still mostly raw JSON and may be partial. +- `Telemetry gap`: upstream load reports are incomplete and may arrive incrementally. +- `Dashboard gap`: the dashboard does not clearly separate confirmed load facts from missing ones. +- `Fallback`: show explicit “unknown/not yet confirmed” states in summary cards. +- `Upstream owner`: shared, with truth from `model-runner` +- `Status`: `partial` + +### GAP-016: Source/path status is missing even when the dashboard is live +- `Question`: Where is this dashboard getting its telemetry from right now? +- `Expected answer`: Source mode and source path should be visible whenever the dashboard is running. +- `Current behavior`: The screenshot still shows `Source Unavailable` and `Path Unavailable`. The newest status capture is `{ "ingestion": null, "note": "status endpoint unavailable on audited backend" }`, so the operator still cannot tell whether the source is genuinely unknown or merely not exposed by this backend version. +- `Telemetry gap`: the audited backend still does not expose inspectable source mode/path state through the status API. +- `Dashboard gap`: the page presents “Unavailable” rather than a more honest statement such as “status endpoint missing on this backend version.” +- `Fallback`: distinguish `dashboard cannot inspect source` from `source genuinely unknown`. +- `Upstream owner`: shared +- `Status`: `open` + +### GAP-027: Freshness summary says `Latest event Not yet` even when events are present +- `Question`: Is telemetry live, stale, or broken? +- `Expected answer`: The summary should show the latest observed event time, or clearly say which evidence path is missing. +- `Current behavior`: `frontend/artifacts/latest.png` still shows `Latest event Not yet`, but `frontend/artifacts/api-session-detail-20260316T080618Z.json` includes `session.last_event_at: "2026-03-16T08:06:17.491Z"` and `latest_runtime.ts: "2026-03-16T08:06:17.488Z"` for the selected session. +- `Telemetry gap`: none required for the audited session; usable freshness timestamps are already present. +- `Dashboard gap`: the freshness card is not wired to available evidence, so it makes live telemetry look absent. +- `Fallback`: render the newest observed timestamp from session/detail data, or state explicitly that the freshness source is unavailable on this backend. +- `Upstream owner`: `lab-llm` +- `Status`: `open` + +### GAP-003: Did the runtime silently change configuration? +- `Question`: Did dtype, quantization, context length, or attention silently differ from what was intended? +- `Expected answer`: The dashboard should surface confirmed runtime values and make mismatches obvious. +- `Current behavior`: The audited detail payload for the selected session `sess_c7acaf69b1cc451081d7038203fd9c1e` still shows one projected `context_length: 49152` in `load_report` and `inspect.load_truth`, with no requested-vs-confirmed split or mismatch hint. The verified upstream session `sess_d8f3abe025ee4201abab2d30fa06188f` now carries `payload.extension.runtime_truth.requested`, `confirmed`, and `mismatches`, including a confirmed `context_length: 262144` vs requested `49152`. +- `Telemetry gap`: canonical runner telemetry is improved for the verified 2026-03-16 HF session, but the audited dashboard/API still does not project the new runtime-truth split or mismatch evidence. +- `Dashboard gap`: there is no mismatch-oriented view yet. +- `Fallback`: never present requested config as runtime truth; show only confirmed facts and mark the rest unknown. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-017: Summary cards overclaim certainty when key fields are absent +- `Question`: Can an operator trust the summary without reading raw JSON? +- `Expected answer`: Summary cards should only state confirmed facts and should not imply a complete explanation when important fields are missing. +- `Current behavior`: The screenshot shows confident summaries for model/runtime while TTFT, stop reason, GPU util, KV util, source, and path are unavailable. +- `Telemetry gap`: some key upstream fields are absent. +- `Dashboard gap`: the page still reads like a complete dashboard rather than a partial truth surface with obvious holes. +- `Fallback`: add stronger “partial data” signaling at the card or section level when critical fields are missing. +- `Upstream owner`: `lab-llm` +- `Status`: `open` + +## GPU and memory gaps +### GAP-004: What is the GPU doing right now? +- `Question`: Is the GPU compute-bound, memory-bound, idle, or blocked? +- `Expected answer`: A non-expert should see the relevant GPU state quickly. +- `Current behavior`: The audited screenshot still shows a `GPU util` card with `Unavailable` and no samples, while `frontend/artifacts/api-session-detail-20260316T080618Z.json` for the selected session still exposes only VRAM bytes. The verified upstream session `sess_d8f3abe025ee4201abab2d30fa06188f` now emits utilization, power, temperature, and explicit GPU `measurement_state` plus `diagnosis_state: "incomplete"`, but that richer truth is not visible on the audited operator surface. +- `Telemetry gap`: canonical runner telemetry is improved because GPU coverage and incompleteness are now explicit for the verified HF session, but the audited dashboard/API still does not expose enough of that truth to answer the operator question. +- `Dashboard gap`: there is no structured GPU state panel or interpretation language yet. +- `Fallback`: present only confirmed metrics and state clearly that diagnosis is incomplete. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-018: GPU card exists without enough signals to answer the GPU question +- `Question`: What is the GPU doing right now? +- `Expected answer`: Either a useful GPU state summary or a clear statement that the necessary signals are not present. +- `Current behavior`: The newest screenshot still shows a `GPU util` chart card with no samples and `GPU util Unavailable`, while `frontend/artifacts/api-session-detail-20260316T080618Z.json` only contains VRAM bytes for the selected live session. Upstream verified evidence for `sess_d8f3abe025ee4201abab2d30fa06188f` now includes `utilization_percent`, `power_usage_watts`, `temperature_celsius`, and explicit GPU diagnosis metadata, but the audited dashboard capture does not surface them. +- `Telemetry gap`: canonical runner telemetry now emits key GPU live signals for the verified HF session, but the audited dashboard/API is still backed by data or projections that expose only VRAM numbers to the operator. +- `Dashboard gap`: the chart panel occupies primary space without telling the operator what is missing or why it matters. +- `Fallback`: replace empty GPU charts with a compact “GPU telemetry incomplete” explanation until enough signals exist. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-005: Is memory behavior healthy? +- `Question`: Is memory fragmentation, offload, or cache pressure happening? +- `Expected answer`: The dashboard should let an operator infer whether memory behavior is normal or suspicious. +- `Current behavior`: runtime JSON may expose some allocator/cache data, but not in an understandable way. +- `Telemetry gap`: allocator/offload/swap signals are incomplete or absent upstream. +- `Dashboard gap`: there is no memory integrity summary layer yet. +- `Fallback`: preserve raw truth, but do not imply memory health when the signals are incomplete. +- `Upstream owner`: shared +- `Status`: `open` + +## KV cache and throughput gaps +### GAP-006: Is KV cache limiting performance? +- `Question`: Is the cache underutilized, saturated, or thrashing? +- `Expected answer`: The dashboard should show cache size, usage trend, and whether cache pressure is a likely bottleneck. +- `Current behavior`: a KV utilization plot exists, but the surrounding explanation is not there. +- `Telemetry gap`: cache-level details like eviction/thrashing are incomplete upstream. +- `Dashboard gap`: the app charts cache numbers without enough operator interpretation. +- `Fallback`: say what is known, do not pretend to diagnose thrashing if the required signals do not exist. +- `Upstream owner`: shared +- `Status`: `open` + +### GAP-019: KV cache panel overpromises while providing no usable signal +- `Question`: Is KV cache helping, limiting, or irrelevant here? +- `Expected answer`: A real cache signal or an explicit statement that cache telemetry is absent. +- `Current behavior`: The screenshot shows `KV cache util Unavailable` and “No samples for this metric yet” in a first-class chart slot. +- `Telemetry gap`: canonical telemetry for the verified session now emits an explicit unavailable state with `measurement_state: "unavailable"` and `unavailable_reason: "backend_does_not_report_kv_cache_runtime_usage"`, but `frontend/artifacts/api-session-detail-20260316T080618Z.json` still omits any KV cache object or reason for the audited session. +- `Dashboard gap`: the app still allocates premium UI space to a chart that cannot answer anything. +- `Fallback`: collapse missing-signal charts into a lower-emphasis telemetry-missing state instead of presenting them like working panels. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-007: Are tok/s numbers trustworthy? +- `Question`: Is generation speed actually good/bad, or is the metric noisy and misleading? +- `Expected answer`: The dashboard should either show a trustworthy tok/s value or clearly say the metric is noisy/unknown. +- `Current behavior`: The newest screenshot and detail capture still headline numeric `Tok/s` while `requests_in_flight` is `0`; throughput is rendered as a plain number with no trust or measurement-state context. Upstream verified evidence for `sess_d8f3abe025ee4201abab2d30fa06188f` now emits `measurement_state`, `trust_level`, `trust_reason`, and idle samples with no numeric tok/s, but the audited dashboard surface does not show those semantics. +- `Telemetry gap`: canonical HF throughput semantics improved in the verified 2026-03-16 session, but the audited dashboard capture still exposes the older numeric-only throughput shape. +- `Dashboard gap`: the UI still renders tok/s with more confidence than the source telemetry deserves, and it cannot yet present the newer trust semantics. +- `Fallback`: mark throughput as provisional/noisy for affected sources until verified; do not imply confidence. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-020: Tok/s is shown as the lead performance number even after the request has completed +- `Question`: What is happening right now, and is tok/s still a live indicator? +- `Expected answer`: The operator should know whether tok/s reflects active generation, completed-turn decay, or stale telemetry. +- `Current behavior`: The newest screenshot highlights `Tok/s 1.199873226785108` and a throughput chart while the newest detail capture simultaneously shows `requests_in_flight: 0` and `requests_completed_total: 2`. +- `Telemetry gap`: verified upstream telemetry no longer emits decaying idle tok/s for the new 2026-03-16 session, but the audited dashboard capture still relies on data/projections that do. +- `Dashboard gap`: the dashboard treats tok/s as the headline live metric even though the run may no longer be actively generating. +- `Fallback`: down-rank tok/s or mark it stale/non-live whenever `requests_in_flight` is zero and no active generation signal exists. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-008: Can the operator distinguish raw speed from effective speed? +- `Question`: Is the system fast at generation, or just hiding latency elsewhere? +- `Expected answer`: Raw generation rate and effective request speed should be clearly distinguished. +- `Current behavior`: The audited detail payload for `sess_c7acaf69b1cc451081d7038203fd9c1e` still exposes `tokens_generated_per_second` and `effective_tokens_per_second` with the same value, so the operator still gets one merged speed story. The verified upstream session `sess_d8f3abe025ee4201abab2d30fa06188f` now separates idle runtime throughput semantics from completed-turn raw vs effective speed, including `raw_completion_tokens_per_second: 28.016341890322057` and `effective_completion_tokens_per_second: 22.884639184657416`. +- `Telemetry gap`: canonical throughput truth is improved for the verified HF session, but the audited dashboard/API still does not project those separate speed semantics to the operator. +- `Dashboard gap`: the current summaries are still too metric-centric and not explanatory enough. +- `Fallback`: show missing effective-speed metrics as unknown instead of folding them into one “tok/s” idea. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-021: Raw and effective speed are still visually collapsed into one story +- `Question`: Is this speed number raw generation speed, effective request speed, or something else? +- `Expected answer`: Raw and effective speed should be clearly distinct or explicitly merged only when the dashboard can justify it. +- `Current behavior`: The newest detail capture still exposes both `tokens_generated_per_second` and `effective_tokens_per_second` with the same value, so the operator still gets one visually merged speed story. Upstream verified runtime samples for `sess_d8f3abe025ee4201abab2d30fa06188f` now omit fabricated `effective_tokens_per_second` and instead carry measurement basis/trust metadata, but that distinction is not visible in the audited dashboard surface. +- `Telemetry gap`: canonical upstream semantics are improved for the verified session, but the audited API contract still makes raw and effective speed look like the same thing. +- `Dashboard gap`: the UI does not teach the operator the difference, so even correct data is easy to misread. +- `Fallback`: label throughput cards more precisely or show a note when only one trustworthy speed story is available. +- `Upstream owner`: `lab-llm` +- `Status`: `partial` + +## Latency and request flow gaps +### GAP-009: Where is latency accumulating? +- `Question`: Is latency coming from prefill, decode, queueing, or something unknown? +- `Expected answer`: The dashboard should help narrow the source of latency quickly. +- `Current behavior`: some turn-level timing may appear, but not as an interpretable latency story. +- `Telemetry gap`: upstream timing breakdowns are incomplete and inconsistent. +- `Dashboard gap`: there is no latency diagnosis surface yet. +- `Fallback`: expose timing fields separately and mark the diagnosis as incomplete. +- `Upstream owner`: shared +- `Status`: `open` + +### GAP-022: Turn summary hides a very important latency fact by rendering it as unavailable +- `Question`: How long did the last request really take, and where is that shown? +- `Expected answer`: The last-turn summary should surface the most important timing fields that are actually available. +- `Current behavior`: The screenshot still shows `TTFT Unavailable` in the recent-turn strip for the selected audited session, even though latency is otherwise present. In the verified upstream session `sess_d8f3abe025ee4201abab2d30fa06188f`, `turn_finished` now carries `request_latency_seconds: 5.593271493911743`, `time_to_first_token_seconds: 1.0245094299316406`, and `decode_latency_seconds: 4.5687620639801025`, but the audited dashboard/API surface does not expose an equivalent richer turn summary yet. +- `Telemetry gap`: canonical turn telemetry is improved for the verified session, but the audited recent-turn projection still does not show TTFT or a clearer latency breakdown for the operator. +- `Dashboard gap`: the turn summary does not pivot to the strongest available timing signal when a preferred one is missing. +- `Fallback`: show available request/decode latency prominently even when TTFT is unavailable. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-010: Are requests queueing or batching? +- `Question`: Is the engine idle, queue-saturated, batch-limited, or stalled? +- `Expected answer`: The dashboard should clearly show request flow health. +- `Current behavior`: request flow is mostly absent from the current dashboard surface. +- `Telemetry gap`: queue and batch signals are weak or missing for many paths upstream. +- `Dashboard gap`: there is no request flow panel yet. +- `Fallback`: do not imply queueing/batching state unless the metrics are present. +- `Upstream owner`: shared +- `Status`: `open` + +### GAP-023: Session state and request state do not reconcile clearly +- `Question`: Is this session actively generating right now, or merely still open? +- `Expected answer`: `running`, `idle`, `generating`, and `finished` should be distinguishable enough that request state makes sense. +- `Current behavior`: The screenshot shows `Status running` and `Live`, while the raw runtime shows `requests_in_flight: 0` and the only visible turn already finished. +- `Telemetry gap`: canonical telemetry for the verified session now separates lifecycle and activity with `activity_state` and `activity_state_reason`, but the audited session/detail API still exposes only `status` plus request counters. +- `Dashboard gap`: the dashboard conflates “process still alive” with “actively generating now.” +- `Fallback`: introduce an operator-facing derived activity state such as `idle` when the session is open but no request is in flight. +- `Upstream owner`: shared +- `Status`: `partial` + +## Turn/debug gaps +### GAP-011: Can a non-expert understand the last turn quickly? +- `Question`: What happened on the last turn, how many tokens came out, and why did it stop? +- `Expected answer`: A clear plain-language summary with raw detail available if needed. +- `Current behavior`: recent turns are present, but the presentation is still too raw and metric-first. +- `Telemetry gap`: turn stop semantics may still be inconsistent upstream. +- `Dashboard gap`: the last-turn summary is not yet accessible enough for non-experts. +- `Fallback`: keep raw JSON accessible but move toward more readable summaries. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-024: The recent-turn strip is too compressed to be a useful explanation surface +- `Question`: What happened on the last turn? +- `Expected answer`: A non-expert should get outcome, duration, token count, and stop reason from one readable row or card. +- `Current behavior`: The screenshot shows `Turn 1`, `TTFT Unavailable`, and `Out 8192`, but no stop reason and no plain-language explanation of what 8192 means. +- `Telemetry gap`: canonical turn telemetry is improved for the verified session because `turn_finished` now carries `stop_reason` and `stop_reason_source`, but the audited session/detail API still does not project that richer explanation into the rendered turn strip. +- `Dashboard gap`: the turn surface is still too terse and too metric-fragmented. +- `Fallback`: prefer a richer last-turn summary card over a narrow metrics strip when upstream detail is partial. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-012: Can every panel be traced back to raw truth? +- `Question`: What raw data produced this state? +- `Expected answer`: Every important panel has a raw JSON/log/event escape hatch. +- `Current behavior`: projected inspect JSON is present in several places, which is good, but the newest audited detail capture only exposes `load_truth`, `generation_truth`, and `backend_extensions`; it does not expose the canonical raw event fields that now matter for model identity split or throughput trust semantics. +- `Telemetry gap`: none required for the principle itself. +- `Dashboard gap`: the app needs stronger “summary first, raw proof second” discipline across all panels. +- `Fallback`: keep raw JSON close to every summary view. +- `Upstream owner`: `lab-llm` +- `Status`: `partial` + +### GAP-025: Logs currently duplicate timestamps and backend prefixes inside the rendered message +- `Question`: Can the operator read logs quickly without parsing duplicated formatting noise? +- `Expected answer`: The log surface should present source, timestamp, and message cleanly once. +- `Current behavior`: The screenshot still shows log rows like `backend 2026-03-16T06:17:26.487Z [backend] turn_finish ...`, duplicating source and timestamp inside the message body. In the verified upstream session `sess_d8f3abe025ee4201abab2d30fa06188f`, the canonical `log_recorded.payload.message` values are now clean strings like `turn_start id=1 messages=1 stream=True max_new_tokens=128` and `turn_finish id=1 elapsed_s=5.593 think_chars=417 answer_chars=0`, but the audited detail capture still exposes `logs: null`. +- `Telemetry gap`: canonical log message semantics are improved for the verified session, but the audited operator surface is still not showing those cleaner payloads. +- `Dashboard gap`: the dashboard either renders older preformatted log strings or cannot trace the rendered rows back to the captured API cleanly. +- `Fallback`: detect preformatted log messages and strip redundant prefixes at render time, or render only one canonical representation. +- `Upstream owner`: shared +- `Status`: `partial` + +### GAP-026: Canonical telemetry fixes are not reaching the audited dashboard API +- `Question`: Did the newer upstream truth actually make it to the operator surface? +- `Expected answer`: When canonical telemetry adds model display/path split, throughput trust state, or GPU live signals, the session/detail API and inspect surface should expose them. +- `Current behavior`: The inbox verification artifact for `sess_d8f3abe025ee4201abab2d30fa06188f` includes `model_display_name`, `model_path`, activity state, KV measurement state, throughput `measurement_state`/`trust_level`, distinct raw vs effective turn throughput, richer turn latency fields, cleaned log messages, and GPU utilization/power/temperature. The newest audited frontend captures still expose path-only identity, numeric idle tok/s, VRAM-only GPU fields, no activity state, no KV measurement state, no richer turn timing fields, and no raw-event view of those canonical fields. +- `Telemetry gap`: none for the verified upstream session. +- `Dashboard gap`: `lab-llm` read models and API projections are still lagging the canonical event schema, so real upstream fixes do not reliably reach the operator surface. +- `Fallback`: label the audited backend as using an older projection and keep canonical raw-event evidence reachable until the API catches up. +- `Upstream owner`: `lab-llm` +- `Status`: `open` + +## Session navigation gaps +### GAP-028: The session list is too ambiguous to help an operator pick the right run +- `Question`: Which session should I open to inspect the run I care about? +- `Expected answer`: The list should make runs distinguishable by human label plus a small amount of provenance such as status, freshness, ended time, or error state. +- `Current behavior`: In `frontend/artifacts/latest.png`, the left rail shows several nearly identical rows with the same truncated path-like model label and tiny `hf` plus `running` or `finished` badges. The newest sessions API already includes `started_at`, `ended_at`, `last_event_at`, and `has_errors` for each row, including both the selected long-running session `sess_c7acaf69b1cc451081d7038203fd9c1e` and the newer verified finished session `sess_d8f3abe025ee4201abab2d30fa06188f`, but the rendered list does not surface enough of that context to choose confidently. +- `Telemetry gap`: none required for a basic disambiguation pass; the sessions API already has enough timing and error metadata to improve the list materially. +- `Dashboard gap`: the session list hides too much row context, so the operator must click through multiple nearly identical entries to find the relevant run. +- `Fallback`: show a compact secondary line with freshness or ended time and retain a short stable identifier when labels collide. +- `Upstream owner`: `lab-llm` +- `Status`: `open` + +## Synthesis and comparison gaps +### GAP-013: What is currently limiting performance? +- `Question`: What is the most likely bottleneck right now? +- `Expected answer`: The dashboard should at least narrow the diagnosis to compute, memory, KV cache, queueing, or unknown. +- `Current behavior`: the app is not yet capable of giving a credible diagnosis, even manually. +- `Telemetry gap`: multiple upstream signals are incomplete. +- `Dashboard gap`: there is no diagnosis-oriented synthesis layer yet. +- `Fallback`: explicitly say “cannot yet determine” and surface which signals are missing. +- `Upstream owner`: shared +- `Status`: `open` + +### GAP-014: Did this configuration improve anything? +- `Question`: Did a change in backend/config/context/quantization improve performance? +- `Expected answer`: The app should support experiment comparison once the basics are trustworthy. +- `Current behavior`: comparison is effectively absent. +- `Telemetry gap`: experiment labeling and stable comparative signals are not yet mature enough upstream. +- `Dashboard gap`: comparison workflows are not started, which is acceptable until operational debugging is trustworthy. +- `Fallback`: defer comparison until single-run truth is reliable. +- `Upstream owner`: shared +- `Status`: `open` diff --git a/docs/headless_codex_loop_reference.md b/docs/headless_codex_loop_reference.md new file mode 100644 index 0000000..aeb83c2 --- /dev/null +++ b/docs/headless_codex_loop_reference.md @@ -0,0 +1,448 @@ +# Headless Codex Loop Reference + +Purpose: +- capture the generic pattern used here for autonomous multi-pass work across one or more repos +- make it easy to recreate the workflow in a future Codex session without re-deriving the structure +- keep the pattern repo-agnostic + +This is not a product spec. +It is a coordination pattern. + +## When to use this pattern +Use a headless Codex loop when all of the following are true: +- the work spans multiple passes +- the work can be evaluated from files, artifacts, tests, or screenshots +- one repo or surface can audit another +- you want Codex to keep iterating without asking you to restate the same thing +- the main bottleneck is coordination, not raw implementation + +Good fits: +- producer repo and consumer repo +- telemetry producer and dashboard consumer +- library repo and example app repo +- service repo and client repo + +Bad fits: +- work that depends on subjective visual taste every pass +- work that requires frequent human product decisions +- tasks with no reliable audit or verification signal + +## Core idea +Do not coordinate through chat. + +Coordinate through files plus a small set of scripts: +1. a source-of-truth ledger of gaps or goals +2. a generated work slice +3. a mirrored downstream or upstream bundle +4. a required response file +5. an audit step that produces artifacts +6. a reconciliation step that updates the ledger +7. a loop runner that repeats until there is no more eligible work + +The important shift is: +- chat is for setup +- files are for execution + +## Roles +The pattern works best when each repo or surface has one clear role. + +Example generic roles: +- `producer` + - emits the thing being consumed + - fixes truth, semantics, and source behavior +- `consumer` + - audits usefulness + - owns the gap ledger + - defines whether the producer output is good enough + +This matters because one side has to own the accountability surface. + +## Required artifacts +These names are examples, not requirements. + +### 1. Gap ledger +A markdown file that records: +- operator or user question +- expected answer +- current behavior +- upstream gap +- downstream gap +- fallback behavior +- owner +- status + +Why: +- Codex needs a stable backlog that is phrased in product terms, not just code tasks + +### 2. Operator questions +A short doc listing the concrete questions the product must answer. + +Why: +- this prevents iteration from collapsing into generic UI cleanup + +### 3. Current state audit +A short summary of what the current system can and cannot do. + +Why: +- lets Codex compare progress against the last known state + +### 4. Current gap slice +A generated file containing only the next 3-5 work items. + +Why: +- limits scope +- reduces thrash +- gives the upstream repo a bounded contract + +### 5. Verification target +A generated file telling the upstream repo what real workload it should run after relevant changes. + +Why: +- prevents false claims like "fixed" without exercising the changed path + +### 6. Response file +A file the upstream repo must write back after its pass. + +Recommended sections: +- addressed gaps +- gaps attempted but not closed +- gaps blocked +- changes made +- verification run +- evidence paths + +### 7. Loop state +A machine-readable file storing: +- recently selected gaps +- attempt counts +- blocked counts +- addressed counts +- latest response path + +Why: +- lets the next slice avoid blindly retrying the same items + +## Required scripts +You do not need these exact names, but you do need these jobs. + +### 1. Slice generator +Input: +- gap ledger +- loop state + +Output: +- current gap slice + +Responsibilities: +- pick a small number of eligible items +- prioritize high-signal, high-accountability work +- deprioritize recently attempted or blocked items + +### 2. Push script +Input: +- current gap slice +- ledger +- audit docs +- latest artifacts +- verification target + +Output: +- mirrored work bundle in the other repo + +Responsibilities: +- copy only the files needed for the other repo to act +- keep the source of truth in one place + +### 3. Upstream pass runner +Input: +- mirrored bundle + +Output: +- updated code/docs in the upstream repo +- completed response file + +Responsibilities: +- run Codex headlessly in the upstream repo +- instruct it to change only the selected slice +- require a real verification run when behavior changed + +### 4. Pull script +Input: +- upstream response bundle + +Output: +- local inbox copy + +Responsibilities: +- copy response and any upstream evidence back to the consumer repo + +### 5. Audit script +Input: +- running app or service + +Output: +- artifacts such as screenshots, API captures, logs, or test outputs + +Responsibilities: +- produce evidence that the consumer repo can use to judge whether changes helped + +### 6. Reconciliation script +Input: +- upstream response +- latest audit artifacts +- ledger +- loop state + +Output: +- updated ledger +- updated audit summary +- regenerated next slice + +Responsibilities: +- close gaps only when evidence supports it +- add newly obvious gaps +- keep the ledger authoritative + +### 7. Loop runner +Responsibilities: +- run the upstream pass +- pull the response +- rerun the audit +- reconcile +- stop when no eligible gaps remain or a hard limit is reached + +## Prompt design +Headless loops work or fail based on prompt quality. + +The prompt for the upstream repo should be strict: +- read these files +- implement only these gaps +- do not edit the mirrored ledger +- record exact evidence +- do not claim success without verification + +The prompt for reconciliation should also be strict: +- trust audit evidence over prose +- do not close a gap because the other repo said it was fixed +- preserve the ledger as source of truth + +This is the minimum level of specificity that keeps the loop honest. + +## Verification rule +The most important instruction in this pattern is: + +- if code changes affect runtime behavior, telemetry, API shape, or any audited surface, the loop must execute a real verification path before claiming success + +That path should be: +- as narrow as possible +- as representative as necessary +- chosen by a generated target note or clear repo defaults + +Without this rule, the loop degenerates into “edited code, wrote a note, maybe works.” + +For producer-side runtime changes, the stronger version is: +- if the changed code can be bypassed by a stale live process, the loop must kill/restart or cleanly reload before verification +- if the runnable path depends on a build/install step, the loop must rebuild from latest source before the fresh run +- the response must include a fresh session, run, or artifact identifier produced after that reload + +## How to make model or backend selection smart +If the upstream repo has multiple runnable targets, infer a default verification target from: +- configured sink path +- selected gap text +- known backend or model names in the current environment + +Use this order: +1. explicit configured target +2. target inferred from the currently active sink or fixture path +3. target inferred from selected gap text +4. safe default target documented in repo config + +Tell Codex: +- prefer the inferred target +- only override it when the selected gaps clearly concern another path +- record the exact command used + +For dashboard or client-side audits, use the same principle: +- prefer the verified session or artifact named in the upstream response +- if there is no explicit verified target, prefer the newest running session over an older finished one + +## Defaults matter +To run unattended, store defaults in one config file. + +Typical defaults: +- other repo root +- audit URL +- fixture or sink path +- slice size +- iteration count +- execution mode + +If the loop needs arguments every run, it is not really unattended. + +## How to ask for this in a future Codex session +The shortest useful version is: + +1. "Set up a file-based autonomous loop between these two repos." +2. "Repo A owns the gap ledger and audit. Repo B owns implementation." +3. "Use generated gap slices of 3-5 items." +4. "Mirror only the necessary files into Repo B." +5. "Have Repo B write a structured response file back." +6. "After changes that affect runtime or telemetry, require a real verification run." +7. "Reconcile based on artifacts, not claims." +8. "Add a loop state file so selection gets smarter over time." +9. "Give me one command to kick off the unattended loop." + +If you want better results, also provide: +- what counts as proof +- where the live app or service is +- which repo owns truth vs presentation + +## Common failure modes +### 1. No accountability owner +Symptom: +- both repos wait for the other to define “done” + +Fix: +- one repo must own the ledger and audit + +### 2. No verification run +Symptom: +- loops claim fixes that never show up in artifacts + +Fix: +- require a real workload execution after relevant changes + +### 3. Slice too large +Symptom: +- upstream repo works vaguely and returns partial chaos + +Fix: +- keep slices at 3-5 items + +### 4. Response file too loose +Symptom: +- reconciliation cannot tell what was attempted, blocked, or verified + +Fix: +- force a structured response shape + +### 5. Audit too weak +Symptom: +- loop cannot tell whether the system improved + +Fix: +- capture screenshots, API responses, logs, tests, or other artifacts + +### 6. Reconciliation trusts prose +Symptom: +- ledger closes gaps that still appear in the live product + +Fix: +- trust artifacts over claims + +### 7. No loop state +Symptom: +- the same gaps get selected repeatedly + +Fix: +- store attempt, blocked, and addressed history + +### 8. Audit is looking at the wrong artifact or session +Symptom: +- the producer repo verified a real change +- the consumer repo still appears unchanged +- reconciliation cannot tell whether the loop made progress + +Fix: +- make the audit target explicit +- prefer the newest verified session, fixture, or artifact after an upstream pass +- if the upstream response includes a verified session id or evidence path, feed that forward into the next audit instead of letting the consumer inspect an older default target + +### 9. The loop ends because of iteration limits, not because the work is done +Symptom: +- the loop stops cleanly +- there is still an eligible next slice + +Fix: +- distinguish "completed configured passes" from "no more eligible work" +- record which reason ended the loop +- use a conservative iteration cap for unattended runs and inspect the next generated slice in the morning + +## Lessons from a real run +These are generic because they apply to most headless loops, not just this repo. + +### Verification evidence must feed the next audit +It is not enough for the upstream repo to run a real verification command. +The consumer repo also has to audit the resulting session, fixture, or artifact. + +If the upstream response says: +- verified session id +- evidence file path +- exact sink path + +then the next audit should prefer those exact targets. + +Otherwise the loop can improve the system while still auditing an older stale view. + +### Structured response files are worth the effort +The loop improved once the response file had explicit sections for: +- addressed gaps +- attempted gaps +- blocked gaps +- verification run +- evidence + +Without that structure, reconciliation has to infer too much from prose. + +### Small slices still need good ranking +Three to five gaps is the right scale, but selection quality matters. + +A useful selector should prefer: +- truth gaps over polish gaps +- audited failures over speculative ones +- items with clear upstream ownership +- items not just attempted in the previous pass unless fresh evidence says they still deserve priority + +### One-command kickoff is necessary but not sufficient +An unattended loop is only really autonomous if: +- it has defaults +- it has a verification rule +- it has loop memory +- it has a strong audit step + +If any of those are missing, the one-command story is mostly cosmetic. + +### Consumer-side code changes should be conditional +Do not make the consumer repo a mandatory churn target every pass. + +Only change the consumer when: +- the upstream truth is now present +- the audit proves the consumer is still hiding, collapsing, or misprojecting that truth + +If the audit does not show a real local wiring problem, the consumer should update docs and ledger state only. + +## Minimal implementation recipe +If you want the smallest possible version: + +1. create `gap_ledger.md` +2. create `current_gap_slice.md` generator +3. create `push` and `pull` scripts +4. create `response.md` template +5. create one audit script that produces artifacts +6. create one reconciliation script that updates the ledger +7. create one loop script that runs the previous steps + +That is enough to get real leverage. + +## Rule of thumb +The loop is good when: +- you can start it with one command +- it can run multiple passes without asking you what to do next +- it produces evidence, not just edits +- the next work slice changes based on what happened last time + +The loop is not good when: +- it depends on chat as the transport +- it cannot verify its own claims +- it keeps retrying the same work without memory +- it still needs you to restate the goal every pass diff --git a/docs/operator_questions.md b/docs/operator_questions.md new file mode 100644 index 0000000..dc06164 --- /dev/null +++ b/docs/operator_questions.md @@ -0,0 +1,21 @@ +# Operator Questions + +These are the questions the dashboard must answer before it is considered useful. + +The review surface is whether these questions are answerable quickly and honestly, not whether every panel looks complete. + +## Core questions +- What model/backend actually loaded? +- Is telemetry live, stale, or broken? +- What is happening right now: loading, generating, finished, or errored? +- How fast is it going, and how trustworthy is that speed number? +- What is the likely bottleneck: compute, memory/cache, queueing, or unknown? +- What output did the last turn produce, and why did it stop? +- When I need proof, where is the raw JSON? + +## Review rule +Every meaningful dashboard slice should improve at least one of these questions. + +If a change does not improve a question directly, it should either: +- reduce ambiguity about a question, or +- improve the raw-debug path needed to validate an answer. diff --git a/docs/successful_lab.md b/docs/successful_lab.md new file mode 100644 index 0000000..05cd4b9 --- /dev/null +++ b/docs/successful_lab.md @@ -0,0 +1,288 @@ +--- +title: "Lab_Answers" +source: "https://chatgpt.com/c/69b78698-0600-832b-9841-b269f21e4c3b" +--- + +# Operator Questions — Runtime Lab Observability + +These questions define the functional requirements of the dashboard. +The system is complete when an operator can answer these questions within seconds without consulting logs or external tools. + +The purpose is not visual completeness but **operational truthfulness**. + +--- + + +## 1\. Model Reality Questions + +These verify that the runtime environment matches the intended configuration. + +Questions the system must answer: + +• What model actually loaded? +• Which runtime engine loaded it? +• What dtype is the model actually using? +• What quantization method is active? +• What attention implementation is active? +• What context length is actually configured? +• Did the runtime silently change any of the above? + +Evidence sources must include: + +• model metadata panel +• load truth JSON +• backend configuration capture + +Failure cases that must be detectable: + +• dtype promotion +• incorrect quantization path +• partial CPU offload +• mismatched model path + +--- + +## 2\. GPU State Questions + +These verify the physical GPU state during runtime. + +Questions the system must answer: + +• What is the GPU doing right now? +• How much VRAM is allocated vs reserved vs free? +• Is the GPU compute pipeline saturated? +• Is memory bandwidth saturated? +• Is the GPU throttling or power limited? +• Is the model fully resident in VRAM? + +The operator must be able to determine: + +compute bound +memory bound +idle +blocked + +The system must expose a **360-degree GPU view** equivalent to an extended `nvidia-smi`. + +Required signals: + +• VRAM used +• VRAM reserved +• VRAM free +• GPU utilization +• SM occupancy +• memory bandwidth utilization +• power draw +• temperature + +--- + +## 3\. Memory Integrity Questions + +These verify that runtime memory behavior matches expectations. + +Questions the system must answer: + +• Is the model fully loaded into VRAM? +• Is memory fragmentation occurring? +• Is PyTorch reserving more memory than expected? +• Is KV cache allocation correct? +• Is memory being swapped to host? +• Is any unexpected offloading occurring? + +The system must detect: + +hidden CPU offload +GPU swap +cache overflow +allocator fragmentation + +Evidence must come from: + +• torch allocator metrics +• GPU memory metrics +• KV cache metrics + +--- + +## 4\. KV Cache Questions + +These determine whether attention memory is behaving correctly. + +Questions the system must answer: + +• How large is the KV cache? +• How much of it is currently used? +• Is the KV cache filling up? +• Is the cache fragmented? +• Are sequences being evicted? +• Is context size limiting performance? + +Operators must be able to determine: + +cache underutilized +cache saturated +cache thrashing + +--- + +## 5\. Throughput Questions + +These measure raw inference speed. + +Questions the system must answer: + +• How many tokens per second are being produced? +• Is that number stable or fluctuating? +• What is the effective tokens/sec including latency? +• Is batching working? +• Is throughput scaling with batch size? + +The dashboard must distinguish: + +raw generation speed +effective request speed + +--- + +## 6\. Latency Questions + +These reveal pipeline bottlenecks. + +Questions the system must answer: + +• What is the time to first token? +• What is prefill latency? +• What is decode latency? +• What is the full request latency? +• Where is latency accumulating? + +Operators must be able to infer: + +attention bottleneck +KV bandwidth bottleneck +queue delay +token sampling overhead + +--- + +## 7\. Request Flow Questions + +These track inference workload behavior. + +Questions the system must answer: + +• How many requests are active? +• Are requests queueing? +• Is batching occurring? +• Are requests finishing successfully? + +Operators must be able to determine: + +idle +queue saturated +batch limited +engine stalled + +--- + +## 8\. Turn-Level Questions + +These verify individual generation results. + +Questions the system must answer: + +• What did the last turn produce? +• How many tokens were generated? +• Why did generation stop? +• Did streaming occur correctly? + +The system must expose: + +• recent turns +• token counts +• stop reason + +--- + +## 9\. Truth and Debug Questions + +These ensure every observation can be verified. + +Questions the system must answer: + +• What raw data produced this metric? +• What backend event produced this state? +• What timestamp did it occur at? +• What configuration produced this run? + +Every panel must link to a raw truth source: + +raw JSON +backend logs +runtime events + +--- + +## 10\. Bottleneck Diagnosis Questions + +These synthesize the previous metrics. + +Questions the system must answer: + +• What is currently limiting performance? + +Possible diagnoses: + +compute bound +memory bound +kv cache bound +queue bound +unknown + +The dashboard does not need to compute the diagnosis automatically, but the signals required to infer it must be visible. + +--- + +## 11\. Experiment Comparison Questions + +These support tuning workflows. + +Questions the system must answer: + +• Did a configuration change improve performance? +• Which engine performs better? +• Which quantization method performs better? +• Which context size is optimal? + +Operators must be able to compare runs using: + +tokens/sec +latency +VRAM usage +cache efficiency + +--- + +## Review Rule + +Every meaningful dashboard surface must improve at least one operator question. + +If a change does not directly answer a question, it must either: + +• reduce ambiguity around a question +• expose raw truth required to validate an answer + +Panels that do not satisfy either condition are not required. + +--- + +## Definition of Done + +The runtime lab observability system is complete when: + +• a model can be launched from the runner +• runtime behavior can be observed live +• performance bottlenecks can be diagnosed without external tools +• configuration truth can be verified without reading logs +• experiments can be compared with measurable metrics diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 98e253a..e507071 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -13,6 +13,7 @@ }, "devDependencies": { "@preact/preset-vite": "^2.10.2", + "playwright": "^1.58.2", "typescript": "^5.9.2", "vite": "^7.1.3" } @@ -1764,6 +1765,53 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.8", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", diff --git a/frontend/package.json b/frontend/package.json index 8afd838..048318e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,7 +6,8 @@ "scripts": { "dev": "vite", "build": "vite build", - "preview": "vite preview" + "preview": "vite preview", + "screenshot": "node scripts/capture.mjs" }, "dependencies": { "preact": "^10.27.2", @@ -14,6 +15,7 @@ }, "devDependencies": { "@preact/preset-vite": "^2.10.2", + "playwright": "^1.58.2", "typescript": "^5.9.2", "vite": "^7.1.3" } diff --git a/frontend/scripts/capture.mjs b/frontend/scripts/capture.mjs new file mode 100644 index 0000000..75eaf89 --- /dev/null +++ b/frontend/scripts/capture.mjs @@ -0,0 +1,25 @@ +import { chromium } from "playwright"; +import fs from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; + +const url = process.argv[2] ?? "http://127.0.0.1:8001/"; +const output = process.argv[3] ?? path.resolve("artifacts", "latest.png"); + +await fs.mkdir(path.dirname(output), { recursive: true }); +try { + const stamp = new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-"); + const previous = path.join(path.dirname(output), `history-${stamp}.png`); + await fs.copyFile(output, previous); +} catch (error) { + // Ignore missing-file errors on first capture. +} + +const browser = await chromium.launch({ headless: true }); +const page = await browser.newPage({ viewport: { width: 1600, height: 1200 } }); +await page.goto(url, { waitUntil: "domcontentloaded" }); +await page.waitForTimeout(1500); +await page.screenshot({ path: output, fullPage: true }); +await browser.close(); + +console.log(output); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 134abc9..f970fb5 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from "preact/hooks"; import { MetricChart } from "./chart"; -import type { SessionDetail, SessionSummary, SseEnvelope } from "./types"; +import type { IngestionStatus, SessionDetail, SessionSummary, SseEnvelope, TurnSummary, LogRecord, RuntimeSample } from "./types"; async function fetchJson(path: string): Promise { const response = await fetch(path); @@ -15,16 +15,44 @@ function pretty(value: unknown): string { return JSON.stringify(value ?? "Unavailable", null, 2); } +function readPreferredSessionId(): string | null { + if (typeof window === "undefined") { + return null; + } + const params = new URL(window.location.href).searchParams; + return params.get("session"); +} + +function writePreferredSessionId(sessionId: string): void { + if (typeof window === "undefined") { + return; + } + const url = new URL(window.location.href); + url.searchParams.set("session", sessionId); + window.history.replaceState({}, "", url); +} + export function App() { const [sessions, setSessions] = useState([]); - const [selectedSessionId, setSelectedSessionId] = useState(null); + const [selectedSessionId, setSelectedSessionId] = useState(() => readPreferredSessionId()); const [detail, setDetail] = useState(null); const [error, setError] = useState(null); + const [ingestion, setIngestion] = useState(null); async function refreshSessions() { const payload = await fetchJson<{ sessions: SessionSummary[] }>("/api/sessions"); setSessions(payload.sessions); - setSelectedSessionId((current) => current ?? payload.sessions[0]?.session_id ?? null); + setSelectedSessionId((current) => { + const available = new Set(payload.sessions.map((session) => session.session_id)); + const preferred = readPreferredSessionId(); + if (current && available.has(current)) { + return current; + } + if (preferred && available.has(preferred)) { + return preferred; + } + return payload.sessions[0]?.session_id ?? null; + }); } async function refreshDetail(sessionId: string) { @@ -34,6 +62,9 @@ export function App() { useEffect(() => { refreshSessions().catch((err: Error) => setError(err.message)); + fetchJson<{ ingestion: IngestionStatus | null }>("/api/status") + .then((payload) => setIngestion(payload.ingestion)) + .catch((err: Error) => setError(err.message)); }, []); useEffect(() => { @@ -46,19 +77,126 @@ export function App() { useEffect(() => { const source = new EventSource("/api/events"); source.onmessage = () => {}; - const reload = () => { - refreshSessions().catch((err: Error) => setError(err.message)); - if (selectedSessionId) { - refreshDetail(selectedSessionId).catch((err: Error) => setError(err.message)); + const applyEnvelope = (eventType: string, envelope: SseEnvelope) => { + setIngestion((current) => + current + ? { + ...current, + connected: true, + last_event_ts: envelope.ts, + last_event_id: envelope.event_id, + } + : current + ); + + setSessions((current) => { + const next = [...current]; + const index = next.findIndex((session) => session.session_id === envelope.session_id); + if (index === -1) { + next.unshift({ + session_id: envelope.session_id, + status: eventType === "session_finished" ? String(envelope.payload.status ?? "finished") : eventType === "error_reported" ? "error" : "running", + backend_name: envelope.backend_name, + resolved_model_id: envelope.resolved_model_id, + started_at: (envelope.payload.started_at as string | undefined) ?? null, + ended_at: (envelope.payload.ended_at as string | undefined) ?? null, + last_event_at: envelope.ts, + has_errors: eventType === "error_reported", + }); + return next; + } + const existing = next[index]; + next[index] = { + ...existing, + backend_name: envelope.backend_name ?? existing.backend_name, + resolved_model_id: envelope.resolved_model_id ?? existing.resolved_model_id, + last_event_at: envelope.ts, + status: + eventType === "session_finished" + ? String(envelope.payload.status ?? "finished") + : eventType === "error_reported" + ? "error" + : existing.status, + ended_at: + eventType === "session_finished" + ? (envelope.payload.ended_at as string | undefined) ?? existing.ended_at + : existing.ended_at, + has_errors: existing.has_errors || eventType === "error_reported", + }; + next.sort((a, b) => (b.last_event_at ?? "").localeCompare(a.last_event_at ?? "")); + return next; + }); + + if (selectedSessionId !== envelope.session_id) { + return; } + + setDetail((current) => { + if (!current || current.session.session_id !== envelope.session_id) { + return current; + } + const next: SessionDetail = { + ...current, + session: { + ...current.session, + last_event_at: envelope.ts, + }, + }; + if (eventType === "load_reported") { + next.load_report = { ...(current.load_report ?? {}), ...envelope.payload }; + next.inspect = { ...current.inspect, load_truth: next.load_report }; + } else if (eventType === "runtime_sample") { + const sample = envelope.payload as unknown as RuntimeSample; + next.latest_runtime = sample; + next.runtime_history = [...current.runtime_history, sample].slice(-180); + } else if (eventType === "turn_finished") { + const turn = envelope.payload as unknown as TurnSummary; + next.recent_turns = [turn, ...current.recent_turns.filter((item) => item.turn_id !== turn.turn_id)].slice(0, 20); + next.inspect = { ...current.inspect, generation_truth: turn.knobs ?? current.inspect.generation_truth }; + } else if (eventType === "log_recorded") { + const log = envelope.payload as unknown as LogRecord; + next.recent_logs = [...current.recent_logs, log].slice(-200); + } else if (eventType === "error_reported") { + next.session = { ...next.session, status: "error", has_errors: true }; + } else if (eventType === "session_finished") { + next.session = { + ...next.session, + status: String(envelope.payload.status ?? "finished"), + ended_at: (envelope.payload.ended_at as string | undefined) ?? next.session.ended_at, + }; + } else if (eventType === "session_started") { + next.session = { + ...next.session, + status: String(envelope.payload.status ?? "running"), + started_at: (envelope.payload.started_at as string | undefined) ?? next.session.started_at, + }; + } + return next; + }); + }; + + const bind = (eventType: string) => { + source.addEventListener(eventType, (evt) => { + try { + const envelope = JSON.parse((evt as MessageEvent).data) as SseEnvelope; + applyEnvelope(eventType, envelope); + } catch (err) { + setError("Failed to parse live event."); + } + }); }; - source.addEventListener("runtime_sample", reload); - source.addEventListener("turn_finished", reload); - source.addEventListener("session_started", reload); - source.addEventListener("session_finished", reload); - source.addEventListener("log_recorded", reload); - source.addEventListener("error_reported", reload); - source.addEventListener("connected", () => setError(null)); + + bind("runtime_sample"); + bind("turn_finished"); + bind("session_started"); + bind("session_finished"); + bind("log_recorded"); + bind("error_reported"); + bind("load_reported"); + source.addEventListener("connected", () => { + setError(null); + setIngestion((current) => (current ? { ...current, connected: true } : current)); + }); source.onerror = () => setError("Live event stream disconnected."); return () => { source.close(); @@ -70,6 +208,15 @@ export function App() { [selectedSessionId, sessions] ); + useEffect(() => { + if (selectedSessionId) { + writePreferredSessionId(selectedSessionId); + } + }, [selectedSessionId]); + + const latestTurn = detail?.recent_turns[0]; + const loadSummary = detail?.load_report; + return (
@@ -120,6 +267,25 @@ export function App() { {error ?

{error}

:

Live

}
+
+
+ Source + {ingestion ? `${ingestion.mode} · ${ingestion.connected ? "connected" : "waiting"}` : "Unavailable"} +
+
+ Path + {ingestion?.source ?? "Unavailable"} +
+
+ Last event + {ingestion?.last_event_ts ?? "None yet"} +
+
+ Parse errors + {ingestion?.parse_error_count ?? 0} +
+
+ {!detail ? (
Select a session to inspect it.
) : ( @@ -143,6 +309,36 @@ export function App() { +
+
+

Model summary

+
+
Engine
{String(loadSummary?.engine_name ?? "Unavailable")}
+
Dtype
{String(loadSummary?.model_dtype ?? "Unavailable")}
+
Context
{String(loadSummary?.context_length ?? "Unavailable")}
+
Quantization
{String(loadSummary?.quantization_type ?? "Unavailable")}
+
+
+
+

Generation summary

+
+
TTFT
{String(latestTurn?.time_to_first_token_seconds ?? "Unavailable")}
+
Request latency
{String(latestTurn?.request_latency_seconds ?? "Unavailable")}
+
Output tokens
{String(latestTurn?.completion_tokens ?? "Unavailable")}
+
Stop reason
{String(latestTurn?.stop_reason ?? "Unavailable")}
+
+
+
+

Runtime summary

+
+
GPU util
{String(detail.latest_runtime?.gpu?.utilization_percent ?? "Unavailable")}
+
VRAM used
{String(detail.latest_runtime?.gpu?.vram_used_bytes ?? "Unavailable")}
+
KV util
{String(detail.latest_runtime?.kv_cache?.utilization_ratio ?? "Unavailable")}
+
Live tok/s
{String(detail.latest_runtime?.throughput?.tokens_generated_per_second ?? "Unavailable")}
+
+
+
+

Load truth

- 
{pretty(detail.load_report)}
+
+ Raw JSON + 
{pretty(detail.load_report)}
+

Latest runtime

- 
{pretty(detail.latest_runtime)}
+
+ Raw JSON + 
{pretty(detail.latest_runtime)}
+

Recent turns

@@ -203,7 +405,10 @@ export function App() {

Inspect

- 
{pretty(detail.inspect)}
+
+ Raw inspect JSON + 
{pretty(detail.inspect)}
+
diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 5378805..0c77bd2 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -142,6 +142,18 @@ button { margin-bottom: 18px; } +.source-strip, +.summary-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 12px; + margin-bottom: 18px; +} + +.summary-grid { + grid-template-columns: repeat(3, minmax(0, 1fr)); +} + .metric-card, .subpanel, .chart-card { @@ -166,6 +178,52 @@ button { margin: 0; } +.source-item { + background: rgba(198, 90, 33, 0.08); + border: 1px solid var(--border); + border-radius: 14px; + padding: 12px; +} + +.source-path strong { + word-break: break-all; +} + +.label { + display: block; + color: var(--muted); + font-size: 0.8rem; + margin-bottom: 4px; +} + +.summary-list { + display: grid; + gap: 10px; + margin: 0; +} + +.summary-list div { + display: flex; + justify-content: space-between; + gap: 12px; + border-bottom: 1px solid var(--border); + padding-bottom: 8px; +} + +.summary-list div:last-child { + border-bottom: 0; + padding-bottom: 0; +} + +.summary-list dt { + color: var(--muted); +} + +.summary-list dd { + margin: 0; + text-align: right; +} + .chart-empty { color: var(--muted); padding: 24px 0 8px; @@ -209,12 +267,20 @@ pre { font-size: 0.84rem; } +details > summary { + cursor: pointer; + color: var(--muted); + margin-bottom: 10px; +} + @media (max-width: 1100px) { .layout { grid-template-columns: 1fr; } .metric-grid, + .source-strip, + .summary-grid, .chart-grid, .detail-grid { grid-template-columns: 1fr; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 10acdb9..7ffc09f 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -24,13 +24,33 @@ export type RuntimeSample = { }; }; +export type TurnSummary = { + turn_id?: number; + started_at?: string; + ended_at?: string; + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + request_latency_seconds?: number; + time_to_first_token_seconds?: number; + stop_reason?: string; + knobs?: Record; +}; + +export type LogRecord = { + session_id?: string; + ts?: string; + source?: string; + message?: string; +}; + export type SessionDetail = { session: SessionSummary; load_report: Record | null; latest_runtime: RuntimeSample | null; runtime_history: RuntimeSample[]; - recent_turns: Array>; - recent_logs: Array>; + recent_turns: TurnSummary[]; + recent_logs: LogRecord[]; inspect: Record; }; @@ -44,3 +64,13 @@ export type SseEnvelope = { experiment_id: string | null; payload: Record; }; + +export type IngestionStatus = { + mode: string; + source: string; + connected: boolean; + last_event_ts: string | null; + last_event_id: string | null; + parse_error_count: number; + last_error: string | null; +}; diff --git a/scripts/audit-dashboard.sh b/scripts/audit-dashboard.sh new file mode 100755 index 0000000..772fc41 --- /dev/null +++ b/scripts/audit-dashboard.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FRONTEND_DIR="$ROOT_DIR/frontend" +ARTIFACT_DIR="$FRONTEND_DIR/artifacts" +URL="http://127.0.0.1:8001/" +API_BASE="http://127.0.0.1:8001" +STATUS_URL="$API_BASE/api/status" +SESSIONS_URL="$API_BASE/api/sessions" +START_BACKEND=0 +TELEMETRY_JSONL="" +FIXTURES_PATH="" +REPLAY_FIXTURES=1 +REPLAY_SPEED="" +BUILD_FRONTEND=0 +PREFERRED_SESSION_ID="" +PREFERRED_SESSION_SOURCE="default" + +usage() { + cat <<'EOF' +Usage: + ./scripts/audit-dashboard.sh [options] + +Options: + --url Dashboard URL. Default: http://127.0.0.1:8001/ + --api-base API base. Default: http://127.0.0.1:8001 + --start-backend Start the backend automatically if it is not reachable. + --telemetry-jsonl Telemetry JSONL source to pass to dev-backend.sh when starting. + --fixtures Fixture JSONL source to pass to dev-backend.sh when starting. + --no-replay-fixtures Disable fixture replay when starting backend. + --replay-speed Fixture replay speed when starting backend. + --build-frontend Build frontend before audit. + --session-id Prefer a specific session id for API detail capture and screenshot selection. + -h, --help Show this help. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --url) + URL="$2" + shift 2 + ;; + --api-base) + API_BASE="$2" + shift 2 + ;; + --start-backend) + START_BACKEND=1 + shift + ;; + --telemetry-jsonl) + TELEMETRY_JSONL="$2" + shift 2 + ;; + --fixtures) + FIXTURES_PATH="$2" + shift 2 + ;; + --no-replay-fixtures) + REPLAY_FIXTURES=0 + shift + ;; + --replay-speed) + REPLAY_SPEED="$2" + shift 2 + ;; + --build-frontend) + BUILD_FRONTEND=1 + shift + ;; + --session-id) + PREFERRED_SESSION_ID="$2" + PREFERRED_SESSION_SOURCE="arg" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +STATUS_URL="$API_BASE/api/status" +SESSIONS_URL="$API_BASE/api/sessions" +mkdir -p "$ARTIFACT_DIR" + +timestamp="$(date -u +"%Y%m%dT%H%M%SZ")" +status_out="$ARTIFACT_DIR/api-status-$timestamp.json" +sessions_out="$ARTIFACT_DIR/api-sessions-$timestamp.json" +detail_out="$ARTIFACT_DIR/api-session-detail-$timestamp.json" +meta_out="$ARTIFACT_DIR/audit-meta-$timestamp.txt" +screenshot_out="$ARTIFACT_DIR/latest.png" + +backend_pid="" + +health_ok() { + curl -fsS "$STATUS_URL" >/dev/null 2>&1 || curl -fsS "$SESSIONS_URL" >/dev/null 2>&1 +} + +detect_verified_session_id() { + python - "$ROOT_DIR/docs/coordination/inbox/response.md" <<'PY' +import re +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +if not path.exists(): + print("") + raise SystemExit(0) + +text = path.read_text(encoding="utf-8") +match = re.search(r"Verified session id:\s*\n\s*-\s*`([^`]+)`", text) +print(match.group(1) if match else "") +PY +} + +append_session_query() { + python - "$1" "$2" <<'PY' +import sys +from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl + +url = sys.argv[1] +session_id = sys.argv[2] +parts = urlparse(url) +query = dict(parse_qsl(parts.query, keep_blank_values=True)) +query["session"] = session_id +print(urlunparse(parts._replace(query=urlencode(query)))) +PY +} + +cleanup() { + if [[ -n "$backend_pid" ]]; then + kill "$backend_pid" >/dev/null 2>&1 || true + wait "$backend_pid" >/dev/null 2>&1 || true + fi +} +trap cleanup EXIT + +if [[ "$BUILD_FRONTEND" -eq 1 ]]; then + ( + cd "$FRONTEND_DIR" + npm run build + ) +fi + +if ! health_ok; then + if [[ "$START_BACKEND" -ne 1 ]]; then + echo "Dashboard backend is not reachable at $STATUS_URL or $SESSIONS_URL" >&2 + echo "Use --start-backend to launch it from this script." >&2 + exit 1 + fi + + backend_args=() + if [[ -n "$TELEMETRY_JSONL" ]]; then + backend_args+=(--telemetry-jsonl "$TELEMETRY_JSONL") + elif [[ -n "$FIXTURES_PATH" ]]; then + backend_args+=(--fixtures "$FIXTURES_PATH") + if [[ "$REPLAY_FIXTURES" -eq 0 ]]; then + backend_args+=(--no-replay-fixtures) + fi + if [[ -n "$REPLAY_SPEED" ]]; then + backend_args+=(--replay-speed "$REPLAY_SPEED") + fi + fi + + ( + cd "$ROOT_DIR" + ./scripts/dev-backend.sh "${backend_args[@]}" + ) >/tmp/lab-llm-audit-backend.log 2>&1 & + backend_pid="$!" + + for _ in $(seq 1 30); do + if health_ok; then + break + fi + sleep 1 + done + + if ! health_ok; then + echo "Failed to start dashboard backend." >&2 + echo "See /tmp/lab-llm-audit-backend.log" >&2 + exit 1 + fi +fi + +if curl -fsS "$STATUS_URL" > "$status_out" 2>/dev/null; then + : +else + printf '{ "ingestion": null, "note": "status endpoint unavailable on audited backend" }\n' > "$status_out" +fi +curl -fsS "$SESSIONS_URL" > "$sessions_out" + +if [[ -z "$PREFERRED_SESSION_ID" ]]; then + detected_session_id="$(detect_verified_session_id)" + if [[ -n "$detected_session_id" ]]; then + PREFERRED_SESSION_ID="$detected_session_id" + PREFERRED_SESSION_SOURCE="response" + fi +fi + +session_id="$( + python - "$sessions_out" "$PREFERRED_SESSION_ID" <<'PY' +import json +import sys +from pathlib import Path + +obj = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +sessions = obj.get("sessions") or [] +preferred = sys.argv[2] +if preferred and any(session["session_id"] == preferred for session in sessions): + print(preferred) +else: + running = [session for session in sessions if session.get("status") == "running"] + if len(running) == 1: + print(running[0]["session_id"]) + elif running: + running.sort(key=lambda session: session.get("last_event_at") or session.get("started_at") or "", reverse=True) + print(running[0]["session_id"]) + else: + print(sessions[0]["session_id"] if sessions else "") +PY +)" + +if [[ -n "$session_id" ]]; then + curl -fsS "$API_BASE/api/sessions/$session_id" > "$detail_out" +else + printf '{}' > "$detail_out" +fi + +screenshot_url="$URL" +if [[ -n "$session_id" ]]; then + screenshot_url="$(append_session_query "$URL" "$session_id")" +fi + +( + cd "$FRONTEND_DIR" + npm run screenshot -- "$screenshot_url" "$screenshot_out" +) + +{ + echo "timestamp=$timestamp" + echo "url=$screenshot_url" + echo "api_base=$API_BASE" + echo "status_path=$status_out" + echo "sessions_path=$sessions_out" + echo "detail_path=$detail_out" + echo "screenshot_path=$screenshot_out" + if [[ -n "$backend_pid" ]]; then + echo "backend_started_by_script=1" + else + echo "backend_started_by_script=0" + fi + if [[ -n "$session_id" ]]; then + echo "session_id=$session_id" + fi + echo "session_selection_source=$PREFERRED_SESSION_SOURCE" +} > "$meta_out" + +echo "$meta_out" diff --git a/scripts/coordination_config.py b/scripts/coordination_config.py new file mode 100644 index 0000000..e0b3127 --- /dev/null +++ b/scripts/coordination_config.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import tomllib +from pathlib import Path + +DEFAULTS_PATH = Path("docs/coordination.defaults.toml") +EXAMPLE_DEFAULTS_PATH = Path("docs/coordination.defaults.example.toml") + + +def resolve_default_config_path() -> Path: + if DEFAULTS_PATH.exists(): + return DEFAULTS_PATH + return EXAMPLE_DEFAULTS_PATH + + +def main() -> None: + parser = argparse.ArgumentParser(description="Read a coordination config value.") + parser.add_argument("key", help="Top-level TOML key to print.") + parser.add_argument( + "--config", + default=str(resolve_default_config_path()), + help="Path to the TOML config file.", + ) + args = parser.parse_args() + + data = tomllib.loads(Path(args.config).read_text(encoding="utf-8")) + value = data.get(args.key) + if value is None: + raise SystemExit(f"missing config key: {args.key}") + print(value) + + +if __name__ == "__main__": + main() diff --git a/scripts/coordination_state.py b/scripts/coordination_state.py new file mode 100644 index 0000000..05fb471 --- /dev/null +++ b/scripts/coordination_state.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +STATE_PATH = Path("docs/coordination/state.json") +HEADING_RE = re.compile(r"^## (.+)$", re.MULTILINE) +GAP_ID_RE = re.compile(r"\bGAP-\d+\b") +VERIFIED_SESSION_RE = re.compile(r"Verified session id:\s*\n\s*-\s*`([^`]+)`") + + +def default_state() -> dict[str, Any]: + return { + "selected_gap_ids": [], + "previous_selected_gap_ids": [], + "attempt_counts": {}, + "blocked_counts": {}, + "addressed_counts": {}, + "last_attempted_gap_ids": [], + "last_blocked_gap_ids": [], + "last_addressed_gap_ids": [], + "last_response_addressed_gap_ids": [], + "last_response_path": None, + "last_reconciled_at": None, + "previous_verified_session_id": None, + "last_verified_session_id": None, + "last_audited_session_id": None, + "last_audit_detail_hash": None, + "last_session_selection_source": None, + "last_loop_stop_reason": None, + } + + +def load_state(path: Path) -> dict[str, Any]: + if not path.exists(): + return default_state() + state = default_state() + state.update(json.loads(path.read_text(encoding="utf-8"))) + return state + + +def save_state(path: Path, state: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def parse_slice_gap_ids(text: str) -> list[str]: + seen: list[str] = [] + for gap_id in GAP_ID_RE.findall(text): + if gap_id not in seen: + seen.append(gap_id) + return seen + + +def parse_sections(text: str) -> dict[str, str]: + matches = list(HEADING_RE.finditer(text)) + sections: dict[str, str] = {} + for index, match in enumerate(matches): + start = match.end() + end = matches[index + 1].start() if index + 1 < len(matches) else len(text) + sections[match.group(1).strip().lower()] = text[start:end].strip() + return sections + + +def ids_from_sections(sections: dict[str, str], names: tuple[str, ...]) -> list[str]: + seen: list[str] = [] + for name in names: + body = sections.get(name, "") + for gap_id in GAP_ID_RE.findall(body): + if gap_id not in seen: + seen.append(gap_id) + return seen + + +def extract_verified_session_id(text: str) -> str | None: + match = VERIFIED_SESSION_RE.search(text) + return match.group(1) if match else None + + +def record_selection(state_path: Path, slice_path: Path) -> None: + state = load_state(state_path) + gap_ids = parse_slice_gap_ids(slice_path.read_text(encoding="utf-8")) + attempt_counts = defaultdict(int, state.get("attempt_counts", {})) + for gap_id in gap_ids: + attempt_counts[gap_id] += 1 + state["previous_selected_gap_ids"] = state.get("selected_gap_ids", []) + state["selected_gap_ids"] = gap_ids + state["last_attempted_gap_ids"] = gap_ids + state["attempt_counts"] = dict(attempt_counts) + state["last_reconciled_at"] = now_iso() + save_state(state_path, state) + + +def record_response(state_path: Path, response_path: Path) -> None: + state = load_state(state_path) + text = response_path.read_text(encoding="utf-8") + sections = parse_sections(text) + addressed = ids_from_sections(sections, ("addressed gaps",)) + attempted = ids_from_sections( + sections, + ( + "addressed gaps", + "gaps attempted but not closed", + "attempted gaps", + "remaining blockers", + ), + ) + blocked = ids_from_sections(sections, ("gaps blocked", "remaining blockers")) + + blocked_counts = defaultdict(int, state.get("blocked_counts", {})) + addressed_counts = defaultdict(int, state.get("addressed_counts", {})) + for gap_id in blocked: + blocked_counts[gap_id] += 1 + for gap_id in addressed: + addressed_counts[gap_id] += 1 + + state["last_attempted_gap_ids"] = attempted + state["last_blocked_gap_ids"] = blocked + state["last_addressed_gap_ids"] = addressed + state["last_response_addressed_gap_ids"] = addressed + state["blocked_counts"] = dict(blocked_counts) + state["addressed_counts"] = dict(addressed_counts) + state["last_response_path"] = str(response_path) + state["previous_verified_session_id"] = state.get("last_verified_session_id") + state["last_verified_session_id"] = extract_verified_session_id(text) + state["last_reconciled_at"] = now_iso() + save_state(state_path, state) + + +def record_audit(state_path: Path, meta_path: Path) -> None: + state = load_state(state_path) + meta: dict[str, str] = {} + for line in meta_path.read_text(encoding="utf-8").splitlines(): + if "=" in line: + key, value = line.split("=", 1) + meta[key] = value + + detail_hash = None + detail_path = meta.get("detail_path") + if detail_path and Path(detail_path).exists(): + detail_hash = hashlib.sha256(Path(detail_path).read_bytes()).hexdigest() + + state["last_audited_session_id"] = meta.get("session_id") + state["last_audit_detail_hash"] = detail_hash + state["last_session_selection_source"] = meta.get("session_selection_source") + state["last_reconciled_at"] = now_iso() + save_state(state_path, state) + + +def check_loop_stop(state_path: Path) -> None: + state = load_state(state_path) + selected = state.get("selected_gap_ids", []) + previous = state.get("previous_selected_gap_ids", []) + addressed = state.get("last_response_addressed_gap_ids", []) + previous_verified = state.get("previous_verified_session_id") + verified = state.get("last_verified_session_id") + audited = state.get("last_audited_session_id") + + reason: str | None = None + if addressed and not verified: + reason = "missing-verified-session-id" + elif addressed and previous_verified and verified == previous_verified: + reason = f"verified-session-not-fresh:{verified}" + elif verified and audited and verified != audited: + reason = f"audit-session-mismatch:{audited}:expected:{verified}" + elif selected and previous and selected == previous and not addressed: + reason = "no-progress-same-slice-reselected" + + state["last_loop_stop_reason"] = reason + save_state(state_path, state) + if reason: + print(reason) + raise SystemExit(10) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Maintain coordination loop state.") + parser.add_argument("--state", default=str(STATE_PATH), help="Path to the state JSON file.") + subparsers = parser.add_subparsers(dest="command", required=True) + + selection_parser = subparsers.add_parser("record-selection", help="Record a selected gap slice.") + selection_parser.add_argument("--slice", default="docs/coordination/current_gap_slice.md") + + response_parser = subparsers.add_parser("record-response", help="Record an upstream response.") + response_parser.add_argument("--response", default="docs/coordination/inbox/response.md") + + audit_parser = subparsers.add_parser("record-audit", help="Record the latest audit metadata.") + audit_parser.add_argument("--meta", required=True) + + subparsers.add_parser("check-loop-stop", help="Exit non-zero if the loop should stop.") + + args = parser.parse_args() + state_path = Path(args.state) + + if args.command == "record-selection": + record_selection(state_path, Path(args.slice)) + elif args.command == "record-response": + record_response(state_path, Path(args.response)) + elif args.command == "record-audit": + record_audit(state_path, Path(args.meta)) + elif args.command == "check-loop-stop": + check_loop_stop(state_path) + else: + raise SystemExit(f"unknown command: {args.command}") + + +if __name__ == "__main__": + main() diff --git a/scripts/gap_slice.py b/scripts/gap_slice.py new file mode 100755 index 0000000..55c901e --- /dev/null +++ b/scripts/gap_slice.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class Gap: + gap_id: str + title: str + body: str + upstream_owner: str + status: str + current_behavior: str + telemetry_gap: str + dashboard_gap: str + + +GAP_RE = re.compile(r"^### (GAP-\d+): (.+)$", re.MULTILINE) +GAP_ID_RE = re.compile(r"\bGAP-\d+\b") +FIELD_RE = re.compile(r"^- `([^`]+)`: (.+)$", re.MULTILINE) + + +def parse_gaps(text: str) -> list[Gap]: + matches = list(GAP_RE.finditer(text)) + gaps: list[Gap] = [] + for index, match in enumerate(matches): + start = match.end() + end = matches[index + 1].start() if index + 1 < len(matches) else len(text) + body = text[start:end].strip() + fields = {field: normalize(value) for field, value in FIELD_RE.findall(body)} + gaps.append( + Gap( + gap_id=match.group(1), + title=match.group(2), + body=body, + upstream_owner=fields.get("Upstream owner", ""), + status=fields.get("Status", ""), + current_behavior=fields.get("Current behavior", ""), + telemetry_gap=fields.get("Telemetry gap", ""), + dashboard_gap=fields.get("Dashboard gap", ""), + ) + ) + return gaps + + +def normalize(value: str) -> str: + return value.strip().strip("`").strip() + + +def render_slice(gaps: list[Gap], count: int, focus_ids: list[str]) -> str: + eligible = [ + gap + for gap in gaps + if gap.status in {"open", "partial"} and gap.upstream_owner in {"model-runner", "shared"} + ] + focused = [gap for gap in eligible if gap.gap_id in focus_ids] + unfocused = [gap for gap in eligible if gap.gap_id not in focus_ids] + selected = sorted(focused, key=priority_key)[:count] + if len(selected) < count: + selected.extend(sorted(unfocused, key=priority_key)[: count - len(selected)]) + + lines = [ + "# Current Gap Slice", + "", + "This file is generated from `docs/gap_ledger.md`.", + "It is the current upstream-facing work slice for `model-runner`.", + "", + "## Focus", + ] + + if focus_ids: + lines.extend( + [ + "- Focus mode is active.", + f"- Focused gap ids: {', '.join(f'`{gap_id}`' for gap_id in focus_ids)}", + "- Eligible focused gaps are selected before the normal priority ranking.", + ] + ) + else: + lines.extend( + [ + "- No explicit focus ids.", + "- Normal priority ranking is in effect.", + ] + ) + + lines.extend( + [ + "", + "## Working rules", + "- Fix these gaps in upstream telemetry truth or semantics before adding unrelated dashboard-facing changes.", + "- If a gap cannot be fully closed, document the remaining limitation in the response file.", + "- Preserve canonical event truth; do not normalize requested config into runtime fact.", + "", + f"## Selected gaps ({len(selected)})", + ] + ) + + if not selected: + lines.append("- No eligible upstream gaps found.") + return "\n".join(lines) + "\n" + + for gap in selected: + lines.extend( + [ + "", + f"### {gap.gap_id}: {gap.title}", + gap.body, + ] + ) + + lines.extend( + [ + "", + "## Expected response", + "Write the upstream response in `docs/downstream/lab_llm/response.md` in `model-runner`.", + "Include:", + "- which gaps were addressed", + "- which gaps were attempted but not closed", + "- which gaps are blocked", + "- what changed in telemetry/schema/emission", + "- the exact verification run that exercised the telemetry change", + "- any new fixture or sink evidence paths", + ] + ) + return "\n".join(lines) + "\n" + + +def priority_key(gap: Gap) -> tuple[int, int, int, int, int, int, str]: + state = load_state() + owner_score = 0 if gap.upstream_owner == "model-runner" else 1 + + evidence_score = 0 + evidence_text = " ".join([gap.current_behavior, gap.telemetry_gap]).lower() + if any(token in evidence_text for token in ["screenshot", "raw runtime", "raw data", "audited"]): + evidence_score = -1 + + truth_score = 1 + truth_text = " ".join([gap.title, gap.telemetry_gap, gap.current_behavior]).lower() + if any( + token in truth_text + for token in [ + "model", + "path", + "source", + "telemetry", + "runtime truth", + "configuration", + "throughput", + "tok/s", + "gpu", + "latency", + "request", + ] + ): + truth_score = 0 + + blocked_score = state.get("blocked_counts", {}).get(gap.gap_id, 0) + addressed_score = state.get("addressed_counts", {}).get(gap.gap_id, 0) + recent_attempt_score = 0 + if gap.gap_id in state.get("last_attempted_gap_ids", []): + recent_attempt_score += 2 + recent_attempt_score += min(state.get("attempt_counts", {}).get(gap.gap_id, 0), 3) + + return ( + owner_score, + blocked_score, + addressed_score, + evidence_score, + truth_score, + recent_attempt_score, + gap.gap_id, + ) + + +def load_state() -> dict[str, object]: + path = Path("docs/coordination/state.json") + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def load_focus_ids(path: Path) -> list[str]: + if not path.exists(): + return [] + + seen: list[str] = [] + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + match = GAP_ID_RE.search(line) + if not match: + continue + gap_id = match.group(0) + if gap_id not in seen: + seen.append(gap_id) + return seen + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate the current upstream gap slice.") + parser.add_argument( + "--gap-ledger", + default="docs/gap_ledger.md", + help="Path to the gap ledger markdown file.", + ) + parser.add_argument( + "--count", + type=int, + default=5, + help="Maximum number of upstream gaps to include.", + ) + parser.add_argument( + "--output", + default="docs/coordination/current_gap_slice.md", + help="Output markdown path.", + ) + parser.add_argument( + "--focus", + default="docs/coordination/focus.txt", + help="Optional focus file listing gap ids to prioritize.", + ) + args = parser.parse_args() + + text = Path(args.gap_ledger).read_text(encoding="utf-8") + focus_ids = load_focus_ids(Path(args.focus)) + rendered = render_slice(parse_gaps(text), args.count, focus_ids) + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(rendered, encoding="utf-8") + print(output_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/pull-model-runner-response.sh b/scripts/pull-model-runner-response.sh new file mode 100755 index 0000000..20480f4 --- /dev/null +++ b/scripts/pull-model-runner-response.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODEL_RUNNER_ROOT="${MODEL_RUNNER_ROOT:-$(python "$ROOT_DIR/scripts/coordination_config.py" model_runner_root)}" +SRC_DIR="$MODEL_RUNNER_ROOT/docs/downstream/lab_llm" +DEST_DIR="$ROOT_DIR/docs/coordination/inbox" + +mkdir -p "$DEST_DIR" + +for file in response.md current_gap_slice.md README.md; do + if [[ -f "$SRC_DIR/$file" ]]; then + cp "$SRC_DIR/$file" "$DEST_DIR/" + fi +done + +if [[ -d "$SRC_DIR/artifacts" ]]; then + mkdir -p "$DEST_DIR/artifacts" + cp -f "$SRC_DIR"/artifacts/* "$DEST_DIR/artifacts/" 2>/dev/null || true +fi + +echo "$DEST_DIR" diff --git a/scripts/push-gap-slice.sh b/scripts/push-gap-slice.sh new file mode 100755 index 0000000..ab1d089 --- /dev/null +++ b/scripts/push-gap-slice.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODEL_RUNNER_ROOT="${MODEL_RUNNER_ROOT:-$(python "$ROOT_DIR/scripts/coordination_config.py" model_runner_root)}" +DEST_DIR="$MODEL_RUNNER_ROOT/docs/downstream/lab_llm" +COUNT="${1:-$(python "$ROOT_DIR/scripts/coordination_config.py" gap_slice_count)}" + +python "$ROOT_DIR/scripts/gap_slice.py" --count "$COUNT" >/tmp/lab-llm-gap-slice-path.txt +SLICE_PATH="$(cat /tmp/lab-llm-gap-slice-path.txt)" +python "$ROOT_DIR/scripts/verification_target.py" --slice "$ROOT_DIR/$SLICE_PATH" >/tmp/lab-llm-verification-target.txt +VERIFY_PATH="$(cat /tmp/lab-llm-verification-target.txt)" +python "$ROOT_DIR/scripts/coordination_state.py" record-selection --slice "$ROOT_DIR/$SLICE_PATH" + +mkdir -p "$DEST_DIR/artifacts" + +cp "$ROOT_DIR/docs/gap_ledger.md" "$DEST_DIR/gap_ledger.md" +cp "$ROOT_DIR/docs/operator_questions.md" "$DEST_DIR/operator_questions.md" +cp "$ROOT_DIR/docs/current_state_audit.md" "$DEST_DIR/current_state_audit.md" +cp "$ROOT_DIR/$SLICE_PATH" "$DEST_DIR/current_gap_slice.md" +cp "$ROOT_DIR/$VERIFY_PATH" "$DEST_DIR/verification_target.md" + +if [[ -f "$ROOT_DIR/frontend/artifacts/latest.png" ]]; then + cp "$ROOT_DIR/frontend/artifacts/latest.png" "$DEST_DIR/artifacts/latest.png" +fi + +latest_meta="$(ls -1t "$ROOT_DIR"/frontend/artifacts/audit-meta-*.txt 2>/dev/null | head -n 1 || true)" +if [[ -n "$latest_meta" ]]; then + cp "$latest_meta" "$DEST_DIR/artifacts/" + while IFS='=' read -r key value; do + case "$key" in + status_path|sessions_path|detail_path|screenshot_path) + if [[ -f "$value" ]]; then + cp "$value" "$DEST_DIR/artifacts/" + fi + ;; + esac + done <"$latest_meta" +fi + +cat >"$DEST_DIR/README.md" <<'EOF' +# lab-llm downstream bundle + +Source of truth lives in `/home/poop/projects/lab-llm`. + +Files here are mirrored for upstream telemetry work in `model-runner`. +Do not edit the mirrored gap ledger here; update the source repo instead. + +Use: +- `current_gap_slice.md` for the current 3-5 gap work slice +- `verification_target.md` for the default telemetry-producing verification target +- `response.md` to report what upstream changed +EOF + +if [[ ! -f "$DEST_DIR/response.md" ]]; then + cat >"$DEST_DIR/response.md" <<'EOF' +# Upstream Response + +Date: + +## Addressed gaps +- + +## Gaps attempted but not closed +- + +## Gaps blocked +- + +## Telemetry changes +- + +## Verification run +- + +## Evidence +- +EOF +fi + +echo "$DEST_DIR" diff --git a/scripts/reconcile-gap-ledger.sh b/scripts/reconcile-gap-ledger.sh new file mode 100755 index 0000000..296c4cb --- /dev/null +++ b/scripts/reconcile-gap-ledger.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TELEMETRY_JSONL="${TELEMETRY_JSONL:-$(python "$ROOT_DIR/scripts/coordination_config.py" telemetry_jsonl)}" +API_BASE="${API_BASE:-$(python "$ROOT_DIR/scripts/coordination_config.py" api_base)}" +URL="${URL:-$(python "$ROOT_DIR/scripts/coordination_config.py" dashboard_url)}" +COUNT="${COUNT:-$(python "$ROOT_DIR/scripts/coordination_config.py" gap_slice_count)}" + +./scripts/pull-model-runner-response.sh >/tmp/lab-llm-pull-dest.txt + +if [[ -f "$ROOT_DIR/docs/coordination/inbox/response.md" ]]; then + python "$ROOT_DIR/scripts/coordination_state.py" record-response --response "$ROOT_DIR/docs/coordination/inbox/response.md" +fi + +pre_meta_path="$(./scripts/audit-dashboard.sh --api-base "$API_BASE" --url "$URL")" +python "$ROOT_DIR/scripts/coordination_state.py" record-audit --meta "$pre_meta_path" + +cat <<'EOF' | codex exec --dangerously-bypass-approvals-and-sandbox -C "$ROOT_DIR" - +Read: +- docs/gap_ledger.md +- docs/current_state_audit.md +- docs/operator_questions.md +- docs/coordination/inbox/response.md if present +- docs/coordination/inbox/current_gap_slice.md if present +- docs/coordination/state.json +- the newest files under frontend/artifacts/ + +Tasks: +1. Only if the latest audit proves that upstream truth is available but hidden by `lab-llm` read-model/API/UI behavior, implement the minimal local change needed to expose that truth. +2. Update docs/gap_ledger.md based on the latest audit artifacts and upstream response. +3. Close or downgrade only the gaps that are genuinely improved by the new telemetry/dashboard evidence. +4. Add any new obvious operator-usefulness gaps revealed by the audit. +5. Update docs/current_state_audit.md to reflect the latest state. +6. Do not close a gap solely because upstream claims it is fixed; require matching audit evidence or make the gap partial/pending-verification instead. + +Rules: +- be strict about telemetry truth +- do not close a gap just because upstream said it worked +- rely on audit evidence and current rendered behavior +- keep the gap ledger as the source of truth +- do not make speculative `lab-llm` code changes just to "help the loop complete" +- only touch `lab-llm` code when the audit shows a real local wiring/projection/adoption problem +EOF + +post_meta_path="$(./scripts/audit-dashboard.sh --api-base "$API_BASE" --url "$URL")" +python "$ROOT_DIR/scripts/coordination_state.py" record-audit --meta "$post_meta_path" + +python "$ROOT_DIR/scripts/gap_slice.py" --count "$COUNT" +python "$ROOT_DIR/scripts/verification_target.py" --slice "$ROOT_DIR/docs/coordination/current_gap_slice.md" diff --git a/scripts/run-gap-loop.sh b/scripts/run-gap-loop.sh new file mode 100755 index 0000000..bbad37a --- /dev/null +++ b/scripts/run-gap-loop.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ITERATIONS="${1:-$(python "$ROOT_DIR/scripts/coordination_config.py" loop_iterations)}" + +for i in $(seq 1 "$ITERATIONS"); do + echo "== gap loop iteration $i ==" + ./scripts/run-model-runner-gap-pass.sh + ./scripts/reconcile-gap-ledger.sh + + if grep -q "No eligible upstream gaps found" "$ROOT_DIR/docs/coordination/current_gap_slice.md"; then + echo "No eligible upstream gaps remain." + break + fi + + if python "$ROOT_DIR/scripts/coordination_state.py" check-loop-stop; then + : + else + echo "Loop stopped by guardrail." + break + fi +done + +echo "Gap loop complete." diff --git a/scripts/run-model-runner-gap-pass.sh b/scripts/run-model-runner-gap-pass.sh new file mode 100755 index 0000000..414f233 --- /dev/null +++ b/scripts/run-model-runner-gap-pass.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODEL_RUNNER_ROOT="${MODEL_RUNNER_ROOT:-$(python "$ROOT_DIR/scripts/coordination_config.py" model_runner_root)}" +COUNT="${1:-$(python "$ROOT_DIR/scripts/coordination_config.py" gap_slice_count)}" + +./scripts/push-gap-slice.sh "$COUNT" >/tmp/lab-llm-push-dest.txt +DEST_DIR="$(cat /tmp/lab-llm-push-dest.txt)" + +cat < dict[str, str | int]: + import tomllib + + return tomllib.loads(path.read_text(encoding="utf-8")) + + +def infer_from_jsonl_path(path: Path) -> tuple[str, str, str]: + parts = path.parts + backend = "unknown" + model_name = path.stem + model_path = str(path) + if "models" in parts: + index = parts.index("models") + if index + 1 < len(parts): + model_name = parts[index + 1] + model_path = str(Path(*parts[: index + 2])) + if index + 2 < len(parts): + backend = parts[index + 2] + return model_name, backend, model_path + + +def infer_backend_override(slice_text: str, fallback: str) -> str: + lowered = slice_text.lower() + for backend in ("vllm", "sglang", "trtllm", "llama.cpp", "hf"): + if backend in lowered: + return backend + return fallback + + +def render_target(model_name: str, backend: str, model_path: str, telemetry_jsonl: str) -> str: + return "\n".join( + [ + "# Telemetry Verification Target", + "", + "Use this as the default verification target in `model-runner` when a gap slice changes telemetry behavior.", + "", + "## Default target", + f"- `Model label`: `{model_name}`", + f"- `Backend`: `{backend}`", + f"- `Model path`: `{model_path}`", + f"- `Telemetry sink`: `{telemetry_jsonl}`", + "", + "## Verification rule", + "- If you changed telemetry schema, emission semantics, backend adapters, load/runtime/turn fields, or sink wiring, run a real model load plus at least one short generation before claiming success.", + "- If the change affects model identity, startup, load-time truth, or runtime semantics, do not verify against a stale already-running process.", + "- If the runnable path depends on a build/install step, rebuild from the latest source before starting the fresh verification run.", + "- Capture a new session id from the fresh run; do not reuse a session produced before the code change.", + "- Prefer the backend above unless the selected gaps are explicitly about a different backend and you can justify a closer target.", + "- Record the exact fresh-code step, exact run command, backend, model, session id, and evidence path in `response.md`.", + "- If no existing script cleanly supports the target, choose the narrowest existing runner/dev command that emits telemetry to the sink and explain the compromise.", + ] + ) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate a telemetry verification target note.") + parser.add_argument("--config", default=str(DEFAULTS_PATH), help="Path to the TOML defaults file.") + parser.add_argument("--slice", default="docs/coordination/current_gap_slice.md", help="Current gap slice path.") + parser.add_argument("--output", default="docs/coordination/verification_target.md", help="Output markdown path.") + args = parser.parse_args() + + config = read_config(Path(args.config)) + telemetry_jsonl = str(config["telemetry_jsonl"]) + slice_text = Path(args.slice).read_text(encoding="utf-8") if Path(args.slice).exists() else "" + model_name, backend, model_path = infer_from_jsonl_path(Path(telemetry_jsonl)) + backend = infer_backend_override(slice_text, backend) + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + render_target(model_name=model_name, backend=backend, model_path=model_path, telemetry_jsonl=telemetry_jsonl), + encoding="utf-8", + ) + print(output) + + +if __name__ == "__main__": + main() diff --git a/src/lab_llm/live.py b/src/lab_llm/live.py index 778581a..7c3a58f 100644 --- a/src/lab_llm/live.py +++ b/src/lab_llm/live.py @@ -4,6 +4,7 @@ import queue import threading import time +from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Iterable @@ -17,6 +18,51 @@ def _parse_ts(value: str) -> datetime: return datetime.fromisoformat(value.replace("Z", "+00:00")) +@dataclass +class IngestionStatus: + mode: str + source: str + connected: bool = False + last_event_ts: str | None = None + last_event_id: str | None = None + parse_error_count: int = 0 + last_error: str | None = None + + +class IngestionMonitor: + def __init__(self, *, mode: str, source: str) -> None: + self._status = IngestionStatus(mode=mode, source=source) + self._lock = threading.Lock() + + def mark_connected(self, connected: bool) -> None: + with self._lock: + self._status.connected = connected + + def record_event(self, event: TelemetryEvent) -> None: + with self._lock: + self._status.connected = True + self._status.last_event_ts = event.ts + self._status.last_event_id = event.event_id + self._status.last_error = None + + def record_error(self, message: str) -> None: + with self._lock: + self._status.last_error = message + self._status.parse_error_count += 1 + + def snapshot(self) -> dict[str, str | int | bool | None]: + with self._lock: + return { + "mode": self._status.mode, + "source": self._status.source, + "connected": self._status.connected, + "last_event_ts": self._status.last_event_ts, + "last_event_id": self._status.last_event_id, + "parse_error_count": self._status.parse_error_count, + "last_error": self._status.last_error, + } + + class EventBroadcaster: def __init__(self) -> None: self._queues: set[queue.Queue[TelemetryEvent | None]] = set() @@ -61,12 +107,14 @@ def __init__( events: Iterable[TelemetryEvent], store: DashboardStore, broadcaster: EventBroadcaster, + monitor: IngestionMonitor | None = None, *, speed: float = 20.0, ) -> None: self._events = list(events) self._store = store self._broadcaster = broadcaster + self._monitor = monitor self._speed = max(speed, 0.1) self._thread: threading.Thread | None = None @@ -82,10 +130,12 @@ def _run(self) -> None: current = _parse_ts(event.ts) if previous is not None: delay = max((current - previous).total_seconds() / self._speed, 0.0) - if delay > 0: - time.sleep(delay) + if delay > 0: + time.sleep(delay) self._store.ingest(event) self._broadcaster.publish(event) + if self._monitor is not None: + self._monitor.record_event(event) previous = current @@ -95,6 +145,7 @@ def __init__( path: str | Path, store: DashboardStore, broadcaster: EventBroadcaster, + monitor: IngestionMonitor | None = None, *, poll_interval: float = 0.5, read_existing: bool = True, @@ -102,6 +153,7 @@ def __init__( self._path = Path(path) self._store = store self._broadcaster = broadcaster + self._monitor = monitor self._poll_interval = poll_interval self._read_existing = read_existing self._thread: threading.Thread | None = None @@ -125,6 +177,8 @@ def _run(self) -> None: while not self._stop.is_set(): if not self._path.exists(): + if self._monitor is not None: + self._monitor.mark_connected(False) time.sleep(self._poll_interval) continue @@ -147,8 +201,15 @@ def _run(self) -> None: if not line: continue # model-runner JSONL framing is expected to be one event envelope per line. - event = parse_jsonl_line(line, str(self._path)) + try: + event = parse_jsonl_line(line, str(self._path)) + except ValueError as exc: + if self._monitor is not None: + self._monitor.record_error(str(exc)) + continue self._store.ingest(event) self._broadcaster.publish(event) + if self._monitor is not None: + self._monitor.record_event(event) time.sleep(self._poll_interval) diff --git a/src/lab_llm/main.py b/src/lab_llm/main.py index 5ef92ab..87f5744 100644 --- a/src/lab_llm/main.py +++ b/src/lab_llm/main.py @@ -4,7 +4,7 @@ from pathlib import Path from lab_llm.fixtures import load_events_from_jsonl -from lab_llm.live import EventBroadcaster, FixtureReplayer, JsonlTailer +from lab_llm.live import EventBroadcaster, FixtureReplayer, IngestionMonitor, JsonlTailer from lab_llm.server import run_server from lab_llm.store import DashboardStore @@ -43,17 +43,24 @@ def main() -> None: broadcaster = EventBroadcaster() if args.telemetry_jsonl: - tailer = JsonlTailer(args.telemetry_jsonl, store, broadcaster, read_existing=True) + monitor = IngestionMonitor(mode="jsonl", source=args.telemetry_jsonl) + tailer = JsonlTailer( + args.telemetry_jsonl, store, broadcaster, monitor=monitor, read_existing=True + ) tailer.start() else: + monitor = IngestionMonitor(mode="fixture", source=args.fixtures) fixture_path = Path(args.fixtures) events = load_events_from_jsonl(fixture_path) if args.replay_fixtures: - FixtureReplayer(events, store, broadcaster, speed=args.replay_speed).start() + FixtureReplayer( + events, store, broadcaster, monitor=monitor, speed=args.replay_speed + ).start() else: for event in events: store.ingest(event) - run_server(args.host, args.port, store, broadcaster) + monitor.record_event(event) + run_server(args.host, args.port, store, broadcaster, monitor) if __name__ == "__main__": diff --git a/src/lab_llm/server.py b/src/lab_llm/server.py index 22587a4..2a84b3c 100644 --- a/src/lab_llm/server.py +++ b/src/lab_llm/server.py @@ -8,7 +8,7 @@ from typing import Any from urllib.parse import urlparse -from lab_llm.live import EventBroadcaster, format_sse_event +from lab_llm.live import EventBroadcaster, IngestionMonitor, format_sse_event from lab_llm.store import DashboardStore PACKAGE_STATIC_DIR = Path(__file__).with_name("static") @@ -21,10 +21,12 @@ def __init__( server_address: tuple[str, int], store: DashboardStore, broadcaster: EventBroadcaster | None = None, + monitor: IngestionMonitor | None = None, ) -> None: super().__init__(server_address, DashboardRequestHandler) self.store = store self.broadcaster = broadcaster or EventBroadcaster() + self.monitor = monitor class DashboardRequestHandler(BaseHTTPRequestHandler): @@ -40,6 +42,9 @@ def do_GET(self) -> None: # noqa: N802 if path == "/api/events": self._handle_sse() return + if path == "/api/status": + self._send_json({"ingestion": self.server.monitor.snapshot() if self.server.monitor else None}) + return if path == "/app.js": self._serve_static("app.js", "application/javascript; charset=utf-8") return @@ -119,9 +124,13 @@ def _handle_sse(self) -> None: def run_server( - host: str, port: int, store: DashboardStore, broadcaster: EventBroadcaster | None = None + host: str, + port: int, + store: DashboardStore, + broadcaster: EventBroadcaster | None = None, + monitor: IngestionMonitor | None = None, ) -> None: - server = DashboardHTTPServer((host, port), store, broadcaster) + server = DashboardHTTPServer((host, port), store, broadcaster, monitor) print(f"lab-llm dashboard listening on http://{host}:{port}") try: server.serve_forever() diff --git a/src/lab_llm/store.py b/src/lab_llm/store.py index 74d2e79..04df6ff 100644 --- a/src/lab_llm/store.py +++ b/src/lab_llm/store.py @@ -119,8 +119,11 @@ def _handle_session_started(self, session: dict[str, Any], event: TelemetryEvent ) def _handle_load_reported(self, session: dict[str, Any], event: TelemetryEvent) -> None: - session["load_report"] = deepcopy(event.payload) - session["inspect"]["load_truth"] = deepcopy(event.payload) + current = session["load_report"] or {} + merged = deepcopy(current) + merged.update(deepcopy(event.payload)) + session["load_report"] = merged + session["inspect"]["load_truth"] = deepcopy(merged) extension = event.payload.get("extension") if isinstance(extension, dict): session["inspect"]["backend_extensions"]["load_report"] = deepcopy(extension)