From 49d2a992a87dac23e420c0b964f4072e3fa59e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ovidiu=20M=C4=83ru=C8=99?= Date: Sun, 7 Jun 2026 20:11:19 +0000 Subject: [PATCH] Add resource offender breakdowns --- apps/daemon/src/system-health-route.test.ts | 44 ++++ apps/daemon/src/system-health-route.ts | 12 + .../src/system-resource-offenders.test.ts | 52 ++++ apps/daemon/src/system-resource-offenders.ts | 237 ++++++++++++++++++ apps/web/src/chrome.css | 18 ++ apps/web/src/cockpit-bottom-bar.test.ts | 136 +++++++++- apps/web/src/cockpit-bottom-bar.tsx | 188 +++++++++++++- apps/web/src/main.tsx | 1 + apps/web/src/resource-breakdown.css | 127 ++++++++++ packages/contracts/src/system-health.test.ts | 24 +- packages/contracts/src/system-health.ts | 25 ++ 11 files changed, 850 insertions(+), 14 deletions(-) create mode 100644 apps/daemon/src/system-health-route.test.ts create mode 100644 apps/daemon/src/system-resource-offenders.test.ts create mode 100644 apps/daemon/src/system-resource-offenders.ts create mode 100644 apps/web/src/resource-breakdown.css diff --git a/apps/daemon/src/system-health-route.test.ts b/apps/daemon/src/system-health-route.test.ts new file mode 100644 index 00000000..b9264945 --- /dev/null +++ b/apps/daemon/src/system-health-route.test.ts @@ -0,0 +1,44 @@ +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import type { CitadelConfig } from "@citadel/config"; +import type { SystemResourceOffenderBreakdown } from "@citadel/contracts"; +import express from "express"; +import { afterEach, describe, expect, it } from "vitest"; +import { asyncRoute } from "./app-helpers.js"; +import { closeServer, getJson, listen } from "./app-test-helpers.js"; +import { registerSystemHealthRoute } from "./system-health-route.js"; + +const dirs: string[] = []; + +afterEach(() => { + for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 }); +}); + +describe("system health route", () => { + it("serves resource offender breakdowns and rejects unknown resource types", async () => { + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "citadel-health-route-")); + dirs.push(dataDir); + fs.writeFileSync(path.join(dataDir, "fixture.txt"), "fixture\n"); + + const app = express(); + const server = http.createServer(app); + registerSystemHealthRoute({ app, config: { dataDir } as CitadelConfig, asyncRoute }); + const baseUrl = await listen(server); + + try { + const body = await getJson<{ breakdown: SystemResourceOffenderBreakdown }>( + `${baseUrl}/api/system-health/resources/cpu/offenders`, + ); + expect(body.breakdown).toMatchObject({ resource: "cpu", status: "available" }); + expect(body.breakdown.offenders.length).toBeLessThanOrEqual(5); + + const invalid = await fetch(`${baseUrl}/api/system-health/resources/nope/offenders`); + expect(invalid.status).toBe(400); + expect(await invalid.json()).toEqual({ error: "invalid_resource_type" }); + } finally { + await closeServer(server); + } + }); +}); diff --git a/apps/daemon/src/system-health-route.ts b/apps/daemon/src/system-health-route.ts index 7f222b12..0e7315c5 100644 --- a/apps/daemon/src/system-health-route.ts +++ b/apps/daemon/src/system-health-route.ts @@ -1,7 +1,9 @@ import type { CitadelConfig } from "@citadel/config"; +import { SystemResourceTypeSchema } from "@citadel/contracts"; import type express from "express"; import type { asyncRoute as AsyncRoute } from "./app-helpers.js"; import { collectSystemHealthSnapshot } from "./system-health.js"; +import { collectSystemResourceOffenders } from "./system-resource-offenders.js"; export function registerSystemHealthRoute(input: { app: express.Express; @@ -15,4 +17,14 @@ export function registerSystemHealthRoute(input: { res.json({ systemHealth: collectSystemHealthSnapshot({ diskPath: config.dataDir }) }); }), ); + app.get( + "/api/system-health/resources/:resource/offenders", + asyncRoute(async (req, res) => { + const resource = SystemResourceTypeSchema.safeParse(req.params.resource); + if (!resource.success) return res.status(400).json({ error: "invalid_resource_type" }); + res.json({ + breakdown: await collectSystemResourceOffenders({ resource: resource.data, dataDir: config.dataDir }), + }); + }), + ); } diff --git a/apps/daemon/src/system-resource-offenders.test.ts b/apps/daemon/src/system-resource-offenders.test.ts new file mode 100644 index 00000000..e7a793b5 --- /dev/null +++ b/apps/daemon/src/system-resource-offenders.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { parseDuOutput, parsePsProcessTable } from "./system-resource-offenders.js"; + +describe("system resource offender parsing", () => { + it("parses process table rows with command arguments intact", () => { + const rows = parsePsProcessTable(` + 123 42.5 3.1 2048 node node /tmp/citadel/dist/main.js + 456 0.0 0.2 512 bash bash + bad row + `); + + expect(rows).toEqual([ + { + pid: 123, + command: "node", + args: "node /tmp/citadel/dist/main.js", + cpuPercent: 42.5, + memoryPercent: 3.1, + rssBytes: 2 * 1024 * 1024, + }, + { + pid: 456, + command: "bash", + args: "bash", + cpuPercent: 0, + memoryPercent: 0.2, + rssBytes: 512 * 1024, + }, + ]); + }); + + it("turns du rows into the top five disk offenders", () => { + const offenders = parseDuOutput(` + 10 /tmp/citadel/small + 80 /tmp/citadel/big + 30 /tmp/citadel/mid + 20 /tmp/citadel/two + 40 /tmp/citadel/four + 50 /tmp/citadel/five + 60 /tmp/citadel/six + `); + + expect(offenders.map((offender) => offender.label)).toEqual(["big", "six", "five", "four", "mid"]); + expect(offenders[0]).toMatchObject({ + id: "path:/tmp/citadel/big", + detail: "/tmp/citadel/big", + pid: null, + value: 80 * 1024, + unit: "bytes", + }); + }); +}); diff --git a/apps/daemon/src/system-resource-offenders.ts b/apps/daemon/src/system-resource-offenders.ts new file mode 100644 index 00000000..a87a8ee4 --- /dev/null +++ b/apps/daemon/src/system-resource-offenders.ts @@ -0,0 +1,237 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import process from "node:process"; +import { promisify } from "node:util"; +import type { SystemResourceOffender, SystemResourceOffenderBreakdown, SystemResourceType } from "@citadel/contracts"; + +const execFileAsync = promisify(execFile); +const MAX_OFFENDERS = 5; + +type ProcessSnapshot = { + pid: number; + command: string; + args: string; + cpuPercent: number; + memoryPercent: number; + rssBytes: number; +}; + +export async function collectSystemResourceOffenders(input: { + resource: SystemResourceType; + dataDir: string; + now?: Date; +}): Promise { + const checkedAt = (input.now ?? new Date()).toISOString(); + try { + const offenders = await collectOffenders(input.resource, input.dataDir); + return { + resource: input.resource, + checkedAt, + offenders, + status: "available", + reason: offenders.length === 0 ? "No readable offenders for this resource" : null, + }; + } catch (error) { + return { + resource: input.resource, + checkedAt, + offenders: [], + status: "unavailable", + reason: error instanceof Error ? error.message : String(error), + }; + } +} + +async function collectOffenders(resource: SystemResourceType, dataDir: string): Promise { + if (resource === "disk") return collectDiskOffenders(dataDir); + if (resource === "disk_io") return collectDiskIoOffenders(); + + const processes = await readProcessSnapshots(); + if (resource === "cpu") return topProcessOffenders(processes, "cpuPercent", "percent"); + if (resource === "memory") return topProcessOffenders(processes, "rssBytes", "bytes"); + return topCitadelProcesses(processes); +} + +async function readProcessSnapshots(): Promise { + const { stdout } = await execFileAsync("ps", ["-eo", "pid=,pcpu=,pmem=,rss=,comm=,args="], { + timeout: 1500, + maxBuffer: 768 * 1024, + }); + return parsePsProcessTable(stdout); +} + +export function parsePsProcessTable(stdout: string): ProcessSnapshot[] { + const snapshots: ProcessSnapshot[] = []; + for (const line of stdout.split("\n")) { + const match = line.match(/^\s*(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s*(.*)$/); + if (!match) continue; + const pid = Number.parseInt(match[1] ?? "", 10); + const cpuPercent = Number.parseFloat(match[2] ?? ""); + const memoryPercent = Number.parseFloat(match[3] ?? ""); + const rssKiB = Number.parseInt(match[4] ?? "", 10); + const command = match[5] ?? ""; + if (![pid, cpuPercent, memoryPercent, rssKiB].every(Number.isFinite) || !command) continue; + snapshots.push({ + pid, + command, + args: (match[6] ?? "").trim(), + cpuPercent, + memoryPercent, + rssBytes: rssKiB * 1024, + }); + } + return snapshots; +} + +function topProcessOffenders( + processes: ProcessSnapshot[], + metric: "cpuPercent" | "rssBytes", + unit: "percent" | "bytes", +): SystemResourceOffender[] { + return processes + .filter((candidate) => candidate[metric] > 0) + .sort((a, b) => b[metric] - a[metric]) + .slice(0, MAX_OFFENDERS) + .map((candidate) => processOffender(candidate, candidate[metric], unit)); +} + +function topCitadelProcesses(processes: ProcessSnapshot[]): SystemResourceOffender[] { + const citadelProcesses = processes.filter((candidate) => + `${candidate.command} ${candidate.args}`.toLowerCase().includes("citadel"), + ); + const offenders = topProcessOffenders(citadelProcesses, "rssBytes", "bytes"); + if (offenders.length > 0) return offenders; + const rssBytes = process.memoryUsage().rss; + return [ + { + id: `pid:${process.pid}`, + label: "citadel daemon", + detail: process.argv.join(" "), + pid: process.pid, + value: rssBytes, + unit: "bytes", + }, + ]; +} + +function processOffender( + processSnapshot: ProcessSnapshot, + value: number, + unit: "percent" | "bytes", +): SystemResourceOffender { + return { + id: `pid:${processSnapshot.pid}`, + label: processSnapshot.command, + detail: processSnapshot.args || `pid ${processSnapshot.pid}`, + pid: processSnapshot.pid, + value, + unit, + }; +} + +async function collectDiskOffenders(dataDir: string): Promise { + const entries = fs + .readdirSync(dataDir, { withFileTypes: true }) + .filter((entry) => entry.name !== "." && entry.name !== "..") + .slice(0, 250) + .map((entry) => path.join(dataDir, entry.name)); + if (entries.length === 0) return []; + + const { stdout } = await execFileAsync("du", ["-sk", "--", ...entries], { + timeout: 2500, + maxBuffer: 512 * 1024, + }); + return parseDuOutput(stdout); +} + +export function parseDuOutput(stdout: string): SystemResourceOffender[] { + return stdout + .split("\n") + .flatMap((line) => { + const match = line.match(/^\s*(\d+)\s+(.+)$/); + if (!match) return []; + const sizeKiB = Number.parseInt(match[1] ?? "", 10); + const filePath = match[2] ?? ""; + if (!Number.isFinite(sizeKiB) || !filePath) return []; + return [ + { + id: `path:${filePath}`, + label: path.basename(filePath) || filePath, + detail: filePath, + pid: null, + value: sizeKiB * 1024, + unit: "bytes" as const, + }, + ]; + }) + .sort((a, b) => (b.value ?? 0) - (a.value ?? 0)) + .slice(0, MAX_OFFENDERS); +} + +function collectDiskIoOffenders(): SystemResourceOffender[] { + let procEntries: string[]; + try { + procEntries = fs.readdirSync("/proc"); + } catch (error) { + throw new Error(`process I/O telemetry unavailable: ${error instanceof Error ? error.message : String(error)}`); + } + + return procEntries + .flatMap((entry) => diskIoOffenderForPid(entry)) + .filter((offender) => (offender.value ?? 0) > 0) + .sort((a, b) => (b.value ?? 0) - (a.value ?? 0)) + .slice(0, MAX_OFFENDERS); +} + +function diskIoOffenderForPid(entry: string): SystemResourceOffender[] { + if (!/^\d+$/.test(entry)) return []; + const pid = Number.parseInt(entry, 10); + try { + const io = parseProcIo(fs.readFileSync(path.join("/proc", entry, "io"), "utf8")); + const value = io.readBytes + io.writeBytes; + const label = readProcCommand(entry); + return [ + { + id: `pid:${pid}:io`, + label, + detail: `pid ${pid}`, + pid, + value, + unit: "io_bytes", + }, + ]; + } catch { + return []; + } +} + +function parseProcIo(raw: string): { readBytes: number; writeBytes: number } { + let readBytes = 0; + let writeBytes = 0; + for (const line of raw.split("\n")) { + const [key, value] = line.split(":"); + const parsed = Number.parseInt((value ?? "").trim(), 10); + if (!Number.isFinite(parsed)) continue; + if (key === "read_bytes") readBytes = parsed; + if (key === "write_bytes") writeBytes = parsed; + } + return { readBytes, writeBytes }; +} + +function readProcCommand(entry: string): string { + try { + const cmdline = fs + .readFileSync(path.join("/proc", entry, "cmdline"), "utf8") + .replace(/\0/g, " ") + .trim(); + if (cmdline) return cmdline.split(/\s+/)[0] ?? "process"; + } catch { + // Fall through to comm. + } + try { + return fs.readFileSync(path.join("/proc", entry, "comm"), "utf8").trim() || "process"; + } catch { + return "process"; + } +} diff --git a/apps/web/src/chrome.css b/apps/web/src/chrome.css index 4b59f7f1..cdf16567 100644 --- a/apps/web/src/chrome.css +++ b/apps/web/src/chrome.css @@ -450,10 +450,28 @@ min-width: 0; padding: 0 7px; border-left: 0.5px solid var(--c-line-2); + border-top: 0; + border-right: 0; + border-bottom: 0; + background: transparent; + color: inherit; + font: inherit; white-space: nowrap; font-variant-numeric: tabular-nums; } +.cit-bb-metric--hoverable { + height: 22px; + cursor: default; +} + +.cit-bb-metric--hoverable:hover, +.cit-bb-metric--hoverable:focus-visible, +.cit-bb-metric--active { + background: var(--c-elev); + outline: 0; +} + .cit-bb-metric-label { color: var(--c-fg-4); font-weight: 600; diff --git a/apps/web/src/cockpit-bottom-bar.test.ts b/apps/web/src/cockpit-bottom-bar.test.ts index 96e9f33d..090a02c2 100644 --- a/apps/web/src/cockpit-bottom-bar.test.ts +++ b/apps/web/src/cockpit-bottom-bar.test.ts @@ -1,11 +1,12 @@ // @vitest-environment happy-dom -import type { TerminalSession, WorkspaceSession } from "@citadel/contracts"; +import type { SystemHealthSnapshot, TerminalSession, WorkspaceSession } from "@citadel/contracts"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { createElement } from "react"; import { flushSync } from "react-dom"; import { type Root, createRoot } from "react-dom/client"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { api } from "./api.js"; import { BottomBar, formatBytes, formatClock, formatPercent } from "./cockpit-bottom-bar.js"; vi.mock("./api.js", () => ({ @@ -13,6 +14,13 @@ vi.mock("./api.js", () => ({ })); const roots: Root[] = []; +const apiMock = vi.mocked(api); + +beforeEach(() => { + apiMock.mockReset(); + const defaultApi: typeof api = async () => ({ systemHealth: null }) as never; + apiMock.mockImplementation(defaultApi); +}); afterEach(() => { for (const root of roots.splice(0)) root.unmount(); @@ -54,6 +62,86 @@ describe("footer right side", () => { }); }); +describe("footer resource breakdowns", () => { + it("opens an immediate loading modal and fetches top offenders for the hovered resource", async () => { + let resolveBreakdown!: (value: { + breakdown: { + resource: "cpu"; + checkedAt: string; + status: "available"; + reason: null; + offenders: Array<{ + id: string; + label: string; + detail: string; + pid: number; + value: number; + unit: "percent"; + }>; + }; + }) => void; + const breakdown = new Promise<{ + breakdown: { + resource: "cpu"; + checkedAt: string; + status: "available"; + reason: null; + offenders: Array<{ + id: string; + label: string; + detail: string; + pid: number; + value: number; + unit: "percent"; + }>; + }; + }>((resolve) => { + resolveBreakdown = resolve; + }); + const mockApi: typeof api = async (path) => { + if (path === "/api/system-health") return { systemHealth: healthSnapshot } as never; + if (path === "/api/system-health/resources/cpu/offenders") return breakdown as never; + throw new Error(`unexpected path ${path}`); + }; + apiMock.mockImplementation(mockApi); + + renderBottomBar({ activeSession: null, sessions: [] }); + const cpu = document.querySelector('[data-resource-type="cpu"]'); + if (!cpu) throw new Error("CPU metric missing"); + + await flushReact(() => { + cpu.dispatchEvent(new MouseEvent("mouseover", { bubbles: true })); + }); + + await waitFor( + () => document.querySelector(".cit-resource-modal")?.textContent?.includes("Loading breakdown...") === true, + ); + expect(apiMock.mock.calls.some(([path]) => path === "/api/system-health/resources/cpu/offenders")).toBe(true); + + resolveBreakdown({ + breakdown: { + resource: "cpu", + checkedAt: "2026-06-05T12:00:00.000Z", + status: "available", + reason: null, + offenders: [ + { + id: "pid:123", + label: "node", + detail: "node /tmp/citadel/dist/main.js", + pid: 123, + value: 48.4, + unit: "percent", + }, + ], + }, + }); + + await waitFor(() => document.querySelector(".cit-resource-modal")?.textContent?.includes("node") === true); + expect(document.querySelector(".cit-resource-modal")?.textContent).toContain("48%"); + }); +}); + function renderBottomBar(props: { activeSession: WorkspaceSession | null; sessions: WorkspaceSession[] }) { const rootElement = document.createElement("div"); document.body.replaceChildren(rootElement); @@ -73,6 +161,29 @@ function renderBottomBar(props: { activeSession: WorkspaceSession | null; sessio }); } +async function flushReact(callback: () => void | Promise) { + let result: void | Promise = undefined; + flushSync(() => { + result = callback(); + }); + await result; + await settle(); +} + +async function settle() { + await Promise.resolve(); + await Promise.resolve(); +} + +async function waitFor(predicate: () => boolean) { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 1000) break; + await new Promise((resolve) => setTimeout(resolve, 0)); + } + expect(predicate()).toBe(true); +} + function session(overrides: Partial = {}): WorkspaceSession { const base: TerminalSession = { id: "sess_1", @@ -91,3 +202,24 @@ function session(overrides: Partial = {}): WorkspaceSession { }; return { ...base, ...overrides }; } + +const healthSnapshot: SystemHealthSnapshot = { + tone: "healthy", + reason: null, + checkedAt: "2026-06-05T12:00:00.000Z", + machine: { + cpu: { percentUsed: 42, loadAverage1m: 1.2, cores: 8 }, + memory: { totalBytes: 100, usedBytes: 45, freeBytes: 55, percentUsed: 45 }, + disk: { + path: "/tmp/citadel", + device: "sda1", + totalBytes: 100, + usedBytes: 40, + freeBytes: 60, + percentUsed: 40, + ioUtilizationPercent: 2, + error: null, + }, + }, + process: { pid: 123, rssBytes: 32, heapUsedBytes: 16, heapTotalBytes: 24, percentOfMachineMemory: 1 }, +}; diff --git a/apps/web/src/cockpit-bottom-bar.tsx b/apps/web/src/cockpit-bottom-bar.tsx index 06b63bdc..901d0ce4 100644 --- a/apps/web/src/cockpit-bottom-bar.tsx +++ b/apps/web/src/cockpit-bottom-bar.tsx @@ -1,20 +1,71 @@ -import type { SystemHealthSnapshot, SystemHealthTone, WorkspaceSession } from "@citadel/contracts"; +import type { + SystemHealthSnapshot, + SystemHealthTone, + SystemResourceOffender, + SystemResourceOffenderBreakdown, + SystemResourceType, + WorkspaceSession, +} from "@citadel/contracts"; import { useQuery } from "@tanstack/react-query"; import { useEffect, useState } from "react"; import { api } from "./api.js"; type SystemHealthResponse = { systemHealth: SystemHealthSnapshot }; +type SystemResourceOffendersResponse = { breakdown: SystemResourceOffenderBreakdown }; +type ResourceMetricConfig = { type: SystemResourceType; label: string }; +type BreakdownState = + | { status: "idle" } + | { status: "loading" } + | { status: "loaded"; breakdown: SystemResourceOffenderBreakdown } + | { status: "error"; message: string }; + +const RESOURCE_METRICS = { + cpu: { type: "cpu", label: "CPU" }, + memory: { type: "memory", label: "RAM" }, + disk: { type: "disk", label: "Disk" }, + disk_io: { type: "disk_io", label: "I/O" }, + citadel: { type: "citadel", label: "Citadel" }, +} satisfies Record; export function BottomBar(props: { activeSession: WorkspaceSession | null; sessions: WorkspaceSession[]; }) { const [now, setNow] = useState(() => formatClock(new Date())); + const [activeResource, setActiveResource] = useState(null); + const [breakdownState, setBreakdownState] = useState({ status: "idle" }); useEffect(() => { const id = window.setInterval(() => setNow(formatClock(new Date())), 30_000); return () => window.clearInterval(id); }, []); + useEffect(() => { + if (!activeResource) { + setBreakdownState({ status: "idle" }); + return; + } + const controller = new AbortController(); + setBreakdownState({ status: "loading" }); + api(`/api/system-health/resources/${activeResource.type}/offenders`, { + signal: controller.signal, + }) + .then((response) => setBreakdownState({ status: "loaded", breakdown: response.breakdown })) + .catch((error: unknown) => { + if (isAbortError(error)) return; + setBreakdownState({ status: "error", message: errorMessage(error) }); + }); + return () => controller.abort(); + }, [activeResource]); + + useEffect(() => { + if (!activeResource) return; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") setActiveResource(null); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [activeResource]); + const systemHealth = useQuery({ queryKey: ["system-health"], queryFn: () => api("/api/system-health"), @@ -23,7 +74,7 @@ export function BottomBar(props: { const health = systemHealth.data?.systemHealth ?? null; return ( -