Add resource offender breakdowns

apps/daemon/src/system-health-route.test.ts

@@ -0,0 +1,44 @@
+import fs from "node:fs";
+import http from "node:http";
+import os from "node:os";
+import path from "node:path";
+import type { CitadelConfig } from "@citadel/config";
+import type { SystemResourceOffenderBreakdown } from "@citadel/contracts";
+import express from "express";
+import { afterEach, describe, expect, it } from "vitest";
+import { asyncRoute } from "./app-helpers.js";
+import { closeServer, getJson, listen } from "./app-test-helpers.js";
+import { registerSystemHealthRoute } from "./system-health-route.js";
+
+const dirs: string[] = [];
+
+afterEach(() => {
+  for (const dir of dirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
+});
+
+describe("system health route", () => {
+  it("serves resource offender breakdowns and rejects unknown resource types", async () => {
+    const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "citadel-health-route-"));
+    dirs.push(dataDir);
+    fs.writeFileSync(path.join(dataDir, "fixture.txt"), "fixture\n");
+
+    const app = express();
+    const server = http.createServer(app);
+    registerSystemHealthRoute({ app, config: { dataDir } as CitadelConfig, asyncRoute });
+    const baseUrl = await listen(server);
+
+    try {
+      const body = await getJson<{ breakdown: SystemResourceOffenderBreakdown }>(
+        `${baseUrl}/api/system-health/resources/cpu/offenders`,
+      );
+      expect(body.breakdown).toMatchObject({ resource: "cpu", status: "available" });
+      expect(body.breakdown.offenders.length).toBeLessThanOrEqual(5);
+
+      const invalid = await fetch(`${baseUrl}/api/system-health/resources/nope/offenders`);
+      expect(invalid.status).toBe(400);
+      expect(await invalid.json()).toEqual({ error: "invalid_resource_type" });
+    } finally {
+      await closeServer(server);
+    }
+  });
+});

apps/daemon/src/system-health-route.ts

@@ -1,7 +1,9 @@
 import type { CitadelConfig } from "@citadel/config";
+import { SystemResourceTypeSchema } from "@citadel/contracts";
 import type express from "express";
 import type { asyncRoute as AsyncRoute } from "./app-helpers.js";
 import { collectSystemHealthSnapshot } from "./system-health.js";
+import { collectSystemResourceOffenders } from "./system-resource-offenders.js";
 
 export function registerSystemHealthRoute(input: {
   app: express.Express;
@@ -15,4 +17,14 @@ export function registerSystemHealthRoute(input: {
       res.json({ systemHealth: collectSystemHealthSnapshot({ diskPath: config.dataDir }) });
     }),
   );
+  app.get(
+    "/api/system-health/resources/:resource/offenders",
+    asyncRoute(async (req, res) => {
+      const resource = SystemResourceTypeSchema.safeParse(req.params.resource);
+      if (!resource.success) return res.status(400).json({ error: "invalid_resource_type" });
+      res.json({
+        breakdown: await collectSystemResourceOffenders({ resource: resource.data, dataDir: config.dataDir }),
+      });
+    }),
+  );
 }

apps/daemon/src/system-resource-offenders.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import { parseDuOutput, parsePsProcessTable } from "./system-resource-offenders.js";
+
+describe("system resource offender parsing", () => {
+  it("parses process table rows with command arguments intact", () => {
+    const rows = parsePsProcessTable(`
+      123  42.5  3.1  2048 node node /tmp/citadel/dist/main.js
+      456   0.0  0.2   512 bash bash
+      bad row
+    `);
+
+    expect(rows).toEqual([
+      {
+        pid: 123,
+        command: "node",
+        args: "node /tmp/citadel/dist/main.js",
+        cpuPercent: 42.5,
+        memoryPercent: 3.1,
+        rssBytes: 2 * 1024 * 1024,
+      },
+      {
+        pid: 456,
+        command: "bash",
+        args: "bash",
+        cpuPercent: 0,
+        memoryPercent: 0.2,
+        rssBytes: 512 * 1024,
+      },
+    ]);
+  });
+
+  it("turns du rows into the top five disk offenders", () => {
+    const offenders = parseDuOutput(`
+      10 /tmp/citadel/small
+      80 /tmp/citadel/big
+      30 /tmp/citadel/mid
+      20 /tmp/citadel/two
+      40 /tmp/citadel/four
+      50 /tmp/citadel/five
+      60 /tmp/citadel/six
+    `);
+
+    expect(offenders.map((offender) => offender.label)).toEqual(["big", "six", "five", "four", "mid"]);
+    expect(offenders[0]).toMatchObject({
+      id: "path:/tmp/citadel/big",
+      detail: "/tmp/citadel/big",
+      pid: null,
+      value: 80 * 1024,
+      unit: "bytes",
+    });
+  });
+});

apps/daemon/src/system-resource-offenders.ts

@@ -0,0 +1,237 @@
+import { execFile } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { promisify } from "node:util";
+import type { SystemResourceOffender, SystemResourceOffenderBreakdown, SystemResourceType } from "@citadel/contracts";
+
+const execFileAsync = promisify(execFile);
+const MAX_OFFENDERS = 5;
+
+type ProcessSnapshot = {
+  pid: number;
+  command: string;
+  args: string;
+  cpuPercent: number;
+  memoryPercent: number;
+  rssBytes: number;
+};
+
+export async function collectSystemResourceOffenders(input: {
+  resource: SystemResourceType;
+  dataDir: string;
+  now?: Date;
+}): Promise<SystemResourceOffenderBreakdown> {
+  const checkedAt = (input.now ?? new Date()).toISOString();
+  try {
+    const offenders = await collectOffenders(input.resource, input.dataDir);
+    return {
+      resource: input.resource,
+      checkedAt,
+      offenders,
+      status: "available",
+      reason: offenders.length === 0 ? "No readable offenders for this resource" : null,
+    };
+  } catch (error) {
+    return {
+      resource: input.resource,
+      checkedAt,
+      offenders: [],
+      status: "unavailable",
+      reason: error instanceof Error ? error.message : String(error),
+    };
+  }
+}
+
+async function collectOffenders(resource: SystemResourceType, dataDir: string): Promise<SystemResourceOffender[]> {
+  if (resource === "disk") return collectDiskOffenders(dataDir);
+  if (resource === "disk_io") return collectDiskIoOffenders();
+
+  const processes = await readProcessSnapshots();
+  if (resource === "cpu") return topProcessOffenders(processes, "cpuPercent", "percent");
+  if (resource === "memory") return topProcessOffenders(processes, "rssBytes", "bytes");
+  return topCitadelProcesses(processes);
+}
+
+async function readProcessSnapshots(): Promise<ProcessSnapshot[]> {
+  const { stdout } = await execFileAsync("ps", ["-eo", "pid=,pcpu=,pmem=,rss=,comm=,args="], {
+    timeout: 1500,
+    maxBuffer: 768 * 1024,
+  });
+  return parsePsProcessTable(stdout);
+}
+
+export function parsePsProcessTable(stdout: string): ProcessSnapshot[] {
+  const snapshots: ProcessSnapshot[] = [];
+  for (const line of stdout.split("\n")) {
+    const match = line.match(/^\s*(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\d+)\s+(\S+)\s*(.*)$/);
+    if (!match) continue;
+    const pid = Number.parseInt(match[1] ?? "", 10);
+    const cpuPercent = Number.parseFloat(match[2] ?? "");
+    const memoryPercent = Number.parseFloat(match[3] ?? "");
+    const rssKiB = Number.parseInt(match[4] ?? "", 10);
+    const command = match[5] ?? "";
+    if (![pid, cpuPercent, memoryPercent, rssKiB].every(Number.isFinite) || !command) continue;
+    snapshots.push({
+      pid,
+      command,
+      args: (match[6] ?? "").trim(),
+      cpuPercent,
+      memoryPercent,
+      rssBytes: rssKiB * 1024,
+    });
+  }
+  return snapshots;
+}
+
+function topProcessOffenders(
+  processes: ProcessSnapshot[],
+  metric: "cpuPercent" | "rssBytes",
+  unit: "percent" | "bytes",
+): SystemResourceOffender[] {
+  return processes
+    .filter((candidate) => candidate[metric] > 0)
+    .sort((a, b) => b[metric] - a[metric])
+    .slice(0, MAX_OFFENDERS)
+    .map((candidate) => processOffender(candidate, candidate[metric], unit));
+}
+
+function topCitadelProcesses(processes: ProcessSnapshot[]): SystemResourceOffender[] {
+  const citadelProcesses = processes.filter((candidate) =>
+    `${candidate.command} ${candidate.args}`.toLowerCase().includes("citadel"),
+  );
+  const offenders = topProcessOffenders(citadelProcesses, "rssBytes", "bytes");
+  if (offenders.length > 0) return offenders;
+  const rssBytes = process.memoryUsage().rss;
+  return [
+    {
+      id: `pid:${process.pid}`,
+      label: "citadel daemon",
+      detail: process.argv.join(" "),
+      pid: process.pid,
+      value: rssBytes,
+      unit: "bytes",
+    },
+  ];
+}
+
+function processOffender(
+  processSnapshot: ProcessSnapshot,
+  value: number,
+  unit: "percent" | "bytes",
+): SystemResourceOffender {
+  return {
+    id: `pid:${processSnapshot.pid}`,
+    label: processSnapshot.command,
+    detail: processSnapshot.args || `pid ${processSnapshot.pid}`,
+    pid: processSnapshot.pid,
+    value,
+    unit,
+  };
+}
+
+async function collectDiskOffenders(dataDir: string): Promise<SystemResourceOffender[]> {
+  const entries = fs
+    .readdirSync(dataDir, { withFileTypes: true })
+    .filter((entry) => entry.name !== "." && entry.name !== "..")
+    .slice(0, 250)
+    .map((entry) => path.join(dataDir, entry.name));
+  if (entries.length === 0) return [];
+
+  const { stdout } = await execFileAsync("du", ["-sk", "--", ...entries], {
+    timeout: 2500,
+    maxBuffer: 512 * 1024,
+  });
+  return parseDuOutput(stdout);
+}
+
+export function parseDuOutput(stdout: string): SystemResourceOffender[] {
+  return stdout
+    .split("\n")
+    .flatMap((line) => {
+      const match = line.match(/^\s*(\d+)\s+(.+)$/);
+      if (!match) return [];
+      const sizeKiB = Number.parseInt(match[1] ?? "", 10);
+      const filePath = match[2] ?? "";
+      if (!Number.isFinite(sizeKiB) || !filePath) return [];
+      return [
+        {
+          id: `path:${filePath}`,
+          label: path.basename(filePath) || filePath,
+          detail: filePath,
+          pid: null,
+          value: sizeKiB * 1024,
+          unit: "bytes" as const,
+        },
+      ];
+    })
+    .sort((a, b) => (b.value ?? 0) - (a.value ?? 0))
+    .slice(0, MAX_OFFENDERS);
+}
+
+function collectDiskIoOffenders(): SystemResourceOffender[] {
+  let procEntries: string[];
+  try {
+    procEntries = fs.readdirSync("/proc");
+  } catch (error) {
+    throw new Error(`process I/O telemetry unavailable: ${error instanceof Error ? error.message : String(error)}`);
+  }
+
+  return procEntries
+    .flatMap((entry) => diskIoOffenderForPid(entry))
+    .filter((offender) => (offender.value ?? 0) > 0)
+    .sort((a, b) => (b.value ?? 0) - (a.value ?? 0))
+    .slice(0, MAX_OFFENDERS);
+}
+
+function diskIoOffenderForPid(entry: string): SystemResourceOffender[] {
+  if (!/^\d+$/.test(entry)) return [];
+  const pid = Number.parseInt(entry, 10);
+  try {
+    const io = parseProcIo(fs.readFileSync(path.join("/proc", entry, "io"), "utf8"));
+    const value = io.readBytes + io.writeBytes;
+    const label = readProcCommand(entry);
+    return [
+      {
+        id: `pid:${pid}:io`,
+        label,
+        detail: `pid ${pid}`,
+        pid,
+        value,
+        unit: "io_bytes",
+      },
+    ];
+  } catch {
+    return [];
+  }
+}
+
+function parseProcIo(raw: string): { readBytes: number; writeBytes: number } {
+  let readBytes = 0;
+  let writeBytes = 0;
+  for (const line of raw.split("\n")) {
+    const [key, value] = line.split(":");
+    const parsed = Number.parseInt((value ?? "").trim(), 10);
+    if (!Number.isFinite(parsed)) continue;
+    if (key === "read_bytes") readBytes = parsed;
+    if (key === "write_bytes") writeBytes = parsed;
+  }
+  return { readBytes, writeBytes };
+}
+
+function readProcCommand(entry: string): string {
+  try {
+    const cmdline = fs
+      .readFileSync(path.join("/proc", entry, "cmdline"), "utf8")
+      .replace(/\0/g, " ")
+      .trim();
+    if (cmdline) return cmdline.split(/\s+/)[0] ?? "process";
+  } catch {
+    // Fall through to comm.
+  }
+  try {
+    return fs.readFileSync(path.join("/proc", entry, "comm"), "utf8").trim() || "process";
+  } catch {
+    return "process";
+  }
+}

apps/web/src/chrome.css

@@ -450,10 +450,28 @@
   min-width: 0;
   padding: 0 7px;
   border-left: 0.5px solid var(--c-line-2);
+  border-top: 0;
+  border-right: 0;
+  border-bottom: 0;
+  background: transparent;
+  color: inherit;
+  font: inherit;
   white-space: nowrap;
   font-variant-numeric: tabular-nums;
 }
 
+.cit-bb-metric--hoverable {
+  height: 22px;
+  cursor: default;
+}
+
+.cit-bb-metric--hoverable:hover,
+.cit-bb-metric--hoverable:focus-visible,
+.cit-bb-metric--active {
+  background: var(--c-elev);
+  outline: 0;
+}
+
 .cit-bb-metric-label {
   color: var(--c-fg-4);
   font-weight: 600;