From 506987cfa9c519a03963bdd53f0b735541eb45c8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 1 Jun 2026 16:55:39 +0000 Subject: [PATCH] =?UTF-8?q?feat(cli):=20ulk=20agents=20stats=20=E2=80=94?= =?UTF-8?q?=20registry=20composition=20+=20count=20reconciliation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements chantier #1 (agent-usage-stats). New subcommand: ulk agents stats [--json] [--usage] [--source DIR] - Reads framework/agents/registry.json (source of truth) → composition by category / model / phase, with internal count-field consistency check. - Reconciles the agent count declared in registry.md / CLAUDE.md / AGENTS.md / framework/agents/CLAUDE.md against the registry, surfacing drift. - --usage overlays best-effort runtime usage from accountability.jsonl, with a loud attribution-coverage caveat (NEVER presented as proof of disuse). Reconciled the declared count to 98 everywhere (was 95 / 94). Design note (blocker): accountability.jsonl HAS an 'agent' field but attribution is not yet auto-propagated (protocol v1.1 → v1.2 roadmap), so most entries are 'unknown'. Per-agent invocation stats and reliable 'dead agent' detection are therefore NOT derivable today — the command reports attribution coverage instead of claiming disuse. This blocker should feed the accountability v1.2 work. New code: internal/registry/agents.go (loader + Stats + usage parser), cmd/agents.go (command + reconciliation). Tests: agents_test.go in both packages. go build + go vet + go test ./... all green. https://claude.ai/code/session_01JrFhrsWDBZVZBRMN6MiXG4 --- AGENTS.md | 4 +- CLAUDE.md | 2 +- GEMINI.md | 4 +- framework/agents/CLAUDE.md | 4 +- framework/cli/cmd/agents.go | 207 ++++++++++++++++++ framework/cli/cmd/agents_test.go | 57 +++++ framework/cli/internal/registry/agents.go | 135 ++++++++++++ .../cli/internal/registry/agents_test.go | 116 ++++++++++ 8 files changed, 522 insertions(+), 7 deletions(-) create mode 100644 framework/cli/cmd/agents.go create mode 100644 framework/cli/cmd/agents_test.go create mode 100644 framework/cli/internal/registry/agents.go create mode 100644 framework/cli/internal/registry/agents_test.go diff --git a/AGENTS.md b/AGENTS.md index 38b936fd..93ce1174 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ > **Compatible avec** : OpenAI Codex CLI · Mistral Vibe · Google Gemini CLI (voir aussi `GEMINI.md`) > -> ulk est un monorepo de **94 agents IA spécialisés** pour Claude Code, portable vers d'autres CLI LLM. +> ulk est un monorepo de **98 agents IA spécialisés** pour Claude Code, portable vers d'autres CLI LLM. > Ce fichier fournit le contexte projet pour que l'agent puisse travailler efficacement dans ce repo. ## Qu'est-ce qu'ulk ? @@ -12,7 +12,7 @@ ulk est une boîte à outils de développement assisté par IA — une collectio **Structure principale :** ``` framework/ -├── agents/ # 94 agents .md avec frontmatter YAML +├── agents/ # 98 agents .md avec frontmatter YAML │ ├── _shared/ # Protocoles partagés (base-rules, context-hygiene, etc.) │ ├── orchestrators/ # Bruce, Blackemperor, Tony, Mathieu, Lovecraft │ ├── audit/ # Sargeras, ED-209, Harper, Vision, Perf, SEO, A11y diff --git a/CLAUDE.md b/CLAUDE.md index d37f8375..a983fcb1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -**ulk** is an AI-assisted development toolkit — a monorepo of 95 specialized AI agents for Claude Code. +**ulk** is an AI-assisted development toolkit — a monorepo of 98 specialized AI agents for Claude Code. - **framework/agents/** — Agent definitions (custom commands) - **framework/packages/core** — Shared TypeScript library (parser, types, GitHub client) diff --git a/GEMINI.md b/GEMINI.md index 8ac96bce..e33ef5bd 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -6,14 +6,14 @@ ## Présentation -ulk est un monorepo de **94 agents IA spécialisés**, architecturé autour de Claude Code et portable vers Gemini CLI via ce fichier + les sous-agents dans `.gemini/agents/`. +ulk est un monorepo de **98 agents IA spécialisés**, architecturé autour de Claude Code et portable vers Gemini CLI via ce fichier + les sous-agents dans `.gemini/agents/`. Gemini CLI reconnaît automatiquement ce fichier `GEMINI.md` ainsi que les sous-agents dans `.gemini/agents/.md`. ## Structure du projet ``` -framework/agents/ # 94 agents source (format Claude Code) +framework/agents/ # 98 agents source (format Claude Code) _shared/ # Protocoles partagés orchestrators/ # Bruce, Blackemperor, Tony, Mathieu audit/ # Sargeras, ED-209, Harper, Vision diff --git a/framework/agents/CLAUDE.md b/framework/agents/CLAUDE.md index afc06114..b21143ee 100644 --- a/framework/agents/CLAUDE.md +++ b/framework/agents/CLAUDE.md @@ -4,7 +4,7 @@ Guidance for Claude Code when working in this directory. ## Overview -94 specialized AI agent definitions for the ulk toolkit. Each agent is a Markdown file with YAML frontmatter defining a focused, autonomous sub-agent for Claude Code. +98 specialized AI agent definitions for the ulk toolkit. Each agent is a Markdown file with YAML frontmatter defining a focused, autonomous sub-agent for Claude Code. ## Agent Definition Structure @@ -26,7 +26,7 @@ Full spec: `.claude/rules/agents-authoring.md` > Auto-generated — do not edit manually. -- **Machine-readable**: `registry.json` — 94 agents, full frontmatter +- **Machine-readable**: `registry.json` — 98 agents, full frontmatter - **Human-readable**: `registry.md` — table by category - **Regenerate**: `node framework/cheatheet/generate-registry.cjs` diff --git a/framework/cli/cmd/agents.go b/framework/cli/cmd/agents.go new file mode 100644 index 00000000..31003e58 --- /dev/null +++ b/framework/cli/cmd/agents.go @@ -0,0 +1,207 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + + "github.com/izo/ulk/internal/registry" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(agentsCmd) + agentsCmd.AddCommand(agentsStatsCmd) + agentsStatsCmd.Flags().Bool("json", false, "Output as JSON") + agentsStatsCmd.Flags().Bool("usage", false, "Overlay best-effort runtime usage from .ulk-reports/accountability.jsonl") + agentsStatsCmd.Flags().String("source", "", "Path to the ulk source dir (default: auto-detect)") +} + +var agentsCmd = &cobra.Command{ + Use: "agents", + Short: "Inspect the agent registry (composition, count reconciliation, usage)", +} + +var agentsStatsCmd = &cobra.Command{ + Use: "stats", + Short: "Show agent registry composition and reconcile declared counts", + Long: `Reads framework/agents/registry.json (the source of truth) and reports the +agent composition by category, model and phase. Reconciles the count declared +in CLAUDE.md / AGENTS.md against the registry to surface drift. + +With --usage, overlays best-effort runtime usage from +.ulk-reports/accountability.jsonl. Attribution is incomplete by design until +the accountability protocol auto-propagates the active agent, so the overlay +reports its attribution coverage and is NEVER proof that an agent is unused. + +Examples: + ulk agents stats + ulk agents stats --json + ulk agents stats --usage`, + RunE: runAgentsStats, +} + +// reconcileTargets are docs that declare an agent count and tend to drift. +// Paths are relative to the ulk source dir. +var reconcileTargets = []string{ + "framework/agents/registry.md", + "CLAUDE.md", + "AGENTS.md", + "framework/agents/CLAUDE.md", +} + +// declaredCountRe captures a 2-3 digit count immediately preceding "agent(s)", +// optionally via "specialized [AI] agent". Matches "95 specialized AI agents", +// "94 agents IA", "98 agents", "94 specialized AI agent definitions". +var declaredCountRe = regexp.MustCompile(`(?i)(\d{2,3})\s+(?:specialized\s+(?:ai\s+)?agents?|agents?)`) + +func runAgentsStats(cmd *cobra.Command, _ []string) error { + jsonOut, _ := cmd.Flags().GetBool("json") + withUsage, _ := cmd.Flags().GetBool("usage") + explicitSrc, _ := cmd.Flags().GetString("source") + + src, err := resolveSourceDir(explicitSrc) + if err != nil { + return fmt.Errorf("cannot locate ulk source: %w\nSet ULK_SOURCE or use --source", err) + } + + reg, err := registry.LoadAgentRegistry(filepath.Join(src, "framework", "agents", "registry.json")) + if err != nil { + return fmt.Errorf("cannot load agent registry: %w", err) + } + stats := reg.Stats() + + type reconLine struct { + Path string `json:"path"` + Declared int `json:"declared"` + Found bool `json:"found"` + OK bool `json:"ok"` + } + recon := make([]reconLine, 0, len(reconcileTargets)) + for _, rel := range reconcileTargets { + n, found := declaredAgentCount(filepath.Join(src, rel)) + recon = append(recon, reconLine{Path: rel, Declared: n, Found: found, OK: found && n == stats.Total}) + } + + var usage *registry.AgentUsage + if withUsage { + usage, err = registry.LoadAgentUsage(filepath.Join(src, ".ulk-reports", "accountability.jsonl")) + if err != nil { + return fmt.Errorf("cannot read accountability journal: %w", err) + } + } + + if jsonOut { + out := map[string]any{ + "total": stats.Total, + "count_field_ok": stats.CountFieldOK, + "by_category": stats.ByCategory, + "by_model": stats.ByModel, + "by_phase": stats.ByPhase, + "reconciliation": recon, + } + if usage != nil { + out["usage"] = usage + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(out) + } + + fmt.Printf("🤖 AGENT REGISTRY — stats\n\n") + countMark := "✓" + if !stats.CountFieldOK { + countMark = fmt.Sprintf("✗ registry count field says %d", stats.DeclaredCount) + } + fmt.Printf(" Total agents : %d (registry count field: %s)\n\n", stats.Total, countMark) + + printBucket("By category", stats.ByCategory) + printBucket("By model", stats.ByModel) + printBucket("By phase", stats.ByPhase) + + fmt.Printf(" Count reconciliation (source of truth = registry: %d)\n", stats.Total) + for _, r := range recon { + switch { + case !r.Found: + fmt.Printf(" %-32s (no count declared)\n", r.Path) + case r.OK: + fmt.Printf(" %-32s %d ✓\n", r.Path, r.Declared) + default: + fmt.Printf(" %-32s %d ✗ (drift %+d)\n", r.Path, r.Declared, r.Declared-stats.Total) + } + } + fmt.Println() + + printUsage(withUsage, usage) + return nil +} + +func printBucket(title string, m map[string]int) { + fmt.Printf(" %s\n", title) + for _, kv := range sortedByValueDesc(m) { + fmt.Printf(" %-16s %d\n", kv.key, kv.val) + } + fmt.Println() +} + +func printUsage(enabled bool, u *registry.AgentUsage) { + if !enabled { + fmt.Printf(" Usage overlay : disabled (pass --usage to derive from accountability.jsonl)\n") + return + } + if u == nil || !u.Available { + fmt.Printf(" Usage overlay : .ulk-reports/accountability.jsonl not found — run `ulk install --with-accountability`\n") + return + } + fmt.Printf(" Usage overlay (best-effort — attribution %.0f%% complete; NOT proof of disuse)\n", + u.AttributionRatio*100) + fmt.Printf(" mutations %d · attributed %d · unknown %d\n", u.TotalMutations, u.Attributed, u.Unknown) + top := sortedByValueDesc(u.ByAgent) + if len(top) > 10 { + top = top[:10] + } + for _, kv := range top { + fmt.Printf(" %-16s %d\n", kv.key, kv.val) + } +} + +type kvPair struct { + key string + val int +} + +// sortedByValueDesc returns map entries sorted by value desc, then key asc. +func sortedByValueDesc(m map[string]int) []kvPair { + pairs := make([]kvPair, 0, len(m)) + for k, v := range m { + pairs = append(pairs, kvPair{k, v}) + } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].val != pairs[j].val { + return pairs[i].val > pairs[j].val + } + return pairs[i].key < pairs[j].key + }) + return pairs +} + +// declaredAgentCount extracts the first declared agent count from a file. +// Returns (0, false) if the file is unreadable or declares no count. +func declaredAgentCount(path string) (int, bool) { + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + m := declaredCountRe.FindSubmatch(data) + if m == nil { + return 0, false + } + n := 0 + for _, c := range m[1] { + n = n*10 + int(c-'0') + } + return n, true +} diff --git a/framework/cli/cmd/agents_test.go b/framework/cli/cmd/agents_test.go new file mode 100644 index 00000000..168b76d0 --- /dev/null +++ b/framework/cli/cmd/agents_test.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDeclaredAgentCount(t *testing.T) { + cases := []struct { + name string + content string + want int + found bool + }{ + {"claude-md", "**ulk** is a monorepo of 95 specialized AI agents for Claude Code.", 95, true}, + {"agents-md-fr", "ulk est un monorepo de 94 agents IA spécialisés", 94, true}, + {"registry-md", "**98 agents** — machine-readable: `agents/registry.json`", 98, true}, + {"agents-claude-md", "94 specialized AI agent definitions for the ulk toolkit.", 94, true}, + {"no-count", "This file declares no agent count at all.", 0, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "doc.md") + if err := os.WriteFile(p, []byte(c.content), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + got, found := declaredAgentCount(p) + if found != c.found { + t.Fatalf("found = %v, want %v", found, c.found) + } + if got != c.want { + t.Errorf("count = %d, want %d", got, c.want) + } + }) + } +} + +func TestDeclaredAgentCountMissingFile(t *testing.T) { + if _, found := declaredAgentCount(filepath.Join(t.TempDir(), "nope.md")); found { + t.Errorf("found = true, want false for missing file") + } +} + +func TestSortedByValueDesc(t *testing.T) { + got := sortedByValueDesc(map[string]int{"a": 1, "b": 3, "c": 3}) + // value desc, then key asc → b(3), c(3), a(1) + want := []kvPair{{"b", 3}, {"c", 3}, {"a", 1}} + if len(got) != len(want) { + t.Fatalf("len = %d, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("[%d] = %v, want %v", i, got[i], want[i]) + } + } +} diff --git a/framework/cli/internal/registry/agents.go b/framework/cli/internal/registry/agents.go new file mode 100644 index 00000000..e2dc462a --- /dev/null +++ b/framework/cli/internal/registry/agents.go @@ -0,0 +1,135 @@ +package registry + +import ( + "bufio" + "encoding/json" + "os" + "strings" +) + +// Agent is one entry in framework/agents/registry.json (the subset used by +// `ulk agents stats`; unknown JSON fields are ignored). +type Agent struct { + Name string `json:"name"` + File string `json:"file"` + Category string `json:"category"` + Model string `json:"model"` + Phase string `json:"phase"` + Description string `json:"description"` + Tools []string `json:"tools"` + Effort string `json:"effort"` +} + +// AgentRegistry is the top-level structure of framework/agents/registry.json. +type AgentRegistry struct { + Generated string `json:"generated"` + Generator string `json:"generator"` + Count int `json:"count"` + Agents []Agent `json:"agents"` +} + +// LoadAgentRegistry parses an agents registry.json file. +func LoadAgentRegistry(path string) (*AgentRegistry, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var r AgentRegistry + if err := json.Unmarshal(data, &r); err != nil { + return nil, err + } + return &r, nil +} + +// AgentStats is the registry composition. The true agent count is always +// len(Agents); the registry's own Count field is checked against it. +type AgentStats struct { + Total int `json:"total"` + CountFieldOK bool `json:"count_field_ok"` // registry.Count == len(Agents) + DeclaredCount int `json:"declared_count"` // value of the registry Count field + ByCategory map[string]int `json:"by_category"` + ByModel map[string]int `json:"by_model"` + ByPhase map[string]int `json:"by_phase"` +} + +// Stats computes the composition of the registry. +func (r *AgentRegistry) Stats() AgentStats { + s := AgentStats{ + Total: len(r.Agents), + DeclaredCount: r.Count, + CountFieldOK: r.Count == len(r.Agents), + ByCategory: map[string]int{}, + ByModel: map[string]int{}, + ByPhase: map[string]int{}, + } + for _, a := range r.Agents { + s.ByCategory[orUnset(a.Category)]++ + s.ByModel[orUnset(a.Model)]++ + s.ByPhase[orUnset(a.Phase)]++ + } + return s +} + +func orUnset(s string) string { + if strings.TrimSpace(s) == "" { + return "(unset)" + } + return s +} + +// AgentUsage is best-effort runtime usage derived from +// .ulk-reports/accountability.jsonl. Attribution is incomplete by design: +// until agent attribution is auto-propagated (accountability protocol v1.2), +// many mutations carry agent="unknown". Callers MUST surface AttributionRatio +// and never present absence of activity as proof an agent is unused. +type AgentUsage struct { + Available bool `json:"available"` + TotalMutations int `json:"total_mutations"` + Attributed int `json:"attributed"` + Unknown int `json:"unknown"` + ByAgent map[string]int `json:"by_agent"` + AttributionRatio float64 `json:"attribution_ratio"` +} + +type accountabilityEntry struct { + Agent string `json:"agent"` +} + +// LoadAgentUsage parses .ulk-reports/accountability.jsonl. A missing file is +// not an error: it returns an AgentUsage with Available=false. +func LoadAgentUsage(path string) (*AgentUsage, error) { + u := &AgentUsage{ByAgent: map[string]int{}} + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return u, nil + } + return nil, err + } + defer f.Close() + + u.Available = true + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1024*1024), 4*1024*1024) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + var e accountabilityEntry + if json.Unmarshal([]byte(line), &e) != nil { + continue // skip malformed lines + } + u.TotalMutations++ + if e.Agent == "" || e.Agent == "unknown" { + u.Unknown++ + continue + } + u.Attributed++ + u.ByAgent[e.Agent]++ + } + if u.TotalMutations > 0 { + u.AttributionRatio = float64(u.Attributed) / float64(u.TotalMutations) + } + return u, sc.Err() +} diff --git a/framework/cli/internal/registry/agents_test.go b/framework/cli/internal/registry/agents_test.go new file mode 100644 index 00000000..4a2fcfd0 --- /dev/null +++ b/framework/cli/internal/registry/agents_test.go @@ -0,0 +1,116 @@ +package registry + +import ( + "os" + "path/filepath" + "testing" +) + +const sampleRegistry = `{ + "generated": "2026-06-01", + "count": 3, + "agents": [ + {"name": "bruce", "category": "orchestrators", "model": "opus", "phase": "orchestrator"}, + {"name": "godspeed", "category": "session", "model": "haiku", "phase": "review"}, + {"name": "shuri", "category": "docs", "model": "sonnet", "phase": "define"} + ] +}` + +func writeTemp(t *testing.T, name, content string) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatalf("write temp: %v", err) + } + return p +} + +func TestLoadAgentRegistryAndStats(t *testing.T) { + reg, err := LoadAgentRegistry(writeTemp(t, "registry.json", sampleRegistry)) + if err != nil { + t.Fatalf("LoadAgentRegistry: %v", err) + } + stats := reg.Stats() + + if stats.Total != 3 { + t.Errorf("Total = %d, want 3", stats.Total) + } + if !stats.CountFieldOK { + t.Errorf("CountFieldOK = false, want true (count field 3 == len 3)") + } + for cat, want := range map[string]int{"orchestrators": 1, "session": 1, "docs": 1} { + if stats.ByCategory[cat] != want { + t.Errorf("ByCategory[%s] = %d, want %d", cat, stats.ByCategory[cat], want) + } + } + for model, want := range map[string]int{"opus": 1, "haiku": 1, "sonnet": 1} { + if stats.ByModel[model] != want { + t.Errorf("ByModel[%s] = %d, want %d", model, stats.ByModel[model], want) + } + } +} + +func TestStatsCountFieldDrift(t *testing.T) { + // count field says 99 but there are only 3 agents → drift must be flagged. + drift := `{"count": 99, "agents": [ + {"name": "a", "category": "x", "model": "opus", "phase": "build"}, + {"name": "b", "category": "x", "model": "opus", "phase": "build"}, + {"name": "c", "category": "x", "model": "opus", "phase": "build"} + ]}` + reg, err := LoadAgentRegistry(writeTemp(t, "registry.json", drift)) + if err != nil { + t.Fatalf("LoadAgentRegistry: %v", err) + } + stats := reg.Stats() + if stats.Total != 3 { + t.Errorf("Total = %d, want 3", stats.Total) + } + if stats.CountFieldOK { + t.Errorf("CountFieldOK = true, want false (count 99 != len 3)") + } +} + +func TestLoadAgentUsageMissingFile(t *testing.T) { + u, err := LoadAgentUsage(filepath.Join(t.TempDir(), "does-not-exist.jsonl")) + if err != nil { + t.Fatalf("LoadAgentUsage on missing file should not error: %v", err) + } + if u.Available { + t.Errorf("Available = true, want false for missing file") + } +} + +func TestLoadAgentUsageAttribution(t *testing.T) { + journal := `{"tool":"Edit","agent":"task-runner"} +{"tool":"Write","agent":"bruce"} +{"tool":"Edit","agent":"task-runner"} +{"tool":"Bash","agent":"unknown"} +{"tool":"Edit"} + +not-json-skip-me +` + u, err := LoadAgentUsage(writeTemp(t, "accountability.jsonl", journal)) + if err != nil { + t.Fatalf("LoadAgentUsage: %v", err) + } + if !u.Available { + t.Fatalf("Available = false, want true") + } + // 5 valid JSON lines (the blank line and non-JSON line are skipped). + if u.TotalMutations != 5 { + t.Errorf("TotalMutations = %d, want 5", u.TotalMutations) + } + if u.Attributed != 3 { + t.Errorf("Attributed = %d, want 3", u.Attributed) + } + // "unknown" entry + entry with no agent field both count as unknown. + if u.Unknown != 2 { + t.Errorf("Unknown = %d, want 2", u.Unknown) + } + if u.ByAgent["task-runner"] != 2 { + t.Errorf("ByAgent[task-runner] = %d, want 2", u.ByAgent["task-runner"]) + } + if got := u.AttributionRatio; got < 0.59 || got > 0.61 { + t.Errorf("AttributionRatio = %.3f, want ~0.60", got) + } +}