diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index a14000c..47802b0 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,7 +1,7 @@ { "$schema": "https://anthropic.com/claude-code/marketplace.schema.json", "name": "ce-lite", - "description": "Lightweight-delegation variant of compound-engineering. Drops the persistent agent registrations (~58.8k tokens baseline) and converts orchestrator skills to spawn Task subagents on demand from references/agent-prompts/. Auto-regenerated daily from upstream EveryInc/compound-engineering-plugin.", + "description": "Lightweight-delegation variant of compound-engineering. Drops the persistent agent registrations (~2k idle tokens per agent, ~88k on upstream v3.8.4) and converts orchestrator skills to spawn Task subagents on demand from references/agent-prompts/. Auto-regenerated daily from upstream EveryInc/compound-engineering-plugin.", "owner": { "name": "ak2k", "url": "https://github.com/ak2k" diff --git a/CLAUDE.md b/CLAUDE.md index 6e2d15c..714f516 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,8 @@ Lightweight-delegation variant of upstream [`EveryInc/compound-engineering-plugin`](https://github.com/EveryInc/compound-engineering-plugin): this repo's converter (`converter/*.py`) reads upstream's `agents/*.agent.md` files and emits a derived plugin in `dist/` with persistent agent -registrations stripped (~58.8k tokens of idle context savings), specialist +registrations stripped (~2k idle tokens per registered agent — ~88k on +upstream v3.8.4, measured via `/context`), specialist prompts relocated to `references/agent-prompts/`, and orchestrator skills rewritten to dispatch via a `bin/ce-lite-persona` resolver shim. @@ -38,9 +39,10 @@ byte-equivalent between upstream and dist (round-trip property). was extracted from (e.g. `compound-engineering-v3.8.3`). The `lite_suffix_from_git` versioning scheme keys off changes to this file. - `tests/test_converter.py` — pytest suite, fast (<2s). -- `tests/integration/` — quota-spending integration eval (`claude -p` - routing tests); run via `nix run .#integration-eval`, NOT part of - `nix flake check`. +- `tests/integration/` — opt-in, quota-spending routing probe driven by + `claude -p`; run via `nix run .#integration-eval`. Headless `-p` routing is + noisy/unreliable — treat output as a rough signal, not a passing/failing + test. NOT part of `nix flake check`. - `flake.nix` — `nix flake check` runs format + actionlint + pytest. `nix run .#integration-eval` for the quota-spending eval. diff --git a/README.md b/README.md index 36230a8..a902ed6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Lightweight-delegation variant of [compound-engineering-plugin](https://github.com/EveryInc/compound-engineering-plugin). -Drops the persistent agent registrations (**measured: ~99.9k tokens of idle context, gone**) and dispatches every specialist via a single resolver shim instead. Specialist prompts move to `references/agent-prompts/`; the resolver (`bin/ce-lite-persona`) loads them on demand. Same coverage as upstream CE, near-zero idle cost, and a `UserPromptSubmit` hook restores proactive specialist invocation across all 49 personas. +Drops the persistent agent registrations (**~2k idle tokens per registered agent — ~88k of idle context on upstream v3.8.4, gone**) and dispatches every specialist via a single resolver shim instead. Specialist prompts move to `references/agent-prompts/`; the resolver (`bin/ce-lite-persona`) loads them on demand. Same coverage as upstream CE, near-zero idle cost, and a `UserPromptSubmit` hook restores proactive specialist invocation across all 49 personas. ## Install @@ -15,17 +15,16 @@ During trial, slash commands are namespaced under `/ce-lite:*` so the original ` ## Trade-offs vs. upstream CE -Measured against an interactive Claude Code session (Opus 4.7 1M-context, two isolated `CLAUDE_CONFIG_DIR`s, identical state except the installed plugin): +Measured via interactive `/context` (Opus 4.7 1M-context, isolated `CLAUDE_CONFIG_DIR`s, identical state except the installed plugin), upstream **compound-engineering-v3.8.4** vs ce-lite: -| Context slot | Upstream CE | ce-lite | Δ | +| Context slot | Upstream CE (v3.8.4) | ce-lite | Δ | |---|---:|---:|---:| -| **Total** | **192.5k** | **87.3k** | **−105.2k** | -| Custom agents | 99.9k (49 subagent registrations) | 0 (no registrations) | −99.9k | -| System tools | 35.5k | 22.3k | −13.2k | -| Skills | 4.6k | 12.5k | +7.9k | -| System prompt / MCP / memory / messages | 52.5k | 52.5k | 0 | +| Custom agents | ~88k (measured) | 0 (no registrations) | **≈ −88k** | +| System tools | 28.2k | 14.7k | −13.5k | +| Skills | 4.6k | 12.1k | +7.5k | +| **Net idle** | | | **≈ −94k** | -The ~99.9k of custom-agents context — Claude Code loads each registered subagent's full body for proactive selection — is the dominant savings. ce-lite trades a +7.9k skills bump (88 SKILL.md descriptions vs upstream's 29) for that. +Claude Code loads each registered subagent's **full body** into idle context (~2k tokens per agent — the "Custom agents" slot above) so the model can proactively select it; ce-lite ships zero registrations, emptying that slot. The total scales with how many agents upstream registers (~2k each); an isolated `/context` on v3.8.4 measured ~88k. ce-lite trades a +7.5k skills bump (88 SKILL.md descriptions vs upstream's 38) for eliminating the agent slot — while keeping all 49 specialist personas addressable. | Behavior | Upstream CE | ce-lite | |---|---|---| @@ -70,7 +69,7 @@ Run locally: ```sh nix develop # devshell with python + pyyaml + pytest + ruff + actionlint nix flake check # treefmt (nixpkgs-fmt + ruff-format + ruff-check) + actionlint + pytest -nix run .#integration-eval # Tier 3 routing eval against `claude -p` (opt-in, quota-spending) +nix run .#integration-eval # opt-in routing probe via `claude -p` (quota-spending; headless routing is noisy — a rough signal, not a reliable test/gate) ``` Re-refine the `UserPromptSubmit` keyword rules with `claude -p` (sonnet) before a release: diff --git a/converter/extract.py b/converter/extract.py index 0efa4b3..265091d 100644 --- a/converter/extract.py +++ b/converter/extract.py @@ -3,7 +3,7 @@ For CE v3.x layout: upstream/plugins/compound-engineering/ ├── .claude-plugin/plugin.json - ├── agents/.agent.md ← these get extracted out + ├── agents/.md ← extracted out (≤v3.8.3: .agent.md) ├── skills//SKILL.md ← copied as-is (rewrite.py prepends a preamble) └── ... (other top-level files) @@ -29,8 +29,11 @@ # Frontmatter is fenced with "---" lines at the top of each agent .md file. FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)$", re.DOTALL) -# Agent files in v3.x are named ".agent.md" inside agents/. -AGENT_FILE_SUFFIX = ".agent.md" +# Agent files live in agents/. Upstream ≤ v3.8.3 named them ".agent.md"; +# v3.8.4 renamed them to plain ".md" (the agents/ dir disambiguates now, +# not the extension). Accept both so the converter keeps tracking upstream +# across the rename. Longest suffix first so name-derivation prefers ".agent.md". +AGENT_FILE_SUFFIXES = (".agent.md", ".md") # How we rename the plugin during trial. Switch to "ce" before filing upstream PR. LITE_NAME = "ce-lite" @@ -113,7 +116,7 @@ def extract_agents(plugin_root: Path) -> list[AgentRecord]: raise FileNotFoundError(f"no agents/ directory at {agents_dir}") records: list[AgentRecord] = [] - for path in sorted(agents_dir.glob(f"*{AGENT_FILE_SUFFIX}")): + for path in sorted(agents_dir.glob("*.md")): text = path.read_text(encoding="utf-8") try: fm, body = parse_frontmatter(text) @@ -124,11 +127,14 @@ def extract_agents(plugin_root: Path) -> list[AgentRecord]: if not name: raise ValueError(f"{path}: agent has no 'name' in frontmatter") - # Sanity: filename should match name - expected_filename = f"{name}{AGENT_FILE_SUFFIX}" - if path.name != expected_filename: + # Sanity: filename should match name under one accepted suffix + # (.agent.md ≤v3.8.3, .md from v3.8.4). A stray non-agent *.md in + # agents/ (e.g. a README) fails loud here — intended. + accepted = {f"{name}{sfx}" for sfx in AGENT_FILE_SUFFIXES} + if path.name not in accepted: raise ValueError( - f"{path}: filename does not match name (expected {expected_filename!r})" + f"{path}: filename does not match frontmatter name {name!r} " + f"(expected one of {sorted(accepted)})" ) records.append( diff --git a/converter/generate_wrappers.py b/converter/generate_wrappers.py index b3a11ca..645fe31 100644 --- a/converter/generate_wrappers.py +++ b/converter/generate_wrappers.py @@ -7,18 +7,19 @@ security-sensitive code, without re-paying the persistent-agent registration cost. -Measured upstream baseline (compound-engineering-v3.8.3, May 2026): -49 agent files totalling ~308KB ≈ ~77k tokens of body text (avg ~1.6k -tokens/agent), plus frontmatter (~67 tokens × 49 ≈ ~3.3k tokens). If -Claude Code loads the full body into idle context for every registered -agent — which the lived experience of pre-ce-lite session bloat and the -community-cited "~1.2k tokens per agent registration" figure both -suggest — ce-lite's deferral saves ~71k tokens (factoring in ~5.3k -tokens of SKILL.md descriptions ce-lite ships idle). If Claude loads -only frontmatter into the registration slot, the savings collapse to -roughly break-even. The prior "~58.8k baseline" in this docstring was a -midpoint estimate; treat any specific number as approximate until -empirically A/B'd against a vanilla install. +Measured via interactive `/context` (Opus 4.7, isolated CLAUDE_CONFIG_DIRs, +only the installed plugin differs): Claude Code loads each registered +agent's full body into the "Custom agents" idle slot — ~2k tokens/agent. +Upstream compound-engineering-v3.8.4 reads ~88k in that slot; ce-lite ships +zero registrations, so it's empty. Net idle saving ≈ 94k (the ~88k agent +slot + ~13.5k fewer agent descriptions in the Task-tool schema, less ~7.5k +of SKILL.md descriptions ce-lite ships idle). The total scales with how many +agents upstream registers (~2k each). (The earlier hedge here — "savings collapse to +break-even if only frontmatter loads" — is resolved: full bodies do load +idle. The prior "~58.8k"/"~71k" figures were char-count approximations, +superseded by this measurement. NB `claude plugin details` projects only +the ~50-tok/agent description as "always-on" and misses this slot — trust +`/context`.) Dispatch lookup is handled by a single small Python shim at `dist/bin/ce-lite-persona` (this directory is on PATH because Claude Code @@ -819,9 +820,10 @@ def write_persona_resolver(dist: Path) -> Path: # ---- UserPromptSubmit hook (Phase B.7) ---- # -# Phase B.5's meta-agent (ce-specialist) didn't fire autonomously — the -# integration eval (Phase B.6) measured 0/2 on autonomous review prompts even -# with the agent registered and visible to claude -p. Industry-wide +# Phase B.5's meta-agent (ce-specialist) didn't fire autonomously. A headless +# `claude -p` probe (Phase B.6) hinted at ~0/2 on autonomous review prompts with +# the agent registered — but headless routing is unreliable, so weight it +# lightly. Industry-wide # research (scottspence.com, paddo.dev, jefflester/claude-skills-supercharged, # spences10/claude-skills-cli) converges on the same finding: skills/agents # auto-activate ~20% of the time even with good descriptions, because Claude diff --git a/converter/scripts/measure-baseline.py b/converter/scripts/measure-baseline.py index 83f01f7..b95867c 100755 --- a/converter/scripts/measure-baseline.py +++ b/converter/scripts/measure-baseline.py @@ -5,11 +5,15 @@ # /// """A/B measure idle-context cost: upstream CE vs ce-lite vs bare. -The `generate_wrappers.py` docstring claims ~71k tokens of idle context savings -from ce-lite's deferred-loading design, conditional on Claude Code loading agent -bodies into the registration slot. This script answers the open question -empirically by running `claude -p` against three isolated config dirs and -reading `usage.input_tokens` from the JSON envelope. +Interactive `/context` is the authoritative measurement of this saving (≈94k vs +upstream; ~88k in the "Custom agents" slot on upstream v3.8.4 — Claude Code +loads each agent's full body into that idle slot, ~2k/agent). + +This script sketches a headless A/B via `claude -p` (three isolated config dirs, +reading `usage.input_tokens`), but the headless `-p` path does NOT reliably +reproduce interactive idle-context loading — treat it as scaffolding, not a +working test, and do not cite its output as confirmation. Use the interactive +`/context` A/B. Invocation pattern follows ~/.claude/memory/claude_p_headless_subscription.md: NO `--system-prompt` (we want the default Claude Code system prompt loaded diff --git a/converter/validate.py b/converter/validate.py index 7b44077..2dfa011 100644 --- a/converter/validate.py +++ b/converter/validate.py @@ -539,10 +539,13 @@ def check_round_trip(dist: Path, upstream: Path, manifest_names: set[str]) -> No upstream_agents = plugin_root / "agents" dist_prompts = dist / "references" / "agent-prompts" for name in sorted(manifest_names): - upstream_file = upstream_agents / f"{name}.agent.md" + # v3.8.4 renamed agents .agent.md → .md; accept either. + upstream_file = upstream_agents / f"{name}.md" + if not upstream_file.is_file(): + upstream_file = upstream_agents / f"{name}.agent.md" dist_file = dist_prompts / f"{name}.md" if not upstream_file.is_file(): - fail(f"round-trip: upstream missing {upstream_file}") + fail(f"round-trip: upstream missing agents/{name}.md (or .agent.md)") upstream_text = upstream_file.read_text(encoding="utf-8") try: diff --git a/tests/test_converter.py b/tests/test_converter.py index 74d771d..58c55cd 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -23,7 +23,11 @@ sys.path.insert(0, str(CONVERTER_DIR)) from DISPATCH_PATTERNS import AGENT_REFERENCE_RE # noqa: E402 -from extract import find_plugin_root, parse_frontmatter # noqa: E402 +from extract import ( # noqa: E402 + extract_agents, + find_plugin_root, + parse_frontmatter, +) from rewrite import ( # noqa: E402 PREAMBLE_MARKER_BEGIN, PREAMBLE_MARKER_END, @@ -31,6 +35,44 @@ ) +# -------- extract_agents: discovery across the v3.8.4 agent-file rename -------- + + +def _write_agent(agents_dir: Path, filename: str, name: str, body: str) -> None: + agents_dir.mkdir(parents=True, exist_ok=True) + (agents_dir / filename).write_text( + f"---\nname: {name}\ndescription: d\nmodel: inherit\ntools: Read, Grep\n---\n{body}\n", + encoding="utf-8", + ) + + +def test_extract_agents_v384_plain_md(tmp_path): + """v3.8.4 renamed agents .agent.md → .md; extract must find them.""" + root = tmp_path / "plugin" + _write_agent(root / "agents", "ce-foo.md", "ce-foo", "FOO BODY") + records = extract_agents(root) + assert [r.name for r in records] == ["ce-foo"] + assert records[0].body.strip() == "FOO BODY" + assert records[0].source_path == "agents/ce-foo.md" + + +def test_extract_agents_legacy_agent_md_still_works(tmp_path): + """The ≤v3.8.3 .agent.md layout must remain supported.""" + root = tmp_path / "plugin" + _write_agent(root / "agents", "ce-bar.agent.md", "ce-bar", "BAR BODY") + records = extract_agents(root) + assert [r.name for r in records] == ["ce-bar"] + assert records[0].source_path == "agents/ce-bar.agent.md" + + +def test_extract_agents_rejects_filename_name_mismatch(tmp_path): + """Filename must match the frontmatter name under an accepted suffix.""" + root = tmp_path / "plugin" + _write_agent(root / "agents", "ce-foo.md", "ce-different", "BODY") + with pytest.raises(ValueError, match="does not match frontmatter name"): + extract_agents(root) + + # -------- parse_frontmatter --------