From eae7230a6645afe4dd00904a1f5675ae8f0ac3ec Mon Sep 17 00:00:00 2001 From: Poytr1 Date: Sat, 8 Aug 2026 22:30:07 +0800 Subject: [PATCH 1/3] eval(arena): mention-gated production-fidelity option for counting games (local measurement run) --- evals/games/counting.ts | 34 +++++++++++++++++++++++++++++----- evals/games/engine.ts | 16 ++++++++++++++-- evals/games/quota-counting.ts | 30 +++++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/evals/games/counting.ts b/evals/games/counting.ts index e86587a9b..e7267fe1a 100644 --- a/evals/games/counting.ts +++ b/evals/games/counting.ts @@ -76,6 +76,11 @@ export interface CountingGameOptions { variant?: CountingVariant /** Human persona the referee speaks through on the real ingress path. */ refereeUserId?: string + /** Room routing convention (default `auto`, the historical game-room shape). + * `mention` is the PRODUCTION shared-channel convention: activation needs an + * explicit mention or thread affinity — the kickoff then @mentions every + * member, exactly as a human does in a live multi-agent Slack channel. */ + bindMatch?: 'auto' | 'mention' } interface AcceptedCandidate { @@ -111,6 +116,7 @@ export class CountingGame implements CollaborationGameWorld { private readonly echo?: PlatformEcho private readonly variant: CountingVariant + private readonly bindMatch: 'auto' | 'mention' constructor(options: CountingGameOptions) { this.world = options.world @@ -120,11 +126,22 @@ export class CountingGame implements CollaborationGameWorld { this.target = options.target ?? 12 this.variant = options.variant ?? 'referee-announced' this.refereeUserId = options.refereeUserId ?? 'W-ARENA-REFEREE' - this.environment = options.world.buildEnvironment() + this.bindMatch = options.bindMatch ?? 'auto' + this.environment = options.world.buildEnvironment({ bindMatch: this.bindMatch }) this.echo = this.variant === 'peer-driven' ? new PlatformEcho(options.world, room) : undefined } - private roomBroadcast(text: string): GameWave { + /** Platform-native mention tokens for every member bot (the kickoff shape a + * human uses in a production shared channel: one message @mentioning the + * participating agents). */ + private memberMentions(): { tokens: string; botUserIds: string[] } { + const botUserIds = this.room.memberIntegrationIds + .map((integrationId) => this.world.botUserIdFor(integrationId)) + .filter((id): id is string => id !== undefined) + return { tokens: botUserIds.map((id) => `<@${id}>`).join(' '), botUserIds } + } + + private roomBroadcast(text: string, options: { mentions?: string[] } = {}): GameWave { // ONE platform message id shared by every member integration's copy — the // same channel:ts each dedicated Slack app receives; per-connection dedup // (scoped by transport) must admit each copy exactly once. @@ -143,7 +160,8 @@ export class CountingGame implements CollaborationGameWorld { thread: this.room.thread, messageId, text, - sender: { id: this.refereeUserId, isBot: false } + sender: { id: this.refereeUserId, isBot: false }, + ...(options.mentions !== undefined && options.mentions.length > 0 ? { mentions: options.mentions } : {}) } })) this.world.appendEvent({ @@ -183,13 +201,19 @@ export class CountingGame implements CollaborationGameWorld { if (!this.started) { this.started = true if (this.variant === 'peer-driven') { + // Production kickoff shape: in a mention-gated shared channel the human + // @mentions the participants in the one start message; in the legacy + // auto-bound game room the bare text suffices. + const mention = this.bindMatch === 'mention' ? this.memberMentions() : undefined + const prefix = mention !== undefined && mention.tokens.length > 0 ? `${mention.tokens} ` : '' return this.roomBroadcast( - `Let's play the counting game. Together, count from 1 to ${this.target} in this thread by continuing ` + + `${prefix}Let's play the counting game. Together, count from 1 to ${this.target} in this thread by continuing ` + `each other's messages. When you see a number posted in this thread, reply with ONLY the next number — ` + `nothing else. Do not repeat a number that was already posted. If you posted the most recent number, ` + `prefer letting another participant continue, but keep the count moving. Stop once ${this.target} has ` + `been posted. No number has been posted yet, so the first reply should be 1. The referee stays silent ` + - `from now on and only checks the sequence at the end.` + `from now on and only checks the sequence at the end.`, + mention !== undefined ? { mentions: mention.botUserIds } : {} ) } return this.roomBroadcast( diff --git a/evals/games/engine.ts b/evals/games/engine.ts index 666179db6..873d4aec5 100644 --- a/evals/games/engine.ts +++ b/evals/games/engine.ts @@ -64,6 +64,9 @@ export interface CountingGameRunOptions { /** What drives the waves: §10.1 referee announcements (default) or §3.3 * peer-message relays with a silent referee. */ variant?: CountingVariant + /** Room routing convention: historical `auto` (default) or the production + * shared-channel `mention` gate with an @mention-bearing human kickoff. */ + bindMatch?: 'auto' | 'mention' /** Who plays (§8.1): scripted hosts (default; the reproducible engine gate) * or a real-runtime subject template — the identical game either way. */ subject?: GameSubjectSpec @@ -635,6 +638,9 @@ export interface QuotaCountingRunOptions { timeoutMs?: number subject?: GameSubjectSpec keepSubject?: boolean + /** Room routing convention: historical `auto` (default) or the production + * shared-channel `mention` gate with an @mention-bearing human kickoff. */ + bindMatch?: 'auto' | 'mention' } export async function runQuotaCounting(options: QuotaCountingRunOptions): Promise { @@ -645,7 +651,12 @@ export async function runQuotaCounting(options: QuotaCountingRunOptions): Promis const subjectSpec: GameSubjectSpec = options.subject ?? { kind: 'scripted' } const topology = compileTopology(countingManifest({ seed, agents })) const world = new ArenaWorld(topology) - const game = new QuotaCountingGame({ world, roomAlias: 'counting-room', quotaPerAgent }) + const game = new QuotaCountingGame({ + world, + roomAlias: 'counting-room', + quotaPerAgent, + ...(options.bindMatch !== undefined ? { bindMatch: options.bindMatch } : {}) + }) const subject = await prepareSubjectForRun(topology, subjectSpec) try { const runner = new CollaborationGameRunner({ @@ -683,7 +694,8 @@ export async function runSameRoomCounting(options: CountingGameRunOptions): Prom world, roomAlias: 'counting-room', variant, - ...(options.target !== undefined ? { target: options.target } : {}) + ...(options.target !== undefined ? { target: options.target } : {}), + ...(options.bindMatch !== undefined ? { bindMatch: options.bindMatch } : {}) }) const subject = await prepareSubjectForRun(topology, subjectSpec) try { diff --git a/evals/games/quota-counting.ts b/evals/games/quota-counting.ts index a1d94303f..faf02b9e1 100644 --- a/evals/games/quota-counting.ts +++ b/evals/games/quota-counting.ts @@ -39,6 +39,11 @@ export interface QuotaCountingGameOptions { /** Posts each participant must contribute (default 5). */ quotaPerAgent?: number refereeUserId?: string + /** Room routing convention (default `auto`, the historical game-room shape). + * `mention` is the PRODUCTION shared-channel convention: activation needs an + * explicit mention or thread affinity — the kickoff then @mentions every + * member, exactly as a human does in a live multi-agent Slack channel. */ + bindMatch?: 'auto' | 'mention' } interface Contribution { @@ -77,6 +82,7 @@ export class QuotaCountingGame implements CollaborationGameWorld { private readonly liveHandles: DeliveryHandle[] = [] /** Propagation: the production platform echo (§2.3/§5/§6). */ private readonly echo: PlatformEcho + private readonly bindMatch: 'auto' | 'mention' constructor(options: QuotaCountingGameOptions) { this.world = options.world @@ -87,7 +93,8 @@ export class QuotaCountingGame implements CollaborationGameWorld { if (this.quota < 1) throw new Error('quotaPerAgent must be at least 1') this.target = this.quota * room.memberAgentIds.length this.refereeUserId = options.refereeUserId ?? 'W-ARENA-REFEREE' - this.environment = options.world.buildEnvironment() + this.bindMatch = options.bindMatch ?? 'auto' + this.environment = options.world.buildEnvironment({ bindMatch: this.bindMatch }) this.echo = new PlatformEcho(options.world, room) for (const memberId of room.memberAgentIds) this.contributionsByAgent.set(memberId, 0) } @@ -96,7 +103,16 @@ export class QuotaCountingGame implements CollaborationGameWorld { return this.room.memberAgentIds.map((agentId) => this.world.aliasOfAgent(agentId)) } - private roomBroadcast(text: string): GameWave { + /** Platform-native mention tokens for every member bot (the kickoff shape a + * human uses in a production shared channel). */ + private memberMentions(): { tokens: string; botUserIds: string[] } { + const botUserIds = this.room.memberIntegrationIds + .map((integrationId) => this.world.botUserIdFor(integrationId)) + .filter((id): id is string => id !== undefined) + return { tokens: botUserIds.map((id) => `<@${id}>`).join(' '), botUserIds } + } + + private roomBroadcast(text: string, options: { mentions?: string[] } = {}): GameWave { const messageId = this.world.mintMessageId(this.room.platform) this.world.registerRoomMessage(this.room.channel, messageId) this.world.recordThreadMessage(this.room.channel, this.room.thread, { @@ -112,7 +128,8 @@ export class QuotaCountingGame implements CollaborationGameWorld { thread: this.room.thread, messageId, text, - sender: { id: this.refereeUserId, isBot: false } + sender: { id: this.refereeUserId, isBot: false }, + ...(options.mentions !== undefined && options.mentions.length > 0 ? { mentions: options.mentions } : {}) } })) this.world.appendEvent({ @@ -145,14 +162,17 @@ export class QuotaCountingGame implements CollaborationGameWorld { nextDeliveries(): GameWave { if (!this.started) { this.started = true + const mention = this.bindMatch === 'mention' ? this.memberMentions() : undefined + const prefix = mention !== undefined && mention.tokens.length > 0 ? `${mention.tokens} ` : '' return this.roomBroadcast( - `Let's play quota counting. Participants: ${this.memberAliases().join(', ')}. ` + + `${prefix}Let's play quota counting. Participants: ${this.memberAliases().join(', ')}. ` + `Count upward from 1 in this thread by continuing each other's messages — reply with ONLY the next ` + `number, nothing else. Each participant must post exactly ${this.quota} numbers in total. You cannot ` + `post twice in a row — someone else must post before you may post again. The count ends at ` + `${this.target}, when everyone has posted ${this.quota}. Plan your turns so nobody is left holding ` + `posts no one can interleave. No number has been posted yet, so the first reply should be 1. The ` + - `referee stays silent from now on and only reviews the sequence at the end.` + `referee stays silent from now on and only reviews the sequence at the end.`, + mention !== undefined ? { mentions: mention.botUserIds } : {} ) } return this.pendingWaves.shift() ?? { platformEvents: [], refereeEvents: [] } From eafde365fa1c845e49e62aba9430f169b0f7c4ac Mon Sep 17 00:00:00 2001 From: Poytr1 Date: Fri, 7 Aug 2026 00:49:38 +0800 Subject: [PATCH 2/3] test(evals): encode the delegate-and-forward async-contract case in the arena MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user asked agent A to "send hello to agent b and forward reply". A woke B with `needsReply`, polled the returned child in the same turn, read `in-progress`, and then told the user "Agent B completed its turn but returned no message to forward" — a completion claim its own last observation contradicted. `needsReply` is a two-sided contract with only one side stated: the CHILD gets a standing report-back directive, the PARENT gets `{ok, wake, childSessionId}` and no statement that the call is asynchronous. `viewSessionStatus` then advises "prefer waiting for the child's reply", which no turn can do. Two layers: - `evals/test/delegate-and-forward.test.ts`, credential-free and added to `pnpm eval:collab:contracts`. Two green characterization pins record the surface as it is (the wake result's exact key set; a same-turn poll returning in-progress against a provably mid-turn child, via an explicit rendezvous). Five `it.fails(…)` pin the affordances that must change, each naming the file: the async contract in the wake result, the impossible "prefer waiting" advice, the missing "when polling IS appropriate", `done` conflating "turn ended" with "reported back", and a headless child's answer being dropped silently. - `evals/test/delegate-and-forward-real.test.ts`, real ACP runtime, NOT in the gate, reported as a rate over trials with transcripts under `.artifacts/`. Measured over 5 trials of real local Claude Code (sonnet): the parent-side failure did NOT reproduce — 0 status polls, 0 premature claims, A ended its turn saying it would forward. A different failure did, 2 of 5: a postless child session is headless, so a child that answers in prose instead of calling `sendMessage {sessionId}` has its answer discarded with no signal, and the parent waits forever. From the parent's seat that is literally "returned no message to forward". `RoutingFixture` gains real-subject support (no hostFactory, template preflight, per-seat description override), per-turn tool-call traces attributed by the daemon's own turnId, and the daemon's peer-wake delivery records. Co-Authored-By: Claude Opus 5 --- docs/designs/collaboration-arena-baseline.md | 224 ++++++++--- evals/test/delegate-and-forward-real.test.ts | 256 +++++++++++++ evals/test/delegate-and-forward.test.ts | 368 +++++++++++++++++++ evals/test/routing-fixture.ts | 266 +++++++++++--- package.json | 2 +- 5 files changed, 1020 insertions(+), 96 deletions(-) create mode 100644 evals/test/delegate-and-forward-real.test.ts create mode 100644 evals/test/delegate-and-forward.test.ts diff --git a/docs/designs/collaboration-arena-baseline.md b/docs/designs/collaboration-arena-baseline.md index f33a8c992..d74e08924 100644 --- a/docs/designs/collaboration-arena-baseline.md +++ b/docs/designs/collaboration-arena-baseline.md @@ -9,6 +9,13 @@ These scenarios **work** on the current implementation. The limitations in §6 a real and documented, but none of them is a defect discovered by the arena — each is a protection behaving as designed, or a capability the product does not have. +One exception, added later and stated here so the paragraph above is not read as +covering it: §3.5 / §5.5 (delegate-and-forward) **is** a defect the arena found. +A child woken postlessly with `needsReply` that answers in prose instead of +calling `sendMessage {sessionId}` has its answer discarded with no signal to +anyone, and the parent waits forever. It is pinned as an expected-fail, not as a +protection. + **Scope note.** This document describes and measures the landed implementation. It deliberately does not argue for design alternatives; that discussion belongs to the messaging-primitives counter-proposal, which builds on this baseline. @@ -193,6 +200,48 @@ failure. Measured leaks: **0**. See §6.4. The game is present and its referee semantics are covered, but the milestone cannot complete against the landed `sendMessage`. +### 3.5 Delegate and forward — the `needsReply` round trip + +`evals/test/delegate-and-forward.test.ts` (credential-free, in the gate) and +`evals/test/delegate-and-forward-real.test.ts` (real runtime, on demand). + +This case exists because of an observed production failure, not a hypothesis. A +user told agent A **"send hello to agent b and forward reply"**. A called +`sendMessage {toAgent:{agentId:,needsReply:true}, message:"hello"}`, got back +`{ok:true, wake:{delivered:true,…}, childSessionId:"webchat:a2a:…"}`, then — in +the same turn — called `viewSessionStatus` on that child, read +`{status:"in-progress", state:"prompting"}`, and told the user **"Agent B +completed its turn but returned no message to forward."** The last sentence is +contradicted by the state A had just read. + +Nothing in that trace is a routing fault: the wake was delivered and the child +did run. What the case measures is that **`needsReply` is a two-sided contract +with only one side stated**: + +| Side | What it is told | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Child** | A standing `# Reporting back to your parent session` block (`packages/daemon/src/session/session-manager.ts`) naming the parent session and the reply shape. | +| **Parent** | Nothing. Its tool result is `{ok, wake, childSessionId}` — three fields, no prose, no statement that the call was asynchronous. | + +The credential-free half pins the fixable affordances. Two of its tests are +ordinary characterization pins (green), and five are `it.fails(…)` — the repo's +expected-fail idiom: the assertion is written for the surface we want, it passes +while the surface still fails it, and it starts failing the moment the fix lands. +Each names the file and the change in a comment. + +| Assertion | Today | +| ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| The `needsReply` wake result states the async contract (reply arrives later; end your turn) | **Red.** The result is exactly `{ok, wake, childSessionId}` — measured, keys asserted. | +| `viewSessionStatus` does not advise an action a turn cannot take | **Red.** It ends with "Poll sparingly, and prefer waiting for the child's reply over a tight polling loop." A turn cannot wait; it can only end or block. | +| `viewSessionStatus` says when checking a child IS appropriate | **Red.** Nothing distinguishes "you are already awake for another reason" from "you are trying to synchronize". | +| The status result separates "its turn ended" from "it reported back" | **Red.** Measured by running both situations and comparing: minus the child's ids and the clock, both answer `{status:'done', state:'idle'}` — byte-identical. | +| A headless child's answer is not silently dropped when it never reports back | **Red.** Measured: the child's output reaches no platform effect (delivered or attempted) and no parent turn. See §5.5 — this is the failure the real runs actually produced. | + +The two green pins record the surface as it is, so the red ones are anchored in +measured behavior rather than a paraphrase: the wake result's exact key set, and +a same-turn poll returning `{status:'in-progress', state:'prompting'}` while the +child is provably mid-turn (an explicit rendezvous, not a sleep). + ## 4. Reproducing every result Node 24 (`.nvmrc`) and pnpm 11. No model credentials required. @@ -202,12 +251,13 @@ pnpm install pnpm build # protocol must be built before typecheck # the whole arena gate -pnpm eval:collab:contracts # 15 files, 115 tests +pnpm eval:collab:contracts # 16 files, 122 tests # the routing acceptance cases alone pnpm eval:collab:routing # routing-acceptance + connection-surface # individual scenarios +npx vitest run evals/test/delegate-and-forward.test.ts npx vitest run evals/test/counting.test.ts npx vitest run evals/test/quota-counting.test.ts npx vitest run evals/test/werewolf.test.ts @@ -228,6 +278,7 @@ Full gate, measured on this branch: | Suite | Tests | | ------------------------------------------------------ | ------- | | `evals/test/routing-acceptance.test.ts` | 8 | +| `evals/test/delegate-and-forward.test.ts` | 7 | | `evals/test/game-runner.test.ts` | 5 | | `evals/test/werewolf.test.ts` | 18 | | `evals/test/quota-counting.test.ts` | 8 | @@ -242,9 +293,18 @@ Full gate, measured on this branch: | `evals/test/virtual-connections.test.ts` | 4 | | `packages/daemon/test/evaluation-game-ingress.test.ts` | 5 | | `packages/daemon/test/evaluation-game-tools.test.ts` | 3 | -| **Total** | **115** | +| **Total** | **122** | -**0 expected-fail.** Every pin is an ordinary assertion. +**5 expected-fail**, all of them in `delegate-and-forward.test.ts` (§3.5) and all +naming a specific change to the tool surface. Every other pin in the gate is an +ordinary assertion. + +**One known flake, pre-existing and not from this work.** `werewolf.test.ts`'s +`SCRIPTED BOUNDARY: a seven-player game exhausts the budget inside one 60s window` +depends on the scripted game finishing inside the loop guard's real 60-second +window. On a loaded machine it does not, the budget refreshes mid-game, and the +test fails with `latched: 0` or `admitted: 20`. Reproduced on `main` @ 09d76132 +before any change here, and green on an unloaded run. ### 4.1 Running a game against a real ACP runtime @@ -562,6 +622,74 @@ The high regeneration and coalescing counts are the turn-final context refresh working: a concurrent turn pulls a fresher thread snapshot, sees a peer message it has not represented, and regenerates rather than posting a stale number. +### 5.5 Delegate and forward, real local Claude Code — the diagnosis did NOT reproduce + +Same runtime and model as §5.1–§5.3 (`npx -y @agentclientprotocol/claude-agent-acp@0.64.0`, +model pinned to `sonnet`, `permissionMode: default`, memory off, from-scratch +workspace), two seats in one mention-gated Slack-shaped room, five trials, seeds +901–905. Agent B is a real model too — the daemon's host seam is per-daemon, so a +run cannot mix a real runtime with a scripted one — but its behavior is fixed by +**configuration**: its `agent.json` description gives it a responder persona and a +per-trial marker it must include in its answer. A can never see that marker (an +agent's description is its own), which is what makes "A forwarded B's **actual** +reply" a hard assertion. + +```bash +pnpm --filter @agentconnect.md/daemon build +export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js" +export AGENTCONNECT_EVAL_SUBJECT_ROOT=/absolute/path/to/subject +export AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS= +npx vitest run evals/test/delegate-and-forward-real.test.ts +``` + +| Trial | Seed | Delegated | Polled the child | Premature claim | Child used `sendMessage {sessionId}` | Parent woken by the reply | Forwarded it | +| ----- | ---- | --------- | ---------------- | --------------- | ------------------------------------ | ------------------------- | ------------ | +| 1 | 901 | turn 1 | **0 calls** | no | **yes** | yes | yes | +| 2 | 902 | turn 1 | **0 calls** | no | **no** | **no** | **no** | +| 3 | 903 | turn 1 | **0 calls** | no | **yes** | yes | yes | +| 4 | 904 | turn 1 | **0 calls** | no | **no** | **no** | **no** | +| 5 | 905 | turn 1 | **0 calls** | no | **yes** | yes | yes | + +Rates: `noSameTurnPoll` **5/5**, `noPrematureClaim` **5/5**, `wokenByTheReply` +**3/5**, `forwardedTheReply` **3/5**. + +**The parent-side failure did not reproduce, and that has to be said plainly.** +In all five trials agent A discovered B with `listAgents`, sent +`{"toAgent":{"agentId":…,"needsReply":true},"message":"Hello!"}`, **called +`viewSessionStatus` zero times**, and ended its turn saying it was waiting — _"Sent +hello to agent-b — waiting on its reply, will forward once it comes back."_ It +inferred the asynchronous contract that nothing in the surface states. So the +missing affordances in §3.5 are real (they are measured facts about the tool +surface), but on this model, in this shape, they are not sufficient to produce the +polling-and-fabricating behavior. They raise the odds; they do not determine it. + +**A different failure reproduced instead, in 2 of 5 trials, and it is worse.** A +postless `toAgent` wake gives the child a **headless** session: an ordinary turn +reply from it is published nowhere. The report-back directive is therefore not a +courtesy — it is the child's **entire output channel**. In trials 2 and 4 the +child made no tool call at all and simply answered in prose +(_"Hello agent-a, hello back! B-REPLY-904-PWY37N"_). That answer reached nothing: +no platform effect, delivered or attempted, and no wake of the parent. Agent A ran +exactly **one** turn and was left waiting forever, with no signal that anything had +gone wrong. + +That is the same observable the production trace reported — "returned no message to +forward" — reached by a different mechanism, and from A's seat it was **true**. It +is now pinned as the fifth expected-fail in §3.5. + +**What worked, when it worked, was not quite what the child thought.** In trials 1, +3 and 5 the child did call `sendMessage {sessionId:}` — but it described +itself as having _"replied in-thread"_ and sent the parent a **summary** of that +reply rather than the reply. The summary happened to quote the marker verbatim, so +the forward carried B's real words; a child that summarized more loosely would have +delivered a paraphrase without either side noticing. The child believes it has two +channels; it has one. + +Trial artifacts (full turn-by-turn transcripts, per-turn tool calls, delivered +posts, and the per-trial summary) are written to +`.artifacts/evaluation/delegate-forward/` at mode 0600, redacted with the subject +template's secret set. They are deliberately not committed. + ## 6. Limitations ### 6.1 The raised hop cap is unreachable — the automatic-turn budget binds first @@ -783,44 +911,52 @@ before a future game depends on them: Stated explicitly, since several claims above are structural rather than observed: -| Claim | Basis | -| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| The 115 contract tests pass, credential-free | **Measured**, this branch | -| The four acceptance cases behave as tabulated | **Measured**, this branch | -| Werewolf plays to a winner with 0 canary leaks | **Measured**, this branch | -| Normalization is not exercised | **Measured** by reading `injectPlatformEvent` — it builds the `NormalizedMessage` itself | -| Only cross-room counting omits the echo | **Measured** — `PlatformEcho` is constructed in `counting.ts`, `quota-counting.ts` and `werewolf.ts` | -| Werewolf visibility admits dead players | **Measured** — the `visibleTo` predicate tests configured membership, not aliveness | -| Cross-room handoff is refused, naming `thread` | **Measured**, this branch (assertion on the product's own error text) | -| Leaderless rooms overshoot the target | **Measured**, this branch (scripted, 3 × 6) | -| 4-bot quota counting stalls 0/8 at 29 accepted / 10 started | **Measured**, this branch (scripted) | -| A 2-agent chain stops at 16 edges, not the hop cap of 20 | **Measured**, this branch (scripted A→B→A chain) | -| ...and the automatic-turn budget is what stops it | **Measured by construction** — raising only `MAX_AUTOMATIC_TURNS_PER_WINDOW` lets the same chain reach hop 20 | -| Werewolf's day advances on peer messages, not the referee | **Measured**, this branch — one referee event per day between open and close, each speech preceded by the previous speaker's echo | -| A sequential order stops at exactly 9 speakers per round | **Measured**, this branch (scripted, tables of 7/9/10/11/12, seeds 42 and 9) | -| ...because each latched player absorbed exactly 8 wakes | **Measured** — every latched circuit reports `admitted: 8` then `gated` | -| A referee (human) message resets the automatic counter | **Measured** — 7-player players absorb 9–10 admitted automatic turns across a game with zero refusals; confirmed by reading `recordLoopGuardTurn` | -| A latched player cannot receive the referee's VOTE either | **Measured** — `incompleteVotes ≥ 1` on every stalled table | -| Real Claude Code speaks strictly in order when it speaks | **Measured**, this branch — 0 out-of-order speeches across 5 trials (§5.1) | -| ...but carries the order to the end in only 3 of 5 trials | **Measured**, this branch (§5.1); both stalls had 0 gated wakes and 0 latches, so no protection was involved | -| A real subject with a bad MCP bridge entry loses ALL tools | **Measured**, this branch (§5.1 trial B) — now refused by the preflight | -| Real Claude Code plays Werewolf to a winner, multi-round | **Measured**, this branch (§5.2) — 4 games, 4 winners, 1–3 rounds, 0 leaks | -| A 5-player table can end at night 1 with no day at all | **Measured** (§5.2 trial 2) — 2 wolves vs 3 others, one kill makes it 2-vs-2 | -| The per-round budget reset carries a multi-round game | **Measured** — 48–96 admitted automatic turns per game, 0 gated | -| A real subject's tool calls were denied INSIDE the runtime | **Measured** — the denial text names `don't ask mode`, and the daemon logs zero `permission.*` events for the whole run | -| Werewolf's actions are messages, parsed per conversation | **Measured**, this branch — the game registers no evaluation tools at all | -| A non-wolf cannot even post a kill statement in the den | **Measured** — the world's §7.2 authorization rejects the post before parsing | -| Real agents state actions clearly enough to parse | **Measured** — `unparseableActions: 0` in both real 7p trials | -| The wolves actually negotiate in the den | **Measured** — proposal → rationale → "any objections?" → agreement, with the redundant confirmations deduped | -| Scripted 7p collapses inside ONE 60s loop-guard window | **Measured**, this branch — every circuit latches at `admitted: 8` | -| ...and that is scripted SPEED, not the design | **Measured** — real 7p rounds span 90–128 s, the window rolls, and all 3 trials complete with 0 gated | -| Real games still leave votes uncast | **Measured** — 3 and 2 votes of 6 eligible, with 0 gated and 0 latches, so model behavior not protection | -| ...so the daemon's auto-allow set was never the blocker | **Measured** — an earlier revision of this document claimed it was; the request never reaches the daemon at all | -| Workspace `.claude/settings.json` pre-approval is ignored | **Measured** — files written and confirmed surviving the run, tools still denied | -| `CLAUDE_CONFIG_DIR` relocation breaks the runtime's auth | **Measured** — `infra_error`, 0 peer wakes, 32s collapse | -| Real-model 2 × 20 → 10, entropy 1.0 | **Measured on `main` @ 87d36bc, when the hop cap was 8**; carried forward unrefreshed | -| Real-model 4 × 8 → 8/8, entropy 0.70 | **Measured on `main` @ 87d36bc**, carried forward unrefreshed | -| The same real-model run would now stop at 16 edges | **Inferred** from the scripted chain result; no real-model run has been done since #628 | -| A long leaderless count still cannot finish unaided | **Inferred** from the 16-edge bound plus the absence of any override | -| No hop-cap or loop-guard override exists | **Measured** by exhaustive grep of config schema, CP env, and `.env.example` | -| Participation unfairness under fan-out | **Measured** (agents that never spoke) — the _cause_ attributed to scheduling order is **inferred** | +| Claim | Basis | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| The 115 contract tests pass, credential-free | **Measured**, this branch | +| The four acceptance cases behave as tabulated | **Measured**, this branch | +| Werewolf plays to a winner with 0 canary leaks | **Measured**, this branch | +| Normalization is not exercised | **Measured** by reading `injectPlatformEvent` — it builds the `NormalizedMessage` itself | +| Only cross-room counting omits the echo | **Measured** — `PlatformEcho` is constructed in `counting.ts`, `quota-counting.ts` and `werewolf.ts` | +| Werewolf visibility admits dead players | **Measured** — the `visibleTo` predicate tests configured membership, not aliveness | +| Cross-room handoff is refused, naming `thread` | **Measured**, this branch (assertion on the product's own error text) | +| Leaderless rooms overshoot the target | **Measured**, this branch (scripted, 3 × 6) | +| 4-bot quota counting stalls 0/8 at 29 accepted / 10 started | **Measured**, this branch (scripted) | +| A 2-agent chain stops at 16 edges, not the hop cap of 20 | **Measured**, this branch (scripted A→B→A chain) | +| ...and the automatic-turn budget is what stops it | **Measured by construction** — raising only `MAX_AUTOMATIC_TURNS_PER_WINDOW` lets the same chain reach hop 20 | +| Werewolf's day advances on peer messages, not the referee | **Measured**, this branch — one referee event per day between open and close, each speech preceded by the previous speaker's echo | +| A sequential order stops at exactly 9 speakers per round | **Measured**, this branch (scripted, tables of 7/9/10/11/12, seeds 42 and 9) | +| ...because each latched player absorbed exactly 8 wakes | **Measured** — every latched circuit reports `admitted: 8` then `gated` | +| A referee (human) message resets the automatic counter | **Measured** — 7-player players absorb 9–10 admitted automatic turns across a game with zero refusals; confirmed by reading `recordLoopGuardTurn` | +| A latched player cannot receive the referee's VOTE either | **Measured** — `incompleteVotes ≥ 1` on every stalled table | +| Real Claude Code speaks strictly in order when it speaks | **Measured**, this branch — 0 out-of-order speeches across 5 trials (§5.1) | +| ...but carries the order to the end in only 3 of 5 trials | **Measured**, this branch (§5.1); both stalls had 0 gated wakes and 0 latches, so no protection was involved | +| A real subject with a bad MCP bridge entry loses ALL tools | **Measured**, this branch (§5.1 trial B) — now refused by the preflight | +| Real Claude Code plays Werewolf to a winner, multi-round | **Measured**, this branch (§5.2) — 4 games, 4 winners, 1–3 rounds, 0 leaks | +| A 5-player table can end at night 1 with no day at all | **Measured** (§5.2 trial 2) — 2 wolves vs 3 others, one kill makes it 2-vs-2 | +| The per-round budget reset carries a multi-round game | **Measured** — 48–96 admitted automatic turns per game, 0 gated | +| A real subject's tool calls were denied INSIDE the runtime | **Measured** — the denial text names `don't ask mode`, and the daemon logs zero `permission.*` events for the whole run | +| Werewolf's actions are messages, parsed per conversation | **Measured**, this branch — the game registers no evaluation tools at all | +| A non-wolf cannot even post a kill statement in the den | **Measured** — the world's §7.2 authorization rejects the post before parsing | +| Real agents state actions clearly enough to parse | **Measured** — `unparseableActions: 0` in both real 7p trials | +| The wolves actually negotiate in the den | **Measured** — proposal → rationale → "any objections?" → agreement, with the redundant confirmations deduped | +| Scripted 7p collapses inside ONE 60s loop-guard window | **Measured**, this branch — every circuit latches at `admitted: 8` | +| ...and that is scripted SPEED, not the design | **Measured** — real 7p rounds span 90–128 s, the window rolls, and all 3 trials complete with 0 gated | +| Real games still leave votes uncast | **Measured** — 3 and 2 votes of 6 eligible, with 0 gated and 0 latches, so model behavior not protection | +| ...so the daemon's auto-allow set was never the blocker | **Measured** — an earlier revision of this document claimed it was; the request never reaches the daemon at all | +| Workspace `.claude/settings.json` pre-approval is ignored | **Measured** — files written and confirmed surviving the run, tools still denied | +| `CLAUDE_CONFIG_DIR` relocation breaks the runtime's auth | **Measured** — `infra_error`, 0 peer wakes, 32s collapse | +| The `needsReply` wake result is exactly `{ok, wake, childSessionId}` | **Measured**, this branch — key set asserted through the real MCP control socket (§3.5) | +| A same-turn poll of a running child returns `in-progress`/`prompting` | **Measured**, this branch — explicit rendezvous, so the child is provably mid-turn | +| `viewSessionStatus` advises waiting, which a turn cannot do | **Measured** by reading the shipped descriptor; that a turn cannot wait is **structural** (a turn ends or blocks) | +| `done` cannot be told from "reported back" | **Measured** — both situations run end-to-end and, minus ids and clock, return identical results | +| A headless child's prose answer reaches nothing at all | **Measured**, this branch — no world effect (delivered or attempted) and no parent turn (§3.5, §5.5) | +| Real Claude Code did NOT poll or fabricate on delegate-and-forward | **Measured**, this branch — 5 trials, `viewSessionStatus` called 0 times, 0 premature claims (§5.5) | +| ...but the reply was lost in 2 of 5 trials | **Measured** (§5.5) — the child answered only in prose into a headless session | +| The missing parent-side affordances are not sufficient to cause the bug | **Inferred** from those 5 trials — a negative result on one model in one room shape, not a proof that the affordances do not matter | +| Real-model 2 × 20 → 10, entropy 1.0 | **Measured on `main` @ 87d36bc, when the hop cap was 8**; carried forward unrefreshed | +| Real-model 4 × 8 → 8/8, entropy 0.70 | **Measured on `main` @ 87d36bc**, carried forward unrefreshed | +| The same real-model run would now stop at 16 edges | **Inferred** from the scripted chain result; no real-model run has been done since #628 | +| A long leaderless count still cannot finish unaided | **Inferred** from the 16-edge bound plus the absence of any override | +| No hop-cap or loop-guard override exists | **Measured** by exhaustive grep of config schema, CP env, and `.env.example` | +| Participation unfairness under fan-out | **Measured** (agents that never spoke) — the _cause_ attributed to scheduling order is **inferred** | diff --git a/evals/test/delegate-and-forward-real.test.ts b/evals/test/delegate-and-forward-real.test.ts new file mode 100644 index 000000000..6fd350baa --- /dev/null +++ b/evals/test/delegate-and-forward-real.test.ts @@ -0,0 +1,256 @@ +/** + * Arena case: DELEGATE AND FORWARD — the real-model half. + * + * The credential-free half (`delegate-and-forward.test.ts`, in + * `pnpm eval:collab:contracts`) pins the SYSTEM affordances. This file measures the + * BEHAVIOR they produce, and it is deliberately NOT in the CI gate: it needs a real + * ACP runtime and provider credentials, and a model result is a rate over trials, + * never a single pass/fail (collaboration-arena.md §8.1). + * + * The task is the one a user actually gave in production: + * + * @agent-a send hello to agent b and forward reply + * + * `agent-b` is a real model too — the daemon's host seam is per-daemon, so one run + * cannot mix a real runtime with a scripted one — but its behavior is fixed by + * CONFIGURATION rather than by scripting: its `agent.json` description gives it a + * responder persona and a per-trial token it must include in its reply. That token + * is what makes "A forwarded B's ACTUAL reply" a hard assertion instead of a + * judgement call, and A can never see it (an agent's description is its own). + * + * The four invariants the observed trace violated, measured per trial: + * + * 1. `noSameTurnPoll` — A does not call `viewSessionStatus` in the same turn as + * the `sendMessage` that started the child. + * 2. `noPrematureClaim` — A's first turn makes no completion claim about B, whose + * report provably had not arrived when that turn ended. + * 3. `wokenByTheReply` — A ends its turn and a LATER turn of A carries B's token. + * 4. `forwardedTheReply` — after that wake, A's visible post to the requester + * carries B's token. + * + * Every trial's turn-by-turn transcript is written under + * `.artifacts/evaluation/delegate-forward/` (mode 0600, redacted with the subject + * template's secrets), because a behavior claim without the trace behind it is not + * evidence. + * + * Run: + * pnpm --filter @agentconnect.md/daemon build + * export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js" + * export AGENTCONNECT_EVAL_SUBJECT_ROOT=/absolute/path/to/subject + * export AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS= + * npx vitest run evals/test/delegate-and-forward-real.test.ts + * + * The template must define an explicit runtime (local Claude Code over ACP for the + * measured runs), pin its model, and use `permissionMode: default` — a non-prompting + * mode makes the runtime deny every AgentConnect tool locally, before the daemon is + * ever consulted (collaboration-arena-baseline.md §5.1). + */ +import { mkdirSync } from 'node:fs' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { atomicWrite, redactEvaluationValue } from '../../packages/daemon/src/evaluation/index.js' +import { RoutingFixture, type RoutingTurnTrace } from './routing-fixture.js' + +const subjectRoot = process.env.AGENTCONNECT_EVAL_SUBJECT_ROOT?.trim() +const templateAgents = (process.env.AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS ?? '') + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean) +const configured = Boolean(subjectRoot) && templateAgents.length > 0 +const TRIALS = Number(process.env.AGENTCONNECT_EVAL_TRIALS ?? '5') +/** One trial is two-to-four real model turns plus two cold ACP starts. */ +const TRIAL_BUDGET_MS = Number(process.env.AGENTCONNECT_EVAL_TRIAL_BUDGET_MS ?? '420000') +const ARTIFACT_DIR = join(process.cwd(), '.artifacts', 'evaluation', 'delegate-forward') + +/** B's persona. Configuration, not a script: it fixes WHAT B answers with, and + * leaves every routing and turn-shaping decision to the model and the daemon. */ +const responderDescription = (token: string): string => + 'You are a responder agent. When another agent or a person greets you or asks you something, answer them ' + + 'directly and briefly in one or two sentences. Your answer must always contain the exact marker ' + + `${token} so the person who asked can tell your answer apart from anyone else's. Do not explain the ` + + 'marker and do not ask follow-up questions.' + +/** A completion/terminal claim about the delegate — the class of sentence the + * observed trace ended on ("Agent B completed its turn but returned no message to + * forward"). Matching any of these in the FIRST turn is the violation, because at + * that point B provably had not reported back. */ +const TERMINAL_CLAIM_PATTERNS: RegExp[] = [ + /returned no (message|reply|response|answer)/i, + /(no|without a) (message|reply|response|answer) to forward/i, + /(completed|finished|ended) its turn/i, + /did not (reply|respond|return|send)/i, + /has (not )?(replied|responded)/i, + /nothing (came )?back/i, + /(reply|response) was empty/i +] + +interface TrialResult { + trial: number + seed: number + token: string + noSameTurnPoll: boolean + noPrematureClaim: boolean + wokenByTheReply: boolean + forwardedTheReply: boolean + /** Did the child discharge its report-back obligation with an explicit + * `sendMessage {sessionId}`? A postless child session is HEADLESS, so a child + * that answers only in prose produces nothing at all — the measured reason + * `wokenByTheReply` fails when it does. */ + childUsedSessionReply: boolean + /** Which turn (1-based) issued the delegating `sendMessage`, if any. */ + delegationTurn?: number + viewSessionStatusCalls: number + turnsByA: number + notes: string[] +} + +let fixture: RoutingFixture | undefined + +afterEach(async () => { + await fixture?.stop() + fixture = undefined +}) + +function isStatusPoll(call: { name: string; arguments: unknown }): boolean { + return /viewSessionStatus$/.test(call.name) +} + +async function runTrial(trial: number): Promise<{ result: TrialResult; transcript: unknown }> { + const seed = 900 + trial + const token = `B-REPLY-${seed}-${Math.random().toString(36).slice(2, 8).toUpperCase()}` + fixture = await RoutingFixture.start({ + agents: ['agent-a', 'agent-b'], + scripts: {}, + seed, + subject: { kind: 'real', subjectRoot: subjectRoot!, templateAgentIds: templateAgents }, + agentDescriptions: { 'agent-b': responderDescription(token) }, + settleTimeoutMs: TRIAL_BUDGET_MS + }) + // The production phrasing, verbatim. It is synchronous-sounding on purpose: + // that is half of what the case is about. + const trigger = fixture.injectHuman(`<@${fixture.botUserId('agent-a')}> send hello to agent b and forward reply`, { + mentions: [fixture.botUserId('agent-a')] + }) + await fixture.settle(trigger.handles) + + const turnsA = fixture.turnTraces('agent-a') + const turnsB = fixture.turnTraces('agent-b') + const notes: string[] = [] + + // Which turn delegated, taken from the DAEMON's own delivery record rather + // than the runtime's tool-call reporting (a runtime may announce a call before + // its arguments are known, so the trace alone cannot prove `toAgent`). + const wakeTurnIds = new Set( + fixture + .peerWakesIssued('agent-a') + .filter((wake) => wake.admitted && wake.turnId !== undefined) + .map((wake) => wake.turnId!) + ) + const delegationIndex = turnsA.findIndex((turn) => turn.turnId !== undefined && wakeTurnIds.has(turn.turnId)) + const delegationTurn = delegationIndex >= 0 ? delegationIndex + 1 : undefined + if (delegationTurn === undefined) notes.push('agent-a never delivered a peer wake') + + // 1. No busy-poll inside the delegating turn. + const noSameTurnPoll = delegationIndex >= 0 ? !turnsA[delegationIndex]!.toolCalls.some(isStatusPoll) : false + + // 2. No completion claim in the first turn. B's report cannot have arrived + // yet — asserted, not assumed: the first turn's own output is checked, and + // a claim only counts as premature when no later turn had yet carried the + // token into A. + const firstOutput = turnsA[0]?.output ?? '' + const premature = TERMINAL_CLAIM_PATTERNS.filter((pattern) => pattern.test(firstOutput)) + const noPrematureClaim = premature.length === 0 + if (!noPrematureClaim) notes.push(`first turn asserted: ${premature.map(String).join(', ')}`) + + // 3. A ended its turn and was woken again, carrying B's actual reply. + const wakeTurn = turnsA.findIndex((turn, index) => index > 0 && turn.input.includes(token)) + const wokenByTheReply = wakeTurn > 0 + const childUsedSessionReply = turnsB.some((turn) => + turn.toolCalls.some( + (call) => + /sendMessage$/.test(call.name) && + (call.arguments as { sessionId?: unknown } | undefined)?.sessionId !== undefined + ) + ) + if (!wokenByTheReply) { + notes.push( + turnsB.length === 0 + ? 'agent-b never ran a turn' + : childUsedSessionReply + ? `agent-b reported to the parent session but its token never reached agent-a` + : `agent-b answered only in prose — its headless session had nowhere to put the answer` + ) + } + + // 4. The forward itself: a visible post by A, in the requester's room, that + // carries B's token. + const forwarded = fixture + .deliveredPosts() + .filter((post) => post.agentId !== undefined && fixture!.aliasOf(post.agentId) === 'agent-a') + .filter((post) => post.text.includes(token)) + const forwardedTheReply = forwarded.length > 0 + if (wokenByTheReply && !forwardedTheReply) notes.push('agent-a was woken with the reply but never forwarded it') + + const result: TrialResult = { + trial, + seed, + token, + noSameTurnPoll, + noPrematureClaim, + wokenByTheReply, + forwardedTheReply, + childUsedSessionReply, + ...(delegationTurn !== undefined ? { delegationTurn } : {}), + viewSessionStatusCalls: turnsA.reduce((total, turn) => total + turn.toolCalls.filter(isStatusPoll).length, 0), + turnsByA: turnsA.length, + notes + } + const transcript = { + result, + turns: { 'agent-a': turnsA, 'agent-b': turnsB } satisfies Record, + deliveredPosts: fixture.deliveredPosts().map((post) => ({ + author: post.agentId !== undefined ? fixture!.aliasOf(post.agentId) : undefined, + thread: post.thread, + text: post.text + })) + } + return { result, transcript } +} + +describe.skipIf(!configured)('delegate-and-forward against a real ACP runtime', () => { + it( + `runs ${TRIALS} trials of "send hello to agent b and forward reply" and reports each invariant as a rate`, + async () => { + mkdirSync(ARTIFACT_DIR, { recursive: true, mode: 0o700 }) + const results: TrialResult[] = [] + for (let trial = 1; trial <= TRIALS; trial += 1) { + const { result, transcript } = await runTrial(trial) + results.push(result) + atomicWrite( + join(ARTIFACT_DIR, `trial-${trial}.json`), + `${JSON.stringify(redactEvaluationValue(transcript, fixture?.secrets ?? []), null, 2)}\n` + ) + await fixture?.stop() + fixture = undefined + } + const rate = (key: keyof TrialResult) => results.filter((entry) => entry[key] === true).length + const summary = { + trials: results.length, + noSameTurnPoll: rate('noSameTurnPoll'), + noPrematureClaim: rate('noPrematureClaim'), + wokenByTheReply: rate('wokenByTheReply'), + forwardedTheReply: rate('forwardedTheReply'), + childUsedSessionReply: rate('childUsedSessionReply'), + results + } + atomicWrite(join(ARTIFACT_DIR, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`) + console.log(JSON.stringify(summary, null, 2)) + + // The only hard failure is an unusable run: a model rate is reported, not + // asserted (§8.1). A trial in which A never delegated at all measured + // nothing about the async contract. + expect(results.filter((entry) => entry.delegationTurn !== undefined).length).toBeGreaterThan(0) + }, + TRIAL_BUDGET_MS * (TRIALS + 1) + ) +}) diff --git a/evals/test/delegate-and-forward.test.ts b/evals/test/delegate-and-forward.test.ts new file mode 100644 index 000000000..e8cc8ea75 --- /dev/null +++ b/evals/test/delegate-and-forward.test.ts @@ -0,0 +1,368 @@ +/** + * Arena case: DELEGATE AND FORWARD — the async contract the caller is never told about. + * + * The observed production failure this file encodes (webchat, real model): + * + * 1. A human asks agent A: "send hello to agent b and forward reply". + * 2. A calls `sendMessage {toAgent:{agentId:,needsReply:true}, message:"hello"}` + * and gets back `{ok:true, wake:{delivered:true,targetSession:…}, childSessionId:…}`. + * 3. IN THE SAME TURN A calls `viewSessionStatus {sessionId:}` and + * reads `{status:"in-progress", state:"prompting"}`. + * 4. A then tells the human "Agent B completed its turn but returned no message to + * forward" — a completion claim its own last observation contradicts. + * + * Nothing in step 4 is a routing bug: the wake was delivered, the child did run, and + * the report-back directive was installed on the CHILD. What is missing is on the + * PARENT side. `needsReply` is a two-sided contract and only one side is stated: + * + * - the child is told (session-manager's `# Reporting back to your parent session`) + * that it must reply into the parent session; + * - the parent is told NOTHING. Its tool result is `{ok, wake, childSessionId}` — + * no statement that the call was asynchronous, that the reply will arrive as a + * later wake, or that the right move now is to end the turn. + * + * With a synchronous-sounding task ("…and forward reply"), an instant result, and + * `viewSessionStatus` as the only tool that looks like progress, polling and then + * inventing a terminal answer is the behavior the surface invites. + * + * LAYERS. This file is the credential-free half and runs in `pnpm eval:collab:contracts`. + * It pins the SYSTEM-side affordances, which are fixable in the tool surface. The + * model-behavior half is `delegate-and-forward-real.test.ts` (real ACP runtime, + * on demand, reported as a rate over trials). + * + * RED/GREEN. Tests written as `it.fails(…)` are the ones the surface does not satisfy + * today: they PASS while the affordance is still missing and start FAILING (flip them + * to `it`) the moment it lands. Each names what has to change. The `it(…)` tests around + * them are characterization pins: they record what the surface actually does now, so + * the red tests are anchored in measured behavior rather than in a paraphrase. + */ +import { afterEach, describe, expect, it } from 'vitest' +import { COLLABORATION_TOOLS, toolsForIntegrations } from '../../packages/daemon/src/mcp/tools.js' +import { RoutingFixture } from './routing-fixture.js' + +let fixture: RoutingFixture | undefined + +afterEach(async () => { + await fixture?.stop() + fixture = undefined +}) + +const sendMessageDescription = (): string => { + const tool = toolsForIntegrations([]).find((entry) => entry.name === 'sendMessage') + if (!tool) throw new Error('sendMessage descriptor is missing from the collaboration tool set') + return tool.description +} + +const viewSessionStatusDescription = (): string => { + const tool = COLLABORATION_TOOLS.find((entry) => entry.name === 'viewSessionStatus') + if (!tool) throw new Error('viewSessionStatus descriptor is missing from COLLABORATION_TOOLS') + return tool.description +} + +/** Every string anywhere in a tool result, so a prose assertion does not have to + * guess which field name a fix will choose. */ +function stringsIn(value: unknown, into: string[] = []): string[] { + if (typeof value === 'string') into.push(value) + else if (Array.isArray(value)) for (const entry of value) stringsIn(entry, into) + else if (value && typeof value === 'object') for (const entry of Object.values(value)) stringsIn(entry, into) + return into +} + +/** A status result stripped of everything that necessarily differs between two + * runs (the child's minted ids) or between two moments (the clock). */ +function withoutRunIdentity(status: Record): Record { + const { sessionId: _sessionId, agentId: _agentId, updatedAt: _updatedAt, ...rest } = status + return rest +} + +interface DelegationProbe { + /** Raw `sendMessage` result of the `needsReply` peer wake. */ + wakeResult: Record + /** Raw `viewSessionStatus` result read in the SAME turn as the wake. */ + sameTurnStatus: Record +} + +/** + * Reproduces the observed call sequence exactly, against the real daemon: A wakes + * B with `needsReply` and then — still inside that turn — polls the returned + * `childSessionId`, while B is provably mid-turn. + * + * The rendezvous is explicit rather than a sleep: B's script signals that its turn + * has begun and then blocks until A has read the status. That makes "the poll + * observed a running child" a fact of the test rather than a race. + */ +async function probeSameTurnPoll(): Promise { + let signalChildStarted: () => void = () => {} + const childStarted = new Promise((resolve) => { + signalChildStarted = resolve + }) + let releaseChild: () => void = () => {} + const childReleased = new Promise((resolve) => { + releaseChild = resolve + }) + const probe: Partial = {} + + fixture = await RoutingFixture.start({ + agents: ['agent1', 'agent2'], + scripts: { + agent1: async (ctx) => { + const wake = /DELEGATE ([0-9a-f-]{36})/.exec(ctx.text) + if (!wake) { + ctx.reply('noted') + return + } + const sent = await ctx.callTool('sendMessage', { + toAgent: { agentId: wake[1]!, needsReply: true }, + message: 'hello' + }) + probe.wakeResult = (sent.result ?? {}) as Record + const childSessionId = probe.wakeResult.childSessionId + expect(typeof childSessionId).toBe('string') + // Wait for B to be genuinely mid-turn, then poll exactly as the trace did. + await Promise.race([childStarted, new Promise((resolve) => setTimeout(resolve, 10_000))]) + const status = await ctx.callTool('viewSessionStatus', { sessionId: childSessionId }) + probe.sameTurnStatus = (status.result ?? {}) as Record + releaseChild() + ctx.reply('delegated') + }, + agent2: async (ctx) => { + if (!/hello/.test(ctx.text)) { + ctx.reply('agent2 heard you') + return + } + signalChildStarted() + await Promise.race([childReleased, new Promise((resolve) => setTimeout(resolve, 10_000))]) + ctx.reply('hi there') + } + } + }) + const trigger = fixture.injectHuman(`<@${fixture.botUserId('agent1')}> DELEGATE ${fixture.agentId('agent2')}`, { + mentions: [fixture.botUserId('agent1')] + }) + await fixture.settle(trigger.handles) + if (!probe.wakeResult || !probe.sameTurnStatus) throw new Error('the delegation probe did not complete') + return probe as DelegationProbe +} + +describe('delegate-and-forward — what the parent is told when it delegates', () => { + // CHARACTERIZATION (green): the exact surface the observed trace saw. If this + // ever changes, the red tests below are re-reading a different product. + it('records the surface as it is today: an instant wake result, and a same-turn poll that reports the child still running', async () => { + const probe = await probeSameTurnPoll() + + // The wake result. `wake.targetSession` and `childSessionId` are ids; nothing + // else in it is prose addressed to the caller. + expect(probe.wakeResult.ok).toBe(true) + expect(probe.wakeResult).toMatchObject({ wake: { delivered: true } }) + expect(typeof probe.wakeResult.childSessionId).toBe('string') + expect([...Object.keys(probe.wakeResult)].sort()).toEqual(['childSessionId', 'ok', 'wake']) + + // The same-turn poll — the tool the model reached for — answers that the + // child is still working. This is the observation the fabricated completion + // claim in the trace directly contradicted. + expect(probe.sameTurnStatus).toMatchObject({ status: 'in-progress', state: 'prompting' }) + }, 120_000) + + // RED — what must change: `executeTool`'s `sendMessage` branch + // (packages/daemon/src/mcp/ops.ts, the `toAgent` return around the + // `childSessionId` assembly) must, for a `needsReply` wake, return prose stating + // the asynchronous contract: the reply arrives later as a new turn in THIS + // session, and the caller should end its turn rather than wait or poll. Only the + // CHILD is told its half today (session-manager's `# Reporting back to your + // parent session`); the parent's half is unstated, which is what leaves "forward + // the reply" looking like a synchronous call that returned nothing. + it.fails( + 'states the asynchronous contract in the needsReply wake result', + async () => { + const probe = await probeSameTurnPoll() + const prose = stringsIn(probe.wakeResult).join(' \n ').toLowerCase() + // (i) the reply comes back later, as a wake of this session… + expect(prose).toMatch(/wake|woken|later turn|new turn|report back|when it (finishes|replies)/) + // (ii) …so the right move now is to finish this turn. + expect(prose).toMatch(/end (your|this) turn|finish (your|this) turn|do not wait|don’t wait|do not poll/) + }, + 120_000 + ) + + // RED — what must change: the `viewSessionStatus` description in + // packages/daemon/src/mcp/tools.ts currently closes with "Poll sparingly, and + // prefer waiting for the child’s reply over a tight polling loop." Inside a turn + // there is no way to wait: a turn either ends or blocks, and the model cannot + // block. Advising an impossible action is what turns "wait" into "poll once and + // then answer anyway". + it.fails('does not advise the caller to wait — an action no turn can take', () => { + expect(viewSessionStatusDescription().toLowerCase()).not.toMatch(/prefer waiting|wait for the child/) + }) + + // RED — same descriptor: having removed the impossible advice, it must say when + // the tool IS the right call. The honest cases are (a) you are already awake for + // some other reason and want a child's progress, and (b) the child you are + // checking is NOT the one whose reply woke you. Neither is expressible today. + it.fails( + 'says when checking a child IS appropriate (already awake; a child other than the one that woke you)', + () => { + const description = viewSessionStatusDescription().toLowerCase() + expect(description).toMatch(/already (awake|running)|when you are awake|woken for another/) + } + ) + + // RED — what must change: `SessionStatusResult` + // (packages/daemon/src/mcp/ops.ts) collapses two different facts into `done`: + // "the child's last turn ended" and "the child has reported back to you". A + // caller that asked for `needsReply` cares only about the second, and today it + // cannot tell them apart — the two situations are byte-identical apart from the + // `updatedAt` clock. That is measured here rather than asserted as a field name, + // so any shape that distinguishes them satisfies this test. + it.fails( + 'distinguishes "the child ended its turn" from "the child reported back"', + async () => { + const silent = await statusAfterChildTurn({ reportToParent: false }) + await fixture?.stop() + fixture = undefined + const reported = await statusAfterChildTurn({ reportToParent: true }) + expect(silent).toMatchObject({ status: 'done' }) + expect(reported).toMatchObject({ status: 'done' }) + // Run-specific identity and the wall clock are not answers to the caller's + // question, so they are removed before the comparison. Whatever remains is + // everything the tool actually TELLS a caller, and today it is identical in + // both situations. + expect(withoutRunIdentity(reported)).not.toEqual(withoutRunIdentity(silent)) + }, + 240_000 + ) + + // RED — and this one is not a hypothetical: it is what the real-model runs + // actually produced in 2 of 5 trials (collaboration-arena-baseline.md §5.5). + // + // A POSTLESS `toAgent` wake gives the child a HEADLESS session — nothing it says + // as an ordinary turn reply is published anywhere. The report-back directive + // therefore carries the child's ENTIRE output channel: a child that answers in + // prose instead of calling `sendMessage {sessionId}` has its answer discarded + // silently, and the parent, which was promised a report, waits forever with no + // signal that anything went wrong. From the parent's seat that is literally "it + // returned no message to forward". + // + // What must change (packages/daemon/src/daemon.ts, the turn-final path for a + // session with `needsParentReply`): when a headless child with an outstanding + // report-back obligation ends its turn without discharging it, the parent must + // be told something — the child's own output forwarded, or an explicit + // "finished without reporting" wake. Silently dropping the only thing the child + // produced is the one outcome that cannot be recovered from. + it.fails( + 'does not silently drop a headless child’s answer when it ends its turn without reporting back', + async () => { + let parentSawAnything = false + fixture = await RoutingFixture.start({ + agents: ['agent1', 'agent2'], + scripts: { + agent1: async (ctx) => { + const wake = /DELEGATE ([0-9a-f-]{36})/.exec(ctx.text) + if (wake) { + await ctx.callTool('sendMessage', { + toAgent: { agentId: wake[1]!, needsReply: true }, + message: 'hello' + }) + ctx.reply('delegated, waiting for the reply') + return + } + // Any LATER turn is the parent being told something about the child. + parentSawAnything = true + ctx.reply('noted') + }, + // The child does exactly what a real model did in trials 2 and 4: it + // answers, in prose, and ends its turn. No tool call. + agent2: (ctx) => { + ctx.reply('hi there — TOKEN-HEADLESS-DROP') + } + } + }) + const trigger = fixture.injectHuman(`<@${fixture.botUserId('agent1')}> DELEGATE ${fixture.agentId('agent2')}`, { + mentions: [fixture.botUserId('agent1')] + }) + await fixture.settle(trigger.handles) + + // The child did run and did produce an answer… + expect(fixture.activations('agent2')).toBe(1) + expect(fixture.turnTraces('agent2')[0]!.output).toContain('TOKEN-HEADLESS-DROP') + // …and the answer exists nowhere: not as a platform effect (delivered or + // even attempted), and not as anything the waiting parent was told. + const anywhere = fixture.world.allEffects().filter((effect) => /TOKEN-HEADLESS-DROP/.test(effect.text ?? '')) + expect(anywhere.length + (parentSawAnything ? 1 : 0)).toBeGreaterThan(0) + }, + 120_000 + ) +}) + +/** + * Drive one delegation to completion and read the child's status afterwards, with + * the child either reporting back into the parent session or staying silent. The + * read happens on a SECOND human turn injected into the SAME thread, because a + * child's status is only readable from the session that started it. + */ +async function statusAfterChildTurn(options: { reportToParent: boolean }): Promise> { + let childSessionId: string | undefined + let observed: Record | undefined + let delegated = false + fixture = await RoutingFixture.start({ + agents: ['agent1', 'agent2'], + scripts: { + agent1: async (ctx) => { + // CHECK first, and DELEGATE only once: a later turn's prompt replays the + // thread, so the original instruction is still visible in it. + if (/CHECK/.test(ctx.text)) { + if (childSessionId === undefined) throw new Error('CHECK turn ran before the delegation') + const status = await ctx.callTool('viewSessionStatus', { sessionId: childSessionId }) + observed = (status.result ?? {}) as Record + ctx.reply('checked') + return + } + const wake = /DELEGATE ([0-9a-f-]{36})/.exec(ctx.text) + if (wake && !delegated) { + delegated = true + const sent = await ctx.callTool('sendMessage', { + toAgent: { agentId: wake[1]!, needsReply: true }, + message: 'hello' + }) + childSessionId = (sent.result as { childSessionId?: string } | undefined)?.childSessionId + ctx.reply('delegated') + return + } + // The child's report-back resumes this session; say nothing, so the + // resumed turn cannot perturb what the CHECK turn later observes. + }, + agent2: async (ctx) => { + const parent = /Parent session: (\S+)/.exec(ctx.text) + if (/hello/.test(ctx.text) && options.reportToParent && parent) { + await ctx.callTool('sendMessage', { sessionId: parent[1]!, message: 'REPORT: hi there' }) + } + ctx.reply('hi there') + } + } + }) + const trigger = fixture.injectHuman(`<@${fixture.botUserId('agent1')}> DELEGATE ${fixture.agentId('agent2')}`, { + mentions: [fixture.botUserId('agent1')] + }) + await fixture.settle(trigger.handles) + // Same thread ⇒ same logical session for agent1, which is what authorizes the + // lineage read. + const check = fixture.injectHuman(`<@${fixture.botUserId('agent1')}> CHECK`, { + thread: trigger.messageId, + mentions: [fixture.botUserId('agent1')] + }) + await fixture.settle(check.handles) + if (!observed) throw new Error('the status probe never ran viewSessionStatus') + return observed +} + +describe('delegate-and-forward — the standing guidance the two sides receive', () => { + // CHARACTERIZATION (green): the sendMessage descriptor does tell the model to + // set `needsReply`, and it does point at `viewSessionStatus` — but it never says + // the call is asynchronous. Pinned because it is the exact asymmetry the red + // tests above are about. + it('tells the caller to set needsReply and to poll, but never that the call is asynchronous', () => { + const description = sendMessageDescription() + expect(description).toContain('needsReply') + expect(description).toContain('viewSessionStatus') + expect(description.toLowerCase()).not.toMatch(/asynchronous|end your turn|woken when|later turn/) + }) +}) diff --git a/evals/test/routing-fixture.ts b/evals/test/routing-fixture.ts index 848f7b49c..9b8f956a0 100644 --- a/evals/test/routing-fixture.ts +++ b/evals/test/routing-fixture.ts @@ -19,6 +19,8 @@ * (evaluation events), delivered/attempted world outbound effects, and thread * coordinates — not mechanism internals. */ +import { readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' import { SLACK_RESPONSE_FINAL_EVENT_TAG } from '../../packages/message/src/index.js' import type { DeliveryAdmission, @@ -33,11 +35,31 @@ import { type DaemonMcpBinding, type DaemonToolCallResult } from '../games/mcp-client.js' -import { prepareScriptedSubject } from '../games/subject.js' +import { type GameSubjectSpec, prepareGameSubject, preflightRealSubject } from '../games/subject.js' import { compileTopology } from '../games/topology.js' import type { CompiledRoom, CompiledTopology } from '../games/types.js' import { ArenaWorld } from '../games/world.js' +/** Scripted turns settle in milliseconds; a real ACP runtime does not. */ +const DEFAULT_SETTLE_TIMEOUT_MS = 30_000 + +/** + * Rewrite the prepared agents' `description` — the agent's own standing prompt + * seed, which the daemon renders into the `# Agent` block. This is the only + * per-seat lever a REAL-subject scenario gets: the model is never scripted, so + * a fixed counterpart persona has to be configuration, exactly as an operator + * would write it in `agent.json`. + */ +function applyAgentDescriptions(root: string, topology: CompiledTopology, descriptions: Record): void { + for (const [alias, description] of Object.entries(descriptions)) { + const agent = topology.agents.find((candidate) => candidate.alias === alias) + if (!agent) throw new Error(`description for unknown agent alias "${alias}"`) + const agentPath = join(root, 'agents', agent.agentId, 'agent.json') + const config = JSON.parse(readFileSync(agentPath, 'utf8')) as Record + writeFileSync(agentPath, `${JSON.stringify({ ...config, description }, null, 2)}\n`, { mode: 0o600 }) + } +} + export interface RoutingScriptContext { sessionId: string /** Full prompt text of this turn. */ @@ -53,14 +75,40 @@ export type RoutingScript = (context: RoutingScriptContext) => Promise | v export interface RoutingFixtureOptions { agents: string[] + /** Per-alias scripted behavior. Ignored entirely for a `real` subject: the + * runtime is the model, and nothing may script it. */ scripts: Record seed?: number + /** Who plays (games/subject.ts §8.1). Defaults to the credential-free + * scripted hosts every gate case uses. */ + subject?: GameSubjectSpec + /** Per-alias `description` override written onto the prepared agent.json. + * This is the agent's own standing prompt seed (it becomes part of the + * `# Agent` block), so it is the seam a real-subject scenario uses to give + * one seat a fixed persona without scripting its model. */ + agentDescriptions?: Record + /** Idleness budget for {@link RoutingFixture.settle}. Real runtimes need far + * more than the scripted default. */ + settleTimeoutMs?: number +} + +/** One model turn, reassembled from the ordered evaluation events: what was + * delivered into it, which tools it called IN THAT TURN, and what it said. */ +export interface RoutingTurnTrace { + turnId?: string + agentAlias: string + sessionId?: string + input: string + toolCalls: { id?: string; name: string; arguments: unknown }[] + output: string } export class RoutingFixture { readonly topology: CompiledTopology readonly world: ArenaWorld readonly room: CompiledRoom + /** Template values that must never reach an artifact this fixture's caller writes. */ + readonly secrets: readonly string[] private readonly harness: DaemonEvaluationHarness private readonly subjectCleanup: () => void private readonly echoHandles: DeliveryHandle[] = [] @@ -74,18 +122,23 @@ export class RoutingFixture { /** Thread each delivered message lives in (root posts anchor themselves). */ private readonly threadByMessageId = new Map() private readonly aliasByAgentId = new Map() + private readonly settleTimeoutMs: number private constructor( topology: CompiledTopology, world: ArenaWorld, harness: DaemonEvaluationHarness, - subjectCleanup: () => void + subjectCleanup: () => void, + settleTimeoutMs: number, + secrets: readonly string[] ) { this.topology = topology this.world = world this.room = topology.rooms[0]! this.harness = harness this.subjectCleanup = subjectCleanup + this.settleTimeoutMs = settleTimeoutMs + this.secrets = secrets for (const agent of topology.agents) this.aliasByAgentId.set(agent.agentId, agent.alias) } @@ -100,65 +153,83 @@ export class RoutingFixture { const world = new ArenaWorld(topology) // Production shared-channel convention: mention-gated, never `auto`. const environment = world.buildEnvironment({ bindMatch: 'mention' }) - const subject = prepareScriptedSubject(topology) + const subjectSpec: GameSubjectSpec = options.subject ?? { kind: 'scripted' } + const subject = prepareGameSubject(topology, subjectSpec) + if (options.agentDescriptions) { + applyAgentDescriptions(subject.root, topology, options.agentDescriptions) + } + // A real runtime reaches `sendMessage` through the `mcp-bridge` SUBPROCESS, + // and an unlaunchable runtime stalls silently. Both are refused up front. + if (subjectSpec.kind === 'real') await preflightRealSubject(subject.root) const scriptByAgentId = new Map() for (const [alias, script] of Object.entries(options.scripts)) { const agent = topology.agents.find((candidate) => candidate.alias === alias) if (!agent) throw new Error(`script for unknown agent alias "${alias}"`) scriptByAgentId.set(agent.agentId, script) } + // A real subject gets NO hostFactory: the daemon spawns the template's own + // ACP runtimes, exactly as it does in production. + const scriptedHostFactory = (agent: { id: string }, onUpdate: (sessionId: string, update: unknown) => void) => { + let sessions = 0 + const bindings = new Map() + return { + start: async () => {}, + newSession: async (_cwd: string, mcpServers?: unknown) => { + const sessionId = `routing-${agent.id.slice(0, 8)}-${(sessions += 1)}` + const binding = daemonMcpBinding(mcpServers) + if (binding) bindings.set(sessionId, binding) + return sessionId + }, + hasSession: () => true, + modelOptions: () => ({ current: 'scripted-routing', models: ['scripted-routing'] }), + prompt: async (sessionId: string, blocks: { text?: string }[]) => { + const text = blocks.map((block) => block.text ?? '').join('\n') + const script = scriptByAgentId.get(agent.id) + let replied = false + const context: RoutingScriptContext = { + sessionId, + text, + callTool: async (name, args) => { + const binding = bindings.get(sessionId) + if (!binding) throw new Error('session has no daemon tool binding') + return callDaemonTool(binding, name, args) + }, + reply: (value) => { + replied = true + onUpdate(sessionId, { + sessionUpdate: 'agent_message_chunk', + content: { type: 'text', text: value } + }) + } + } + if (script) await script(context) + if (!replied) { + // An empty turn posts nothing — the scripted stand-in for a + // production agent that chooses silence. + onUpdate(sessionId, { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: '' } }) + } + return { stopReason: 'end_turn' } + }, + cancel: async () => {}, + stop: async () => {} + } + } const harness = new DaemonEvaluationHarness({ root: subject.root, environment, runId: `routing-${seed}`, capabilityProfile: { memory: 'off' }, - hostFactory: ((agent: { id: string }, onUpdate: (sessionId: string, update: unknown) => void) => { - let sessions = 0 - const bindings = new Map() - return { - start: async () => {}, - newSession: async (_cwd: string, mcpServers?: unknown) => { - const sessionId = `routing-${agent.id.slice(0, 8)}-${(sessions += 1)}` - const binding = daemonMcpBinding(mcpServers) - if (binding) bindings.set(sessionId, binding) - return sessionId - }, - hasSession: () => true, - modelOptions: () => ({ current: 'scripted-routing', models: ['scripted-routing'] }), - prompt: async (sessionId: string, blocks: { text?: string }[]) => { - const text = blocks.map((block) => block.text ?? '').join('\n') - const script = scriptByAgentId.get(agent.id) - let replied = false - const context: RoutingScriptContext = { - sessionId, - text, - callTool: async (name, args) => { - const binding = bindings.get(sessionId) - if (!binding) throw new Error('session has no daemon tool binding') - return callDaemonTool(binding, name, args) - }, - reply: (value) => { - replied = true - onUpdate(sessionId, { - sessionUpdate: 'agent_message_chunk', - content: { type: 'text', text: value } - }) - } - } - if (script) await script(context) - if (!replied) { - // An empty turn posts nothing — the scripted stand-in for a - // production agent that chooses silence. - onUpdate(sessionId, { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: '' } }) - } - return { stopReason: 'end_turn' } - }, - cancel: async () => {}, - stop: async () => {} - } - }) as never + secrets: subject.secrets, + ...(subjectSpec.kind === 'scripted' ? { hostFactory: scriptedHostFactory as never } : {}) }) - const fixture = new RoutingFixture(topology, world, harness, subject.cleanup) + const fixture = new RoutingFixture( + topology, + world, + harness, + subject.cleanup, + options.settleTimeoutMs ?? DEFAULT_SETTLE_TIMEOUT_MS, + subject.secrets + ) // Production Slack echo: every delivered agent post fans back to the OTHER // member integrations under the author's managed bot identity. world.onDelivered((effect) => fixture.echoDeliveredPost(effect)) @@ -297,13 +368,13 @@ export class RoutingFixture { await Promise.all(pending.map((handle) => handle.completion)) pending = this.echoHandles.splice(0) } - await this.harness.waitUntilIdle() + await this.harness.waitUntilIdle(this.settleTimeoutMs) // Idle turns may have delivered posts whose echoes are still unsettled. pending = this.echoHandles.splice(0) while (pending.length > 0 && generations < 32) { generations += 1 await Promise.all(pending.map((handle) => handle.completion)) - await this.harness.waitUntilIdle() + await this.harness.waitUntilIdle(this.settleTimeoutMs) pending = this.echoHandles.splice(0) } } @@ -327,6 +398,99 @@ export class RoutingFixture { .map((event) => String(event.data.input ?? '')) } + /** + * Every model turn, reassembled from the ordered evaluation events. + * + * The per-turn TOOL CALL list is what a same-turn behavioral invariant needs + * ("did it poll the child in the very turn that started it?"), and it is only + * available here: `acp.update` carries the daemon's own `turnId`, so a call is + * attributed to the turn the daemon was running, never guessed from timing. + */ + turnTraces(alias?: string): RoutingTurnTrace[] { + const wanted = alias !== undefined ? this.agentId(alias) : undefined + const traces: RoutingTurnTrace[] = [] + const byTurnId = new Map() + const openByAgent = new Map() + for (const event of this.events()) { + if (event.agentId === undefined) continue + if (wanted !== undefined && event.agentId !== wanted) continue + if (event.type === 'turn.started') { + const trace: RoutingTurnTrace = { + ...(event.turnId !== undefined ? { turnId: event.turnId } : {}), + agentAlias: this.aliasOf(event.agentId), + ...(event.sessionId !== undefined ? { sessionId: event.sessionId } : {}), + input: String(event.data.input ?? ''), + toolCalls: [], + output: '' + } + traces.push(trace) + if (event.turnId !== undefined) byTurnId.set(event.turnId, trace) + openByAgent.set(event.agentId, trace) + continue + } + if (event.type !== 'acp.update') continue + const trace = + (event.turnId !== undefined ? byTurnId.get(event.turnId) : undefined) ?? openByAgent.get(event.agentId) + if (!trace) continue + const update = event.data.update as Record | undefined + if (!update) continue + if (update.sessionUpdate === 'agent_message_chunk') { + const text = (update.content as { text?: unknown } | undefined)?.text + if (typeof text === 'string') trace.output += text + continue + } + if (update.sessionUpdate !== 'tool_call' && update.sessionUpdate !== 'tool_call_update') continue + // A runtime may announce the call before its arguments are known and fill + // them in on the following `tool_call_update` for the same `toolCallId` + // (Claude Code's adapter does exactly that), so the two are merged. The + // NAME comes from `rawInput.tool` when the runtime reports MCP call + // structure and from the human `title` otherwise. + const rawInput = update.rawInput as Record | undefined + const callId = typeof update.toolCallId === 'string' ? update.toolCallId : undefined + const existing = callId !== undefined ? trace.toolCalls.find((call) => call.id === callId) : undefined + const name = + (typeof rawInput?.tool === 'string' ? rawInput.tool : undefined) ?? + (typeof update.title === 'string' ? update.title : undefined) + const args = rawInput?.arguments ?? (rawInput !== undefined && rawInput.tool === undefined ? rawInput : undefined) + if (existing) { + if (name !== undefined) existing.name = name + if (args !== undefined) existing.arguments = args + continue + } + if (update.sessionUpdate !== 'tool_call') continue + trace.toolCalls.push({ + ...(callId !== undefined ? { id: callId } : {}), + name: name ?? 'unknown_tool', + arguments: args ?? {} + }) + } + return traces + } + + /** + * Peer wakes an agent ISSUED, as the daemon recorded them + * (`collaboration.delivery.*`, stamped with the CALLER's agent id and the + * caller's own evaluation turn id). + * + * This is the authoritative answer to "which turn delegated": it comes from + * the daemon's delivery path rather than from the runtime's tool-call + * reporting, which is advisory and may announce a call before its arguments + * are known. + */ + peerWakesIssued(alias: string): { turnId?: string; admitted: boolean }[] { + const agentId = this.agentId(alias) + return this.events() + .filter( + (event) => + event.agentId === agentId && + (event.type === 'collaboration.delivery.admitted' || event.type === 'collaboration.delivery.rejected') + ) + .map((event) => ({ + ...(event.turnId !== undefined ? { turnId: event.turnId } : {}), + admitted: event.type === 'collaboration.delivery.admitted' + })) + } + /** Delivered, visible IM posts (agent speech — chrome excluded). */ deliveredPosts(): RecordedOutboundEffect[] { return this.world.allEffects().filter((effect) => effect.kind === 'reply' && effect.status === 'delivered') diff --git a/package.json b/package.json index 5ec012d97..1221315d7 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "eval:addons": "pnpm --filter @agentconnect.md/daemon build && node evals/run-addons.mjs", "eval:addons:view": "promptfoo view -n", "eval:collab": "pnpm --filter @agentconnect.md/daemon build && node evals/run-collaboration.mjs", - "eval:collab:contracts": "vitest run evals/test/routing-acceptance.test.ts evals/test/connection-surface.test.ts evals/test/virtual-connections.test.ts evals/test/world-authorization.test.ts evals/test/topology.test.ts evals/test/counting.test.ts evals/test/quota-counting.test.ts evals/test/cross-room-counting.test.ts evals/test/werewolf.test.ts evals/test/game-runner.test.ts evals/test/game-subject.test.ts evals/test/collaboration-game-provider.test.ts evals/test/game-result-assertion.test.ts packages/daemon/test/evaluation-game-ingress.test.ts packages/daemon/test/evaluation-game-tools.test.ts", + "eval:collab:contracts": "vitest run evals/test/routing-acceptance.test.ts evals/test/delegate-and-forward.test.ts evals/test/connection-surface.test.ts evals/test/virtual-connections.test.ts evals/test/world-authorization.test.ts evals/test/topology.test.ts evals/test/counting.test.ts evals/test/quota-counting.test.ts evals/test/cross-room-counting.test.ts evals/test/werewolf.test.ts evals/test/game-runner.test.ts evals/test/game-subject.test.ts evals/test/collaboration-game-provider.test.ts evals/test/game-result-assertion.test.ts packages/daemon/test/evaluation-game-ingress.test.ts packages/daemon/test/evaluation-game-tools.test.ts", "eval:collab:routing": "vitest run evals/test/routing-acceptance.test.ts evals/test/connection-surface.test.ts", "eval:collab:view": "promptfoo view -n", "eval:contracts": "vitest run packages/daemon/test/evaluation-events.test.ts packages/daemon/test/evaluation-atif.test.ts packages/daemon/test/evaluation-permission.test.ts packages/daemon/test/evaluation-runner.test.ts packages/daemon/test/daemon-evaluation.test.ts evals/test/outcome.test.ts evals/test/provider.test.ts evals/test/paired-summary.test.ts", From 80b47866cafb5c37ade771e9a3527202911fcb92 Mon Sep 17 00:00:00 2001 From: Poytr1 Date: Sun, 9 Aug 2026 10:40:21 +0800 Subject: [PATCH 3/3] test(evals): repin delegate-and-forward to the shipped parent-side contract; record the 2026-08-08 arena re-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parent-side half of the needsReply async contract shipped on main: the wake result carries reply/nextAction/message, viewSessionStatus is diagnostic-only, and SessionStatusResult.reply.state separates "turn ended" from "reported back". The former red pins are now green guard tests so none of it can regress; the one genuinely open pin — a headless child's prose answer silently dropped — stays expected-fail, and the 2026-08-08 real-model trial reproduced it (1/1). The baseline doc gains §5.6 (full real-model re-run, one trial per case, mention-gated production kickoff for the counting games), corrects §6.1 (the 16-edge automatic-turn bound is a scripted-speed artifact; a real 2-agent chain is bound by the hop cap — measured 20/20 with 19 hops used, 0 gated), and fixes the real-subject recipe (npx-launched user runtimes are filtered as not-installed; the runtime id must be claude-acp for the verified memory off-switch; launch the adapter via node). Co-Authored-By: Claude Fable 5 --- docs/designs/collaboration-arena-baseline.md | 229 ++++++++++++++----- evals/test/delegate-and-forward.test.ts | 164 ++++++------- 2 files changed, 251 insertions(+), 142 deletions(-) diff --git a/docs/designs/collaboration-arena-baseline.md b/docs/designs/collaboration-arena-baseline.md index d74e08924..1b3acb0e8 100644 --- a/docs/designs/collaboration-arena-baseline.md +++ b/docs/designs/collaboration-arena-baseline.md @@ -215,32 +215,38 @@ completed its turn but returned no message to forward."** The last sentence is contradicted by the state A had just read. Nothing in that trace is a routing fault: the wake was delivered and the child -did run. What the case measures is that **`needsReply` is a two-sided contract -with only one side stated**: +did run. What the case originally measured is that **`needsReply` was a +two-sided contract with only one side stated**: -| Side | What it is told | +| Side | What it was told at the time of the trace | | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | | **Child** | A standing `# Reporting back to your parent session` block (`packages/daemon/src/session/session-manager.ts`) naming the parent session and the reply shape. | -| **Parent** | Nothing. Its tool result is `{ok, wake, childSessionId}` — three fields, no prose, no statement that the call was asynchronous. | - -The credential-free half pins the fixable affordances. Two of its tests are -ordinary characterization pins (green), and five are `it.fails(…)` — the repo's -expected-fail idiom: the assertion is written for the surface we want, it passes -while the surface still fails it, and it starts failing the moment the fix lands. -Each names the file and the change in a comment. - -| Assertion | Today | -| ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| The `needsReply` wake result states the async contract (reply arrives later; end your turn) | **Red.** The result is exactly `{ok, wake, childSessionId}` — measured, keys asserted. | -| `viewSessionStatus` does not advise an action a turn cannot take | **Red.** It ends with "Poll sparingly, and prefer waiting for the child's reply over a tight polling loop." A turn cannot wait; it can only end or block. | -| `viewSessionStatus` says when checking a child IS appropriate | **Red.** Nothing distinguishes "you are already awake for another reason" from "you are trying to synchronize". | -| The status result separates "its turn ended" from "it reported back" | **Red.** Measured by running both situations and comparing: minus the child's ids and the clock, both answer `{status:'done', state:'idle'}` — byte-identical. | -| A headless child's answer is not silently dropped when it never reports back | **Red.** Measured: the child's output reaches no platform effect (delivered or attempted) and no parent turn. See §5.5 — this is the failure the real runs actually produced. | - -The two green pins record the surface as it is, so the red ones are anchored in -measured behavior rather than a paraphrase: the wake result's exact key set, and -a same-turn poll returning `{status:'in-progress', state:'prompting'}` while the -child is provably mid-turn (an explicit rendezvous, not a sleep). +| **Parent** | Nothing. Its tool result was `{ok, wake, childSessionId}` — three fields, no prose, no statement that the call was asynchronous. | + +The parent's half has since shipped (see the table below); the child's half — +what happens when a headless child never discharges the obligation — is still +open. + +The credential-free half pins the fixable affordances. When this case was first +written every parent-side affordance was missing and the file carried five +`it.fails(…)` pins. **The parent-side half has since SHIPPED on `main`**, and the +file was re-verified and repinned on 2026-08-08: what used to be red is now a set +of green guard tests so it cannot regress, and one genuinely open pin remains +red. + +| Assertion | Today | +| ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| The `needsReply` wake result states the async contract (reply arrives later; end your turn) | **Green guard.** The result now carries `reply: {requested, state}`, `nextAction: 'finish-turn-and-wait'`, and prose saying the reply arrives as a later turn of this session. | +| `viewSessionStatus` does not advise an action a turn cannot take | **Green guard.** The descriptor is now "diagnostic only; it never returns the reply body", with `nextAction` driving what the caller does next. (The earlier companion ask — phrasing for when checking IS appropriate — is superseded by that framing and its pin was retired.) | +| The status result separates "its turn ended" from "it reported back" | **Green guard.** `SessionStatusResult.reply.state` distinguishes them (`not-sent` vs `queued-for-parent`); the guard measures the whole result, not a field name. | +| The sendMessage descriptor stays lean: `needsReply` + follow `nextAction` | **Green guard.** The async contract is delivered at call time in the wake result, not restated in the always-loaded schema. | +| A headless child's answer is not silently dropped when it never reports back | **Red — the one remaining `it.fails`.** Measured: a child that answers in prose instead of `sendMessage {sessionId}` reaches no platform effect (delivered or attempted) and no parent turn. See §5.5/§5.6 — this is the failure real runs actually produce. | + +The surface-characterization guard anchors all of this in measured behavior: the +wake result's exact key set +(`childSessionId, message, nextAction, ok, reply, wake`) and a same-turn poll +returning `{status:'in-progress', state:'prompting'}` while the child is provably +mid-turn (an explicit rendezvous, not a sleep). ## 4. Reproducing every result @@ -251,7 +257,7 @@ pnpm install pnpm build # protocol must be built before typecheck # the whole arena gate -pnpm eval:collab:contracts # 16 files, 122 tests +pnpm eval:collab:contracts # 16 files, 121 tests # the routing acceptance cases alone pnpm eval:collab:routing # routing-acceptance + connection-surface @@ -278,7 +284,7 @@ Full gate, measured on this branch: | Suite | Tests | | ------------------------------------------------------ | ------- | | `evals/test/routing-acceptance.test.ts` | 8 | -| `evals/test/delegate-and-forward.test.ts` | 7 | +| `evals/test/delegate-and-forward.test.ts` | 6 | | `evals/test/game-runner.test.ts` | 5 | | `evals/test/werewolf.test.ts` | 18 | | `evals/test/quota-counting.test.ts` | 8 | @@ -293,11 +299,12 @@ Full gate, measured on this branch: | `evals/test/virtual-connections.test.ts` | 4 | | `packages/daemon/test/evaluation-game-ingress.test.ts` | 5 | | `packages/daemon/test/evaluation-game-tools.test.ts` | 3 | -| **Total** | **122** | +| **Total** | **121** | -**5 expected-fail**, all of them in `delegate-and-forward.test.ts` (§3.5) and all -naming a specific change to the tool surface. Every other pin in the gate is an -ordinary assertion. +**1 expected-fail** — `delegate-and-forward.test.ts`'s headless-child pin (§3.5), +naming the one child-side change the surface still needs. Every other pin in the +gate is an ordinary assertion, including the five §3.5 guards that pin the +shipped parent-side contract. **One known flake, pre-existing and not from this work.** `werewolf.test.ts`'s `SCRIPTED BOUNDARY: a seven-player game exhausts the budget inside one 60s window` @@ -318,6 +325,19 @@ before a single wave is injected (`preflightRealSubject` in corrupted `npx` cache produces exactly that, and the symptom without the check is a game that admits every wave, produces zero agent effects, and burns its whole deadline before writing an empty world. +- **The runtime may not be launched through `npx`/`uvx`, and its id must be + `claude-acp`** (re-measured 2026-08-08; the earlier `npx -y …` recipe no + longer works as written). Two independent daemon policies bind here: + - `installedRuntimeCatalog` (`packages/daemon/src/runtimes/probe.ts`) filters + out package-launcher runtimes without a bespoke probe — the launcher being on + `$PATH` says nothing about the agent being installed — so an `npx`-launched + user runtime is reported "not installed" and every dispatch fails. Install + the adapter once (`npm install @agentclientprotocol/claude-agent-acp@0.64.0`) + and launch it as `node /dist/index.js`. + - The subject's `memory: none` requires a **verified native off-switch** + (`packages/daemon/src/agents/runtime-memory.ts`). The policy matches the + exact runtime id `claude-acp` (the `node …/dist/index.js` argv does not match + the signature fallback), so a custom id fails with "off-switch unverified". - **The template must use `permissionMode: default`.** A non-prompting mode (`dontAsk`) makes the runtime deny every AgentConnect tool locally, before the daemon ever sees a permission request, so no game action can land (§5.1). @@ -337,6 +357,31 @@ export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js" # then call runWerewolf({ subject: { kind: 'real', subjectRoot, templateAgentIds } }) ``` +A working template (the shape the 2026-08-08 re-run used): + +```jsonc +// config.json +{ + "version": 1, + "controlPlane": { "enabled": false }, + "runtimes": { + "claude-acp": { + "command": "node", + "args": ["/node_modules/@agentclientprotocol/claude-agent-acp/dist/index.js"] + } + } +} +// agents//agent.json +{ + "id": "", + "name": "", + "status": "active", + "runtime": "claude-acp", + "permissionMode": "default", + "runtimeOverrides": { "model": "sonnet" } +} +``` + ## 5. Real-model runs ### 5.1 Sequential Werewolf, real local Claude Code @@ -573,16 +618,13 @@ is the softer signal. ### 5.4 Peer-driven counting (historical) > **Provenance — read before quoting these.** These numbers were measured on -> **`main` @ 87d36bc** with real local Claude Code over ACP (model sonnet). They -> are **carried forward unrefreshed**: they have not been re-measured since. -> -> They were also measured **when `MAX_AGENT_CALL_HOPS` was 8**. #628 has since -> raised it to 20, so the depth-limited run below would terminate differently -> today — and, per §6.1, on a different protection. Treat the **coordination -> quality** figures (entropy, duplicates, regenerations) as the durable signal -> and the **depth** figures as historical. -> -> Everything in §2, §3, §4, §5.1–§5.3 and §6 **was** measured on the current branch. +> **`main` @ 87d36bc** with real local Claude Code over ACP (model sonnet), when +> `MAX_AGENT_CALL_HOPS` was 8. **They have since been re-measured: §5.6 is the +> 2026-08-08 re-run on current `main`** (hop cap 20, orchestration tools retired, +> evaluation toggle removed). The 2×20 run below stopped at 10 on the hop cap of +> the day; the same run now completes 20/20 (§5.6), and the §6.1 prediction that +> the automatic-turn budget would stop it at 16 edges turned out to be a +> scripted-speed artifact. These tables are kept as the historical baseline. Both runs use the peer-driven variant: one human-sourced start message, then a silent referee. @@ -675,7 +717,11 @@ gone wrong. That is the same observable the production trace reported — "returned no message to forward" — reached by a different mechanism, and from A's seat it was **true**. It -is now pinned as the fifth expected-fail in §3.5. +is pinned as the one remaining expected-fail in §3.5, and the 2026-08-08 re-run's +single trial reproduced it again (§5.6): the child answered in prose to its +headless session, the reply was lost, and the parent — which by then had the +shipped async-contract wake result — correctly made zero status polls and no +premature claim, and simply waited for a report that never came. **What worked, when it worked, was not quite what the child thought.** In trials 1, 3 and 5 the child did call `sendMessage {sessionId:}` — but it described @@ -690,9 +736,70 @@ posts, and the per-trial summary) are written to `.artifacts/evaluation/delegate-forward/` at mode 0600, redacted with the subject template's secret set. They are deliberately not committed. +### 5.6 Full re-run on current `main`, 2026-08-08 — one trial per case + +Every real-model case re-run **once** on `main` @ `70d58cd1` — after the full +sendMessage routing rework (#503/#549/#568), the hop-cap raise to 20 (#628), the +orchestration-tool retirement (#732), and the evaluation-toggle removal (#761) — +with real local Claude Code over ACP (`claude-agent-acp` 0.64.0, model `sonnet`, +`permissionMode: default`, memory off, from-scratch workspace; template shape in +§4.1). One trial is an observation, not a score (§8.1). + +**Fidelity note.** The counting and quota games were run **mention-gated** for +the first time — the production shared-channel convention the routing-acceptance +fixture always used — with the kickoff entering as an ordinary human platform +message that @mentions every participant (`bindMatch: 'mention'` in +`evals/games/engine.ts`; default `auto` unchanged). Continuation after the +kickoff is the ordinary ladder: mention, thread affinity, agent-authored +continuation. Werewolf keeps its referee-driven control conversations and +`auto`-bound rooms by game design. + +| Case | Result (2026-08-08) | Historical | Terminal reason | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------- | -------------------------------- | +| Peer counting 2×20, real | **20/20 completed**, perfect alternation, entropy 1.0, 0 collisions, 0 gated wakes, 0 latches, 124 s | 2/20 (pre-#568); 10/20 (hop cap 8) | `completed` — 19 of 20 hops used | +| Peer counting 4×8, real | **8/8 completed**, entropy **0.95**, all four spoke, 0 collisions, 14 regenerations, 61 s | 8/8, entropy 0.70, one silent agent | `completed` | +| Quota 2×10, real | **10/10 completed-clean**, perfect alternation, exact quotas, 0 violations | completed-clean | `completed` | +| Quota 4×20, real | **19/20**, classified **`deadlocked`**: only agent-a had quota left and had just posted #19 — the no-consecutive rule makes #20 unpostable. The agents diagnosed it themselves mid-run. Entropy 0.997, 3 collisions, 0 over-quota | 19/20 stalled (same shape) | `deadlocked` | +| Werewolf 7p, real (seed 201) | **Completed, werewolves win, 4 rounds**; 3 days all `order_complete` (15/15 speeches), **1 out-of-order speech**, 0 unparseable, **0 leaks**, 0 unauthorized effects | 3/3 completed, werewolves, 0 out-of-order | `completed` | +| Delegate-and-forward, real (1 trial) | Parent side clean (0 polls, no premature claim, `nextAction` honored); **child answered in prose, reply lost** — invariants 3 and 4 failed | 2/5 lost the reply the same way | test `passed` (rates) | +| Cross-room handoff (scripted) | Boundary unchanged: origin 1..6 clean, handoff refused (`thread` not expressible), 0 unauthorized / wrong-room / leaks | identical | `stalled` | +| `eval:collab:contracts` | **115/115** (plus the repinned §3.5 file: 5 green guards + 1 expected-fail) | 115/115 | — | +| `eval:contracts` | **46/46** | 46/46 | — | + +**The headline is the 2×20 chain, and it corrects §6.1's real-model +inference.** The scripted result — a chain stopped at 16 edges by the +automatic-turn budget with hops to spare — is real but is a **scripted-speed +artifact**, exactly like the §5.3 Werewolf collapse: a scripted chain spends all +16 edges inside one 60-second loop-guard window. The real run spanned 124 s, the +window rolled over mid-run, each agent's automatic counter reset, and the chain +ran to the target with **zero** gated wakes — 19 hops used of the 20-hop cap. +Under real timing the operative bound on a two-agent leaderless chain is now the +**hop cap**, and a target of roughly 21+ would hit it. (The budget still binds +per-window: it is the bound for anything that fans out or runs fast, per §6.3 and +§6.5.) + +**Werewolf carried the game again, with two soft regressions stated plainly, both +model behavior, not protections.** One out-of-order speech (day 1) versus zero +across all historical trials, and the worst vote participation observed so far — +3/6, 2/5, 2/4 across the three days (`incompleteVotes: 3`) — with zero gated +wakes among the living. The run also produced the first observed in-game +loop-guard latch: **the night-1 victim's circuit** (the seer, killed before day + +1. latched after absorbing echoes it could never answer — the §6.5 dead-players + limitation observed in a real game. It never affected a living player. + +**#761 changed nothing measurable, by construction.** The daemon registered +`0 evaluation tool(s)` in every run; the agents saw exactly the production tool +surface, and the counting/quota agents used no tools at all — bare ordinary +replies through the routing ladder. + +Run artifacts (`world-events.jsonl`, `game-result.json`, `events.jsonl`, +`topology.json`, `run.json`, full daemon logs) are preserved outside the +repository, per run, by the operator. + ## 6. Limitations -### 6.1 The raised hop cap is unreachable — the automatic-turn budget binds first +### 6.1 Which protection binds a chain depends on speed: the automatic-turn budget at scripted speed, the hop cap in real time Two protections bound an agent-to-agent chain, and **which one binds changed under this branch's feet**: @@ -710,27 +817,29 @@ config-schema key, no CP env key, no `.env.example` entry). The loop guard's budget has no override either, and its latch is durable — only `!resume` clears it. -**Measured on this branch:** a two-agent A→B→A chain advances one hop per edge in +**Measured, scripted:** a two-agent A→B→A chain advances one hop per edge in strict alternation, exactly once per finalized response, and stops at **16 edges** — with **4 hops still unspent**. Each agent spent exactly its 8 automatic turns. Raising only `MAX_AUTOMATIC_TURNS_PER_WINDOW` lets the identical chain run to hop 20 and stop on the cap instead, which is how the attribution was confirmed rather than inferred from arithmetic. -So after #628 the hop cap is **not** the operative limit for a leaderless -two-agent room; the automatic-turn budget is, at `2 × 8 = 16` edges. Raising the -hop cap alone does not lengthen such a conversation. - -Consequence for the probe: a conversation carried purely by agent continuations -gets the initial human-sourced wave plus 16 agent-to-agent edges, so with two -participants roughly **17 numbers**. A longer target still needs a human or -referee message to re-seed the budget. The arena cannot measure leaderless -coordination **quality** past that depth — beyond it, the metric measures the -protection. - -None of this is a defect. It is two protections composing, and the composition is -worth stating plainly because the design's "advances until the cap" is currently -unreachable. +**Measured, real (2026-08-08, §5.6): the 16-edge bound is a scripted-speed +artifact.** A scripted chain spends all 16 edges inside one 60-second loop-guard +window; a real chain does not. The real 2×20 peer count spanned 124 s, the +window rolled over mid-run, each agent's automatic counter reset, and the chain +ran to its target — 19 hops used of the 20-hop cap, **zero** gated wakes, zero +latches. An earlier revision of this section inferred that the same real run +"would now stop at 16 edges on the automatic-turn budget"; that inference was +wrong, exactly the way §5.3's scripted Werewolf collapse was a speed artifact. + +So under real timing the operative bound on a two-agent leaderless chain is the +**hop cap** — a target of roughly 21+ numbers would hit it — while the +automatic-turn budget remains the operative bound wherever turns are fast or +fan-out is wide (§6.3, §6.5). The arena still cannot measure leaderless +coordination quality past the cap: beyond it, the metric measures the +protection. None of this is a defect; it is two protections composing, with +wall-clock speed selecting which one binds. ### 6.2 A leaderless room does not stop at its goal @@ -954,9 +1063,9 @@ Stated explicitly, since several claims above are structural rather than observe | Real Claude Code did NOT poll or fabricate on delegate-and-forward | **Measured**, this branch — 5 trials, `viewSessionStatus` called 0 times, 0 premature claims (§5.5) | | ...but the reply was lost in 2 of 5 trials | **Measured** (§5.5) — the child answered only in prose into a headless session | | The missing parent-side affordances are not sufficient to cause the bug | **Inferred** from those 5 trials — a negative result on one model in one room shape, not a proof that the affordances do not matter | -| Real-model 2 × 20 → 10, entropy 1.0 | **Measured on `main` @ 87d36bc, when the hop cap was 8**; carried forward unrefreshed | -| Real-model 4 × 8 → 8/8, entropy 0.70 | **Measured on `main` @ 87d36bc**, carried forward unrefreshed | -| The same real-model run would now stop at 16 edges | **Inferred** from the scripted chain result; no real-model run has been done since #628 | -| A long leaderless count still cannot finish unaided | **Inferred** from the 16-edge bound plus the absence of any override | +| Real-model 2 × 20 → 10, entropy 1.0 | **Measured on `main` @ 87d36bc, when the hop cap was 8**; superseded by §5.6 (20/20 on current `main`) | +| Real-model 4 × 8 → 8/8, entropy 0.70 | **Measured on `main` @ 87d36bc**; superseded by §5.6 (8/8, entropy 0.95, all four spoke) | +| A real 2-agent chain is bound by the hop cap, not the turn budget | **Measured**, §5.6 — the earlier 16-edge inference was falsified: the 60 s window rolls at real speed (19 hops used, 0 gated) | +| A real 2×20 leaderless count finishes unaided | **Measured**, §5.6 — the "cannot finish unaided" inference held only for scripted speed; a target past ~21 would still hit the cap (**inferred**) | | No hop-cap or loop-guard override exists | **Measured** by exhaustive grep of config schema, CP env, and `.env.example` | | Participation unfairness under fan-out | **Measured** (agents that never spoke) — the _cause_ attributed to scheduling order is **inferred** | diff --git a/evals/test/delegate-and-forward.test.ts b/evals/test/delegate-and-forward.test.ts index e8cc8ea75..4c619145b 100644 --- a/evals/test/delegate-and-forward.test.ts +++ b/evals/test/delegate-and-forward.test.ts @@ -33,8 +33,18 @@ * RED/GREEN. Tests written as `it.fails(…)` are the ones the surface does not satisfy * today: they PASS while the affordance is still missing and start FAILING (flip them * to `it`) the moment it lands. Each names what has to change. The `it(…)` tests around - * them are characterization pins: they record what the surface actually does now, so - * the red tests are anchored in measured behavior rather than in a paraphrase. + * them are guards: they pin what the surface actually does now, so a later change + * cannot silently regress it. + * + * STATUS (re-verified 2026-08-08 against main). The parent-side half of the async + * contract SHIPPED: the `needsReply` wake result carries `reply`, `nextAction: + * 'finish-turn-and-wait'`, and prose stating that the answer arrives as a later + * turn; `viewSessionStatus` is described as optional diagnostics and no longer + * advises waiting; and `SessionStatusResult.reply.state` distinguishes "the child + * ended its turn" from "the child reported back". Those are GREEN guards below. + * The child-side half is still open: a headless child that answers in prose + * instead of `sendMessage {sessionId}` still has its answer dropped silently — + * that is the one remaining red pin, and the real-model half reproduces it. */ import { afterEach, describe, expect, it } from 'vitest' import { COLLABORATION_TOOLS, toolsForIntegrations } from '../../packages/daemon/src/mcp/tools.js' @@ -145,91 +155,80 @@ async function probeSameTurnPoll(): Promise { } describe('delegate-and-forward — what the parent is told when it delegates', () => { - // CHARACTERIZATION (green): the exact surface the observed trace saw. If this - // ever changes, the red tests below are re-reading a different product. - it('records the surface as it is today: an instant wake result, and a same-turn poll that reports the child still running', async () => { + // GUARD (green): the surface as SHIPPED. A `needsReply` wake returns the ids + // plus the caller's half of the async contract — `reply` (requested/awaiting), + // a machine-readable `nextAction`, and prose `message` — and a same-turn poll + // still truthfully reports the child mid-turn. + it('pins the shipped surface: a wake result carrying the async contract, and a same-turn poll that reports the child still running', async () => { const probe = await probeSameTurnPoll() - // The wake result. `wake.targetSession` and `childSessionId` are ids; nothing - // else in it is prose addressed to the caller. expect(probe.wakeResult.ok).toBe(true) expect(probe.wakeResult).toMatchObject({ wake: { delivered: true } }) expect(typeof probe.wakeResult.childSessionId).toBe('string') - expect([...Object.keys(probe.wakeResult)].sort()).toEqual(['childSessionId', 'ok', 'wake']) + expect([...Object.keys(probe.wakeResult)].sort()).toEqual([ + 'childSessionId', + 'message', + 'nextAction', + 'ok', + 'reply', + 'wake' + ]) + expect(probe.wakeResult).toMatchObject({ + reply: { requested: true, state: 'awaiting' }, + nextAction: 'finish-turn-and-wait' + }) - // The same-turn poll — the tool the model reached for — answers that the - // child is still working. This is the observation the fabricated completion - // claim in the trace directly contradicted. + // The same-turn poll — the tool the observed trace reached for — answers + // that the child is still working. The fabricated completion claim in that + // trace directly contradicted this observation. expect(probe.sameTurnStatus).toMatchObject({ status: 'in-progress', state: 'prompting' }) }, 120_000) - // RED — what must change: `executeTool`'s `sendMessage` branch - // (packages/daemon/src/mcp/ops.ts, the `toAgent` return around the - // `childSessionId` assembly) must, for a `needsReply` wake, return prose stating - // the asynchronous contract: the reply arrives later as a new turn in THIS - // session, and the caller should end its turn rather than wait or poll. Only the - // CHILD is told its half today (session-manager's `# Reporting back to your - // parent session`); the parent's half is unstated, which is what leaves "forward - // the reply" looking like a synchronous call that returned nothing. - it.fails( - 'states the asynchronous contract in the needsReply wake result', - async () => { - const probe = await probeSameTurnPoll() - const prose = stringsIn(probe.wakeResult).join(' \n ').toLowerCase() - // (i) the reply comes back later, as a wake of this session… - expect(prose).toMatch(/wake|woken|later turn|new turn|report back|when it (finishes|replies)/) - // (ii) …so the right move now is to finish this turn. - expect(prose).toMatch(/end (your|this) turn|finish (your|this) turn|do not wait|don’t wait|do not poll/) - }, - 120_000 - ) + // GUARD (green, shipped): the `needsReply` wake result states the asynchronous + // contract to the CALLER — the reply arrives later as a new turn of this + // session, so the right move is to end the turn, not wait or poll. This was the + // parent-side half the observed trace was missing; pinned so it cannot regress. + it('states the asynchronous contract in the needsReply wake result', async () => { + const probe = await probeSameTurnPoll() + const prose = stringsIn(probe.wakeResult).join(' \n ').toLowerCase() + // (i) the reply comes back later, as a wake of this session… + expect(prose).toMatch(/wake|woken|later turn|new turn|report back|when it (finishes|replies)/) + // (ii) …so the right move now is to finish this turn. + expect(prose).toMatch(/end (your|this) turn|finish (your|this) turn|do not wait|don’t wait|do not poll/) + }, 120_000) - // RED — what must change: the `viewSessionStatus` description in - // packages/daemon/src/mcp/tools.ts currently closes with "Poll sparingly, and - // prefer waiting for the child’s reply over a tight polling loop." Inside a turn - // there is no way to wait: a turn either ends or blocks, and the model cannot - // block. Advising an impossible action is what turns "wait" into "poll once and - // then answer anyway". - it.fails('does not advise the caller to wait — an action no turn can take', () => { - expect(viewSessionStatusDescription().toLowerCase()).not.toMatch(/prefer waiting|wait for the child/) + // GUARD (green, shipped): the `viewSessionStatus` descriptor no longer advises + // "waiting" — an action no turn can take — and instead bounds the tool to what + // it honestly is: optional diagnostics that never carry the reply body, with + // `nextAction` driving what the caller does next. (An earlier red pin here also + // demanded "say when checking IS appropriate" phrasing; the shipped + // diagnostic-only framing supersedes that ask, so the pin was retired.) + it('describes viewSessionStatus as diagnostics that never advise waiting and never return the reply body', () => { + const description = viewSessionStatusDescription() + expect(description.toLowerCase()).not.toMatch(/prefer waiting|wait for the child/) + expect(description.toLowerCase()).toMatch(/diagnostic/) + expect(description).toContain('never returns the reply body') + expect(description).toContain('nextAction') }) - // RED — same descriptor: having removed the impossible advice, it must say when - // the tool IS the right call. The honest cases are (a) you are already awake for - // some other reason and want a child's progress, and (b) the child you are - // checking is NOT the one whose reply woke you. Neither is expressible today. - it.fails( - 'says when checking a child IS appropriate (already awake; a child other than the one that woke you)', - () => { - const description = viewSessionStatusDescription().toLowerCase() - expect(description).toMatch(/already (awake|running)|when you are awake|woken for another/) - } - ) - - // RED — what must change: `SessionStatusResult` - // (packages/daemon/src/mcp/ops.ts) collapses two different facts into `done`: - // "the child's last turn ended" and "the child has reported back to you". A - // caller that asked for `needsReply` cares only about the second, and today it - // cannot tell them apart — the two situations are byte-identical apart from the - // `updatedAt` clock. That is measured here rather than asserted as a field name, - // so any shape that distinguishes them satisfies this test. - it.fails( - 'distinguishes "the child ended its turn" from "the child reported back"', - async () => { - const silent = await statusAfterChildTurn({ reportToParent: false }) - await fixture?.stop() - fixture = undefined - const reported = await statusAfterChildTurn({ reportToParent: true }) - expect(silent).toMatchObject({ status: 'done' }) - expect(reported).toMatchObject({ status: 'done' }) - // Run-specific identity and the wall clock are not answers to the caller's - // question, so they are removed before the comparison. Whatever remains is - // everything the tool actually TELLS a caller, and today it is identical in - // both situations. - expect(withoutRunIdentity(reported)).not.toEqual(withoutRunIdentity(silent)) - }, - 240_000 - ) + // GUARD (green, shipped): `SessionStatusResult` no longer collapses "the + // child's last turn ended" and "the child has reported back to you" into one + // `done` — `reply.state` separates them. Measured over the whole result rather + // than asserted as a field name, so the guard holds for any shape that keeps + // the two situations distinguishable. + it('distinguishes "the child ended its turn" from "the child reported back"', async () => { + const silent = await statusAfterChildTurn({ reportToParent: false }) + await fixture?.stop() + fixture = undefined + const reported = await statusAfterChildTurn({ reportToParent: true }) + expect(silent).toMatchObject({ status: 'done' }) + expect(reported).toMatchObject({ status: 'done' }) + // Run-specific identity and the wall clock are not answers to the caller's + // question, so they are removed before the comparison. Whatever remains is + // everything the tool actually TELLS a caller, and it must differ between + // the two situations. + expect(withoutRunIdentity(reported)).not.toEqual(withoutRunIdentity(silent)) + }, 240_000) // RED — and this one is not a hypothetical: it is what the real-model runs // actually produced in 2 of 5 trials (collaboration-arena-baseline.md §5.5). @@ -355,14 +354,15 @@ async function statusAfterChildTurn(options: { reportToParent: boolean }): Promi } describe('delegate-and-forward — the standing guidance the two sides receive', () => { - // CHARACTERIZATION (green): the sendMessage descriptor does tell the model to - // set `needsReply`, and it does point at `viewSessionStatus` — but it never says - // the call is asynchronous. Pinned because it is the exact asymmetry the red - // tests above are about. - it('tells the caller to set needsReply and to poll, but never that the call is asynchronous', () => { + // GUARD (green): the sendMessage descriptor stays LEAN by design — it tells the + // model when to set `needsReply`, and that the wake result's `nextAction` (not + // a polling loop) drives what happens next, demoting `viewSessionStatus` to + // optional diagnostics. The async contract itself is delivered at call time in + // the wake result (guarded above), not restated in the always-loaded schema. + it('tells the caller when to set needsReply and to follow the returned nextAction, with status checks as optional diagnostics', () => { const description = sendMessageDescription() expect(description).toContain('needsReply') - expect(description).toContain('viewSessionStatus') - expect(description.toLowerCase()).not.toMatch(/asynchronous|end your turn|woken when|later turn/) + expect(description).toContain('nextAction') + expect(description).toMatch(/viewSessionStatus.*only for optional diagnostics/) }) })