From 3712189590c3ee59fb21982961f0f6f91a741adc Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 13 Jul 2026 16:10:03 -0400 Subject: [PATCH 1/5] Prompt assembly merges tool definitions discovered via readRealmFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tools discovered by reading a skill file become available to the model on the next generation, sourced from room events like every other prompt input — replay-correct and inspectable. The readRealmFile fulfillment embeds each read skill's usable (schema- stamped) tool entries on the result event as data.discoveredTools, each tagged with the source skill URL so the execution leg can verify calls against the declaring skill's indexed frontmatter. Entries without a well-formed definition are dropped; tool-less reads publish exactly the same event shape as before, and attachedFiles is untouched so the attachment-download path and timeline rendering are unchanged. getTools gains a third source: tool-result events' discoveredTools, merged into the tool map before the enabled-skill and message-context sources so those win on functionName conflict. A skill read many times contributes only its latest read's definitions (the file may have changed between reads), and a skill disabled in room state contributes none. CS-12046 Co-Authored-By: Claude Fable 5 --- .../ai-bot/lib/read-realm-file-fulfillment.ts | 61 ++++++- .../ai-bot/tests/prompt-construction-test.ts | 172 ++++++++++++++++++ .../tests/read-realm-file-fulfillment-test.ts | 92 ++++++++++ packages/base/matrix-event.gts | 21 +++ packages/runtime-common/ai/prompt.ts | 47 +++++ 5 files changed, 386 insertions(+), 7 deletions(-) diff --git a/packages/ai-bot/lib/read-realm-file-fulfillment.ts b/packages/ai-bot/lib/read-realm-file-fulfillment.ts index 9b0d8fe24c5..348a8ebc7bc 100644 --- a/packages/ai-bot/lib/read-realm-file-fulfillment.ts +++ b/packages/ai-bot/lib/read-realm-file-fulfillment.ts @@ -13,7 +13,9 @@ import { executeReadRealmFile, fileLabelFromUrl, type ReadRealmFileArgs, + type ReadRealmFileTool, } from './read-realm-file.ts'; +import type { DiscoveredToolDefinition } from '@cardstack/base/matrix-event'; import type { DelegatedUserRealmSessionManager } from './user-delegated-realm-server-session.ts'; let log = logger('ai-bot:read-realm-file'); @@ -119,9 +121,44 @@ export async function fulfillReadRealmFileCalls( } type FileRead = - | { url: string; attachment: Record } + | { + url: string; + attachment: Record; + // Definitions of the tools the read file's indexed frontmatter + // declares, when the file is a skill whose index row carries usable + // (schema-stamped) entries. + discoveredTools?: DiscoveredToolDefinition[]; + } | { url: string; error: string }; +// The read's tool entries that are usable as LLM tool definitions, tagged +// with the skill file they came from. Entries without a well-formed +// definition (e.g. a skill indexed before schema enrichment) are dropped +// here — the prompt can only offer a tool it has a definition for. +function discoveredToolDefinitions( + url: string, + tools: ReadRealmFileTool[] | undefined, +): DiscoveredToolDefinition[] { + return (tools ?? []) + .filter( + (tool) => + tool.definition?.type === 'function' && + typeof tool.definition.function?.name === 'string' && + tool.definition.function.name.length > 0, + ) + .map((tool) => ({ + sourceSkillUrl: url, + ...(tool.codeRef ? { codeRef: tool.codeRef } : {}), + ...(typeof tool.functionName === 'string' + ? { functionName: tool.functionName } + : {}), + ...(typeof tool.requiresApproval === 'boolean' + ? { requiresApproval: tool.requiresApproval } + : {}), + definition: tool.definition as DiscoveredToolDefinition['definition'], + })); +} + // Reads and uploads a single file of a call. An upload failure is folded into // the same shape as a read failure so the caller reports both alike. async function readAndUpload( @@ -144,6 +181,7 @@ async function readAndUpload( log.error(`readRealmFile: upload failed for ${url}: ${e?.message ?? e}`); return { url, error: `could not store ${url} for reading` }; } + let discoveredTools = discoveredToolDefinitions(url, result.tools); return { url, attachment: { @@ -153,6 +191,7 @@ async function readAndUpload( contentType: READ_FILE_CONTENT_TYPE, contentSize: Buffer.byteLength(result.content), }, + ...(discoveredTools.length ? { discoveredTools } : {}), }; } @@ -187,12 +226,19 @@ async function fulfillOne( let reads = await Promise.all( urls.map((url) => readAndUpload(url, deps, upload)), ); - let attachedFiles = reads - .filter( - (read): read is Extract => - 'attachment' in read, - ) - .map((read) => read.attachment); + let successfulReads = reads.filter( + (read): read is Extract => + 'attachment' in read, + ); + let attachedFiles = successfulReads.map((read) => read.attachment); + // Tool definitions the read skills contributed ride the result event next + // to the attachments, so prompt assembly can offer them on later turns + // from room events alone. Kept out of attachedFiles: those entries are + // SerializedFileDefs consumed by the attachment-download path (and the + // timeline), which must not change shape. + let discoveredTools = successfulReads.flatMap( + (read) => read.discoveredTools ?? [], + ); let failureReason = reads .filter( @@ -223,6 +269,7 @@ async function fulfillOne( data: { context: { agentId: deps.agentId }, attachedFiles, + ...(discoveredTools.length ? { discoveredTools } : {}), }, }); return { diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 9590786c9fb..32d60a494a4 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -44,6 +44,7 @@ import { import { absolutizeSkillLinks, buildPromptForModel, + constructHistory, getPromptParts, getRelevantCards, getTools, @@ -7563,6 +7564,177 @@ module('markdown skill tools', (hooks) => { assert.strictEqual(tools.length, 1); assert.strictEqual(tools[0].function.name, 'switch-submode_dd88'); }); + + const DISCOVERED_SKILL_URL = 'https://realm/skills/trip-planner/SKILL.md'; + + // A definition as the fulfillment layer embeds it: the skill's indexed + // frontmatter entry tagged with the skill file it came from. + function discoveredDef( + name: string, + { + skillUrl = DISCOVERED_SKILL_URL, + description = 'Plans a trip', + }: { skillUrl?: string; description?: string } = {}, + ) { + return { + sourceSkillUrl: skillUrl, + codeRef: { module: 'https://realm/commands/plan-trip', name: 'default' }, + functionName: name, + requiresApproval: true, + definition: { + type: 'function', + function: { + name, + description, + parameters: { type: 'object', properties: {} }, + }, + }, + }; + } + + function readResultEvent( + eventId: string, + ts: number, + discoveredTools: unknown[], + ): DiscreteMatrixEvent { + return { + type: APP_BOXEL_TOOL_RESULT_EVENT_TYPE, + event_id: eventId, + origin_server_ts: ts, + sender: '@aibot:localhost', + room_id: 'room1', + content: { + msgtype: APP_BOXEL_TOOL_RESULT_WITH_OUTPUT_MSGTYPE, + commandRequestId: `req-${eventId}`, + 'm.relates_to': { + rel_type: APP_BOXEL_TOOL_RESULT_REL_TYPE, + key: 'applied', + event_id: '$bot-message', + }, + data: { attachedFiles: [], discoveredTools }, + }, + unsigned: { age: 100, transaction_id: 't' }, + status: EventStatus.SENT, + } as unknown as DiscreteMatrixEvent; + } + + test('getTools offers tools discovered by a readRealmFile result event', async () => { + const tools = await getTools( + [readResultEvent('read-1', 2000, [discoveredDef('plan-trip_ab12')])], + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual(tools.length, 1); + assert.strictEqual(tools[0].function.name, 'plan-trip_ab12'); + assert.strictEqual(tools[0].function.description, 'Plans a trip'); + }); + + test('the latest read of a skill replaces its earlier definitions wholesale', async () => { + const tools = await getTools( + [ + readResultEvent('read-1', 1000, [ + discoveredDef('plan-trip_ab12', { description: 'stale' }), + discoveredDef('book-hotel_cd34'), + ]), + readResultEvent('read-2', 2000, [ + discoveredDef('plan-trip_ab12', { description: 'fresh' }), + ]), + ], + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual( + tools.length, + 1, + 'a tool the skill no longer declares disappears with the newer read', + ); + assert.strictEqual(tools[0].function.description, 'fresh'); + }); + + test('a skill disabled in room state contributes no discovered tools', async () => { + const skillsEvent = { + type: APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, + event_id: 'skills-1', + origin_server_ts: 3000, + state_key: '', + content: { + enabledSkillCards: [], + disabledSkillCards: [ + { + sourceUrl: DISCOVERED_SKILL_URL, + url: 'mxc://mock-server/trip-planner', + name: 'SKILL.md', + contentType: 'text/plain', + }, + ], + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { age: 1000 }, + status: EventStatus.SENT, + } as unknown as DiscreteMatrixEvent; + const tools = await getTools( + [ + readResultEvent('read-1', 2000, [discoveredDef('plan-trip_ab12')]), + skillsEvent, + ], + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual(tools.length, 0); + }); + + test('a skill both enabled and read yields one definition; the uploaded one wins', async () => { + const { eventList, enabledSkills } = skillsRoomFixture('toolDefinitions'); + eventList.push( + readResultEvent('read-1', 2000, [ + discoveredDef('switch-submode_dd88', { + skillUrl: 'https://realm/skills/boxel-environment/SKILL.md', + description: 'discovered copy that must lose to the upload', + }), + ]), + ); + const tools = await getTools( + eventList, + enabledSkills, + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual( + tools.length, + 1, + 'one definition for the shared functionName', + ); + assert.strictEqual( + tools[0].function.description, + 'Switch between interact and code submodes', + 'the enabled skill uploaded definition wins the conflict', + ); + }); + + test('discovered definitions round-trip the event encode/decode path', async () => { + // sendMatrixEvent JSON-stringifies `data` on the way out; constructHistory + // parses it back. Feed getTools a history built from the wire shape. + let rawEvent = readResultEvent('read-1', 2000, [ + discoveredDef('plan-trip_ab12'), + ]) as any; + rawEvent.content = { + ...rawEvent.content, + data: JSON.stringify(rawEvent.content.data), + }; + const history = await constructHistory([rawEvent], fakeMatrixClient); + const tools = await getTools( + history, + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual(tools.length, 1); + assert.strictEqual(tools[0].function.name, 'plan-trip_ab12'); + }); }); module('absolutizeSkillLinks', () => { diff --git a/packages/ai-bot/tests/read-realm-file-fulfillment-test.ts b/packages/ai-bot/tests/read-realm-file-fulfillment-test.ts index dcbf20769bc..cf08c06fb7a 100644 --- a/packages/ai-bot/tests/read-realm-file-fulfillment-test.ts +++ b/packages/ai-bot/tests/read-realm-file-fulfillment-test.ts @@ -76,6 +76,98 @@ function dataOf(sentEvent: { content: any }) { } module('fulfillReadRealmFileCalls', () => { + const STAMPED_TOOL = { + codeRef: { + module: 'https://localhost:4201/user/jane/tools/plan-trip', + name: 'default', + }, + functionName: 'plan-trip_ab12', + requiresApproval: false, + definition: { + type: 'function', + function: { + name: 'plan-trip_ab12', + description: 'Plans a trip', + parameters: { type: 'object', properties: {} }, + }, + }, + }; + + // A skill entry the index has not (yet) enriched: no definition, so the + // fulfillment layer must not embed it. + const UNSTAMPED_TOOL = { + codeRef: { + module: 'https://localhost:4201/user/jane/tools/book-hotel', + name: 'default', + }, + }; + + function fileMetaFetch(tools: unknown[]) { + return (async () => + new Response( + JSON.stringify({ + data: { + id: FILE_URL, + type: 'file-meta', + attributes: { + sourceUrl: FILE_URL, + content: '# Trip Planner', + frontmatter: { tools }, + }, + }, + }), + { status: 200 }, + )) as unknown as typeof globalThis.fetch; + } + + test('a skill read embeds its usable tool definitions beside the attachments', async () => { + let { client, sent } = fakeClient(); + + let outcomes = await fulfillReadRealmFileCalls( + [readRealmFileCall('c1', { urls: [FILE_URL] })], + baseDeps(client, { + fetch: fileMetaFetch([STAMPED_TOOL, UNSTAMPED_TOOL]), + uploadText: async () => 'https://localhost/media/trip-planner', + }), + ); + + assert.deepEqual(outcomes, [{ commandRequestId: 'c1', ok: true }]); + let data = dataOf(sent[0]); + assert.strictEqual( + data.attachedFiles.length, + 1, + 'the attachment path is unchanged', + ); + assert.deepEqual( + data.discoveredTools, + [{ sourceSkillUrl: FILE_URL, ...STAMPED_TOOL }], + 'stamped definitions ride the result tagged with the skill URL; the ' + + 'unstamped entry is dropped', + ); + }); + + test('no discoveredTools key when the read carries no definitions', async () => { + let { client, sent } = fakeClient(); + let fetch = (async () => + new Response('# Plain notes', { + status: 200, + })) as unknown as typeof globalThis.fetch; + + await fulfillReadRealmFileCalls( + [readRealmFileCall('c1', { urls: [FILE_URL] })], + baseDeps(client, { + fetch, + uploadText: async () => 'https://localhost/media/notes', + }), + ); + + let data = dataOf(sent[0]); + assert.false( + 'discoveredTools' in data, + 'the result data stays exactly as before for tool-less reads', + ); + }); + test('a successful read attaches the file to an applied command-result event', async () => { let { client, sent } = fakeClient(); let fetch = (async () => diff --git a/packages/base/matrix-event.gts b/packages/base/matrix-event.gts index e41e54b6b56..8b5f14a7124 100644 --- a/packages/base/matrix-event.gts +++ b/packages/base/matrix-event.gts @@ -341,6 +341,22 @@ export interface ToolDefinitionSchema { export type ToolResultStatus = 'applied' | 'failed' | 'invalid'; +// One tool definition the bot discovered by reading a skill markdown file +// (readRealmFile): the entry from the skill's indexed frontmatter, tagged +// with the skill file it came from. Embedded on the read's result event so +// prompt assembly can offer the tool on later turns from room events alone +// (event-sourced, replay-correct), and so execution can verify a call +// against the declaring skill's indexed frontmatter via `sourceSkillUrl`. +export interface DiscoveredToolDefinition { + // The skill markdown file the definition came from (the readRealmFile URL). + sourceSkillUrl: string; + codeRef?: { module?: string; name?: string }; + functionName?: string; + requiresApproval?: boolean; + // The ready-to-use LLM tool definition stamped on the skill's file-meta. + definition: Tool; +} + export interface ToolResultWithOutputContent { 'm.relates_to': { rel_type: ToolResultRelType; @@ -358,6 +374,11 @@ export interface ToolResultWithOutputContent { context?: BoxelContext; attachedFiles?: (SerializedFile & { content?: string; error?: string })[]; attachedCards?: (SerializedFile & { content?: string; error?: string })[]; + // Tool definitions discovered by the read this result reports (present + // only on readRealmFile results whose files carried usable definitions). + // Timeline rendering ignores this key; prompt assembly's getTools reads + // it. + discoveredTools?: DiscoveredToolDefinition[]; }; msgtype: ToolResultWithOutputMsgtype; } diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index c69572adf78..69bc08cf22d 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -31,6 +31,7 @@ import type { CardMessageContent, CardMessageEvent, CodePatchResultEvent, + DiscoveredToolDefinition, ToolResultEvent, MatrixEvent as DiscreteMatrixEvent, EncodedToolRequest, @@ -908,6 +909,52 @@ export async function getTools( } } + // Tools discovered by reading a skill file (readRealmFile): each read's + // result event embeds the definitions in `data.discoveredTools`, so this + // source is event-sourced like every other prompt input. A skill read many + // times contributes only its latest read's definitions (the file may have + // changed between reads); a skill toggled off in room state contributes + // none. Merged into the map first, so an enabled skill's uploaded + // definition or a message-context tool with the same functionName wins on + // conflict. + let disabledSkillIds = new Set(await getDisabledSkillIds(eventList)); + let discoveredBySkill = new Map(); + for (let event of eventList) { + if (!isToolResultEvent(event)) { + continue; + } + let content = event.content; + if (!isToolResultWithOutputContent(content)) { + continue; + } + let discovered = content.data?.discoveredTools; + if (!discovered?.length) { + continue; + } + let bySkill = new Map(); + for (let def of discovered) { + if (!def?.sourceSkillUrl || !def.definition?.function?.name) { + continue; + } + let defs = bySkill.get(def.sourceSkillUrl) ?? []; + defs.push(def); + bySkill.set(def.sourceSkillUrl, defs); + } + // eventList is chronological, so a later read of the same skill + // replaces the earlier one wholesale. + for (let [skillUrl, defs] of bySkill) { + discoveredBySkill.set(skillUrl, defs); + } + } + for (let [skillUrl, defs] of discoveredBySkill) { + if (disabledSkillIds.has(skillUrl)) { + continue; + } + for (let def of defs) { + toolMap.set(def.definition.function.name, def.definition); + } + } + let skillsConfigEvent = findLast( eventList, (event) => event.type === APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, From e95e349932f46790eda4a78df028fabb9448522e Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 13 Jul 2026 16:44:19 -0400 Subject: [PATCH 2/5] Harden the discovered-tools source in getTools Only the bot publishes readRealmFile results, so discovered definitions on anyone else's tool-result event are ignored; and an entry only counts when its definition is a well-formed function tool (type plus function.name), not just any object with a nested name. Co-Authored-By: Claude Fable 5 --- .../ai-bot/tests/prompt-construction-test.ts | 37 +++++++++++++++++++ packages/runtime-common/ai/prompt.ts | 11 +++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 32d60a494a4..45858056297 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -7653,6 +7653,43 @@ module('markdown skill tools', (hooks) => { assert.strictEqual(tools[0].function.description, 'fresh'); }); + test('discovered tools on a non-bot result event are ignored', async () => { + let forged = readResultEvent('read-1', 2000, [ + discoveredDef('plan-trip_ab12'), + ]) as any; + forged.sender = '@someone-else:localhost'; + const tools = await getTools( + [forged], + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual( + tools.length, + 0, + 'only the bot publishes readRealmFile results', + ); + }); + + test('an entry whose definition is not a function tool is skipped', async () => { + const tools = await getTools( + [ + readResultEvent('read-1', 2000, [ + { + ...discoveredDef('plan-trip_ab12'), + definition: { function: { name: 'plan-trip_ab12' } }, + }, + discoveredDef('book-hotel_cd34'), + ]), + ], + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual(tools.length, 1, 'the malformed entry is dropped'); + assert.strictEqual(tools[0].function.name, 'book-hotel_cd34'); + }); + test('a skill disabled in room state contributes no discovered tools', async () => { const skillsEvent = { type: APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index 69bc08cf22d..eccd6401b22 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -923,6 +923,11 @@ export async function getTools( if (!isToolResultEvent(event)) { continue; } + // Only the bot publishes readRealmFile results; a discoveredTools block + // on anyone else's result event is not a legitimate discovery. + if (event.sender !== aiBotUserId) { + continue; + } let content = event.content; if (!isToolResultWithOutputContent(content)) { continue; @@ -933,7 +938,11 @@ export async function getTools( } let bySkill = new Map(); for (let def of discovered) { - if (!def?.sourceSkillUrl || !def.definition?.function?.name) { + if ( + !def?.sourceSkillUrl || + def.definition?.type !== 'function' || + !def.definition.function?.name + ) { continue; } let defs = bySkill.get(def.sourceSkillUrl) ?? []; From 33223d5ffbaaad1b5d9a289304accf876877d3ba Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 13 Jul 2026 16:54:54 -0400 Subject: [PATCH 3/5] Only applied read results contribute discovered tools A failed or invalid readRealmFile result is not evidence the skill was fetched, so its discoveredTools claim is ignored. Co-Authored-By: Claude Fable 5 --- .../ai-bot/tests/prompt-construction-test.ts | 18 ++++++++++++++++++ packages/runtime-common/ai/prompt.ts | 5 +++++ 2 files changed, 23 insertions(+) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 45858056297..3026c854b27 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -7690,6 +7690,24 @@ module('markdown skill tools', (hooks) => { assert.strictEqual(tools[0].function.name, 'book-hotel_cd34'); }); + test('discoveries on a non-applied result are ignored', async () => { + let failed = readResultEvent('read-1', 2000, [ + discoveredDef('plan-trip_ab12'), + ]) as any; + failed.content['m.relates_to'].key = 'invalid'; + const tools = await getTools( + [failed], + [], + '@aibot:localhost', + fakeMatrixClient, + ); + assert.strictEqual( + tools.length, + 0, + 'only an applied read is evidence the skill was actually fetched', + ); + }); + test('a skill disabled in room state contributes no discovered tools', async () => { const skillsEvent = { type: APP_BOXEL_ROOM_SKILLS_EVENT_TYPE, diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index eccd6401b22..c7b80d126a4 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -932,6 +932,11 @@ export async function getTools( if (!isToolResultWithOutputContent(content)) { continue; } + // Only an applied read actually fetched the skill; discoveries claimed + // by a failed or invalid result are not evidence of anything. + if (content['m.relates_to']?.key !== 'applied') { + continue; + } let discovered = content.data?.discoveredTools; if (!discovered?.length) { continue; From c0edde0bc8a6778cdf510695e388bcfad0cbeb6d Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 13 Jul 2026 16:50:37 -0400 Subject: [PATCH 4/5] Host executes tools from read (not enabled) skills after realm verification When a tool call's functionName resolves in no enabled skill, message-builder falls back to the skill named by the latest readRealmFile result event's discoveredTools. That annotation is strictly a lookup hint: the codeRef the host executes is re-derived from the skill's realm-indexed frontmatter, loaded through the store with the user's own permissions, and accepted only when the indexed declaration's functionName matches the call. requiresApproval comes from the same verified declaration (absent means approval required), so auto-execute honors it identically to enabled skills. A forged annotation, a skill the user can't read, or a tool the skill no longer declares all leave the request unresolved, surfacing through the existing unrecognized-command invalid-result path. CS-12047 Co-Authored-By: Claude Fable 5 --- .../app/lib/matrix-classes/message-builder.ts | 27 +++ packages/host/app/lib/skill-tools.ts | 51 +++++ packages/host/tests/acceptance/tools-test.gts | 202 ++++++++++++++++++ 3 files changed, 280 insertions(+) diff --git a/packages/host/app/lib/matrix-classes/message-builder.ts b/packages/host/app/lib/matrix-classes/message-builder.ts index 987bf14e644..accdd09f4f8 100644 --- a/packages/host/app/lib/matrix-classes/message-builder.ts +++ b/packages/host/app/lib/matrix-classes/message-builder.ts @@ -32,6 +32,7 @@ import { } from '@cardstack/runtime-common/matrix-constants'; import { + findDiscoveredToolSkillUrl, getSkillSourceTools, loadSkillSource, } from '@cardstack/host/lib/skill-tools'; @@ -379,6 +380,32 @@ export default class MessageBuilder { } } + // Tool from a read (not enabled) skill: the model may call a tool it + // discovered by reading a skill file via readRealmFile. The bot's result + // event names the declaring skill, but that annotation is strictly a + // lookup hint, never an authorization — the codeRef the host executes is + // re-derived here from the skill's realm-indexed frontmatter, loaded + // through the store with the user's own permissions. A forged annotation + // can't execute anything the named skill doesn't declare, and a skill the + // user can't read resolves nothing; either way the tool stays unresolved + // and surfaces through the existing unrecognized-command failure path. + // `requiresApproval` likewise comes from the verified declaration (absent + // means approval required), exactly as for enabled skills. + if (!skillTool && toolRequest.name) { + let sourceSkillUrl = findDiscoveredToolSkillUrl( + this.builderContext.events, + toolRequest.name, + ); + if (sourceSkillUrl) { + let source = await loadSkillSource(this.store, sourceSkillUrl); + if (source) { + skillTool = getSkillSourceTools(source).find( + (candidate) => candidate.functionName === toolRequest.name, + ); + } + } + } + let actionVerb = 'Apply'; if (skillTool?.codeRef) { let CommandKlass = (await getClass( diff --git a/packages/host/app/lib/skill-tools.ts b/packages/host/app/lib/skill-tools.ts index 54154f3e3b6..5849b8c7a8b 100644 --- a/packages/host/app/lib/skill-tools.ts +++ b/packages/host/app/lib/skill-tools.ts @@ -1,9 +1,18 @@ import { isCardInstance, isMarkdownFile } from '@cardstack/runtime-common'; +import { + isToolResultEventType, + isToolResultWithOutputContent, +} from '@cardstack/runtime-common/matrix-constants'; import { isSkillCard } from './file-def-manager'; import type StoreService from '../services/store'; import type { MarkdownDef } from '@cardstack/base/markdown-file-def'; +import type { + DiscoveredToolDefinition, + MatrixEvent as DiscreteMatrixEvent, + ToolResultEvent, +} from '@cardstack/base/matrix-event'; import type * as SkillModule from '@cardstack/base/skill'; // A skill is either a `Skill` card (commands on `Skill.commands`) or a @@ -95,3 +104,45 @@ export function peekSkillSource( let card = store.peek(id); return isCardInstance(card) && isSkillCardInstance(card) ? card : undefined; } + +// The declaring-skill hint for a tool the model discovered by reading a +// skill file: the latest readRealmFile result event whose +// `data.discoveredTools` names this function returns that entry's +// `sourceSkillUrl`. The hint only says where to look — a caller must +// re-derive the codeRef from that skill's realm-indexed frontmatter (see +// `message-builder`) before executing anything, so a forged or stale +// annotation can point at a skill but never make it declare a tool it +// doesn't have. +export function findDiscoveredToolSkillUrl( + events: DiscreteMatrixEvent[], + functionName: string, +): string | undefined { + let found: string | undefined; + for (let event of events) { + if (!isToolResultEventType(event.type)) { + continue; + } + let content = (event as ToolResultEvent).content; + if (!isToolResultWithOutputContent(content)) { + continue; + } + let discovered = content.data?.discoveredTools; + if (!Array.isArray(discovered)) { + continue; + } + for (let def of discovered as DiscoveredToolDefinition[]) { + if (!def?.sourceSkillUrl) { + continue; + } + if ( + def.definition?.function?.name === functionName || + def.functionName === functionName + ) { + // Keep scanning: events are chronological, and the latest read of a + // skill is the freshest claim about where the tool lives. + found = def.sourceSkillUrl; + } + } + } + return found; +} diff --git a/packages/host/tests/acceptance/tools-test.gts b/packages/host/tests/acceptance/tools-test.gts index f133e683a13..77c647dbb65 100644 --- a/packages/host/tests/acceptance/tools-test.gts +++ b/packages/host/tests/acceptance/tools-test.gts @@ -476,6 +476,39 @@ module('Acceptance | Tools tests', function (hooks) { 'https://i.postimg.cc/VNvHH93M/pawel-czerwinski-Ly-ZLa-A5jti-Y-unsplash.jpg', iconURL: 'https://i.postimg.cc/L8yXRvws/icon.png', }), + // Markdown skills for the read-skill (pull model) execution tests. + // Neither is enabled in any room; their tools only become callable + // after a readRealmFile result event names them. + 'skills/read-tools/SKILL.md': [ + '---', + 'name: read-tools', + 'boxel:', + ' kind: skill', + ' tools:', + ' - codeRef:', + " module: '@cardstack/boxel-host/commands/switch-submode'", + ' name: default', + '---', + '# Read Tools', + '', + 'Use switch-submode to move between interact and code modes.', + '', + ].join('\n'), + 'skills/decoy/SKILL.md': [ + '---', + 'name: decoy', + 'boxel:', + ' kind: skill', + ' tools:', + ' - codeRef:', + ' module: /test/maybe-boom-command', + ' name: default', + '---', + '# Decoy', + '', + 'Declares only maybe-boom.', + '', + ].join('\n'), 'hi.txt': 'hi', }, }); @@ -908,6 +941,175 @@ module('Acceptance | Tools tests', function (hooks) { ); }); + // Simulates the result event the bot publishes after reading a skill via + // readRealmFile: the discovered tool definitions ride `data.discoveredTools` + // tagged with the declaring skill's URL. + function simulateSkillReadResult( + roomId: string, + sourceSkillUrl: string, + functionName: string, + ) { + simulateRemoteMessage( + roomId, + '@aibot:localhost', + { + msgtype: APP_BOXEL_TOOL_RESULT_WITH_OUTPUT_MSGTYPE, + commandRequestId: 'read-1', + 'm.relates_to': { + rel_type: APP_BOXEL_TOOL_RESULT_REL_TYPE, + key: 'applied', + event_id: 'earlier-bot-message', + }, + data: { + attachedFiles: [], + discoveredTools: [ + { + sourceSkillUrl, + codeRef: { + module: '@cardstack/boxel-host/commands/switch-submode', + name: 'default', + }, + functionName, + requiresApproval: true, + definition: { + type: 'function', + function: { + name: functionName, + description: 'Switch between interact and code submodes', + parameters: { type: 'object', properties: {} }, + }, + }, + }, + ], + }, + }, + { type: APP_BOXEL_TOOL_RESULT_EVENT_TYPE }, + ); + } + + test('a tool from a skill read via readRealmFile executes after approval', async function (assert) { + await visitOperatorMode({ + stacks: [[{ id: `${testRealmURL}index`, format: 'isolated' }]], + aiAssistantOpen: true, + }); + await waitFor('[data-room-settled]'); + let roomId = getRoomIds().pop()!; + // The skill is never enabled in the room; the model learned about its + // tool from an earlier readRealmFile result event. + simulateSkillReadResult( + roomId, + `${testRealmURL}skills/read-tools/SKILL.md`, + 'switch-submode_dd88', + ); + simulateRemoteMessage(roomId, '@aibot:localhost', { + body: '', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'call-1', + name: 'switch-submode_dd88', + arguments: JSON.stringify({ + description: 'Switching to code submode', + attributes: { submode: 'code' }, + }), + }, + ], + }); + await waitFor('[data-test-message-idx="0"]'); + await settled(); + + // The skill's frontmatter has no requiresApproval on the tool, so + // approval is required: the request renders an Apply button and nothing + // auto-runs. + assert + .dom('[data-test-message-idx="0"] [data-test-tool-call-apply]') + .exists('the verified read-skill tool offers approval, not auto-run'); + + await click('[data-test-message-idx="0"] [data-test-tool-call-apply]'); + await waitFor('[data-test-submode-switcher=code]', { timeout: 5000 }); + assert.dom('[data-test-submode-switcher=code]').exists(); + + await waitUntil( + () => + getRoomEvents(roomId).find( + (m) => + m.content.msgtype === + APP_BOXEL_TOOL_RESULT_WITH_NO_OUTPUT_MSGTYPE && + m.content.commandRequestId === 'call-1', + ), + { + timeout: 5000, + timeoutMessage: 'timed out waiting for command result event', + }, + ); + let message = getRoomEvents(roomId).find( + (m) => + m.content.msgtype === APP_BOXEL_TOOL_RESULT_WITH_NO_OUTPUT_MSGTYPE && + m.content.commandRequestId === 'call-1', + )!; + assert.strictEqual(message.content['m.relates_to']?.key, 'applied'); + }); + + test('a discovered-tool annotation naming a skill that does not declare the tool is rejected', async function (assert) { + await visitOperatorMode({ + stacks: [[{ id: `${testRealmURL}index`, format: 'isolated' }]], + aiAssistantOpen: true, + }); + await waitFor('[data-room-settled]'); + let roomId = getRoomIds().pop()!; + // The annotation names the decoy skill, which declares only maybe-boom — + // re-deriving from the decoy's indexed frontmatter finds no + // switch-submode, so the codeRef asserted by the event never executes. + simulateSkillReadResult( + roomId, + `${testRealmURL}skills/decoy/SKILL.md`, + 'switch-submode_dd88', + ); + simulateRemoteMessage(roomId, '@aibot:localhost', { + body: '', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'call-forged', + name: 'switch-submode_dd88', + arguments: JSON.stringify({ + description: 'Switching to code submode', + attributes: { submode: 'code' }, + }), + }, + ], + data: { + context: { + agentId: getService('matrix-service').agentId, + }, + }, + }); + await waitFor('[data-test-message-idx="0"]'); + + await waitFor( + '[data-test-message-idx="0"] [data-test-apply-state="invalid"]', + ); + assert + .dom('[data-test-boxel-alert="warning"]') + .containsText('No command for the name "switch-submode_dd88" was found'); + assert + .dom('[data-test-submode-switcher=code]') + .doesNotExist('the forged tool call never executed'); + + let message = getRoomEvents(roomId) + .filter( + (m) => + m.content.msgtype === APP_BOXEL_TOOL_RESULT_WITH_NO_OUTPUT_MSGTYPE, + ) + .pop()!; + assert.strictEqual(message.content['m.relates_to']?.key, 'invalid'); + assert.strictEqual(message.content.commandRequestId, 'call-forged'); + }); + test('rendering a command request without a description fallsback to attributes.description', async function (assert) { await visitOperatorMode({ stacks: [ From fe8413b519a0b6e0f191bacf4c57cb589c2810f6 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 13 Jul 2026 17:10:39 -0400 Subject: [PATCH 5/5] Harden the discovered-tool fallback lookup The skill-URL hint comes from a bot event, so a load that throws on a malformed or unreadable id degrades to an unresolved tool instead of breaking message building. The hint scan also walks newest-first and returns on the first match rather than scanning the full timeline. Co-Authored-By: Claude Fable 5 --- .../app/lib/matrix-classes/message-builder.ts | 18 ++++++++++++++---- packages/host/app/lib/skill-tools.ts | 12 ++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/host/app/lib/matrix-classes/message-builder.ts b/packages/host/app/lib/matrix-classes/message-builder.ts index accdd09f4f8..d7068d58aa9 100644 --- a/packages/host/app/lib/matrix-classes/message-builder.ts +++ b/packages/host/app/lib/matrix-classes/message-builder.ts @@ -397,10 +397,20 @@ export default class MessageBuilder { toolRequest.name, ); if (sourceSkillUrl) { - let source = await loadSkillSource(this.store, sourceSkillUrl); - if (source) { - skillTool = getSkillSourceTools(source).find( - (candidate) => candidate.functionName === toolRequest.name, + // The URL comes from a bot event, so a load blowing up on a + // malformed or unreadable id must degrade to "unresolved tool", not + // break message building for the whole timeline. + try { + let source = await loadSkillSource(this.store, sourceSkillUrl); + if (source) { + skillTool = getSkillSourceTools(source).find( + (candidate) => candidate.functionName === toolRequest.name, + ); + } + } catch (e) { + console.warn( + `could not load skill ${sourceSkillUrl} to resolve tool "${toolRequest.name}":`, + e, ); } } diff --git a/packages/host/app/lib/skill-tools.ts b/packages/host/app/lib/skill-tools.ts index 5849b8c7a8b..92d1b5b1e01 100644 --- a/packages/host/app/lib/skill-tools.ts +++ b/packages/host/app/lib/skill-tools.ts @@ -117,8 +117,10 @@ export function findDiscoveredToolSkillUrl( events: DiscreteMatrixEvent[], functionName: string, ): string | undefined { - let found: string | undefined; - for (let event of events) { + // Walk newest-first and return on the first match: the latest read of a + // skill is the freshest claim about where the tool lives. + for (let i = events.length - 1; i >= 0; i--) { + let event = events[i]; if (!isToolResultEventType(event.type)) { continue; } @@ -138,11 +140,9 @@ export function findDiscoveredToolSkillUrl( def.definition?.function?.name === functionName || def.functionName === functionName ) { - // Keep scanning: events are chronological, and the latest read of a - // skill is the freshest claim about where the tool lives. - found = def.sourceSkillUrl; + return def.sourceSkillUrl; } } } - return found; + return undefined; }