diff --git a/.github/workflows/deploy-fiona-slack-container.yml b/.github/workflows/deploy-fiona-slack-container.yml index 2b6ff9fa..898facca 100644 --- a/.github/workflows/deploy-fiona-slack-container.yml +++ b/.github/workflows/deploy-fiona-slack-container.yml @@ -93,6 +93,8 @@ jobs: cosmosAccountName="${{ vars.COSMOS_ACCOUNT_NAME }}" \ captureAllConversations=${{ vars.CAPTURE_ALL_CONVERSATIONS || 'false' }} \ deploymentType="${{ inputs.environment || 'insiders' }}" \ + escalationUsergroupId=${{ vars.ESCALATION_USERGROUP_ID || '' }} \ + escalationChannel="${{ vars.ESCALATION_CHANNEL || '' }}" \ skipRoleAssignments=true - name: Verify deployment diff --git a/apps/fiona-slack/.env.sample b/apps/fiona-slack/.env.sample index 5c7a132c..c8abae2d 100644 --- a/apps/fiona-slack/.env.sample +++ b/apps/fiona-slack/.env.sample @@ -43,3 +43,9 @@ RATE_LIMIT_WINDOW_MS=3600000 # Optional, override the default system prompt sent to the LLM. # SYSTEM_PROMPT=Your custom system prompt here. + +# Escalation (/fiona escalate). ESCALATION_CHANNEL is the destination channel ID +# (e.g. C0123456789) — the bot must be a member of that channel to post. +# ESCALATION_CHANNEL=C0123456789 +# Optional: a Slack user group ID (e.g. S0123456789) to @-mention on escalation. +# ESCALATION_USERGROUP_ID=S0123456789 diff --git a/apps/fiona-slack/manifest.json b/apps/fiona-slack/manifest.json index 2d766748..ce873974 100644 --- a/apps/fiona-slack/manifest.json +++ b/apps/fiona-slack/manifest.json @@ -22,7 +22,7 @@ { "command": "/fiona", "description": "Interact with Fiona, your Ed-Fi AI assistant", - "usage_hint": "[help | ask | search ]", + "usage_hint": "[help | ask | search | escalate]", "should_escape": false } ] diff --git a/apps/fiona-slack/src/agent/escalation.js b/apps/fiona-slack/src/agent/escalation.js new file mode 100644 index 00000000..b098d50b --- /dev/null +++ b/apps/fiona-slack/src/agent/escalation.js @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { recordFeedback } from './feedback-store.js'; +import { recordInteraction } from './interaction-store.js'; +import { summarizeForEscalation } from './llm-caller.js'; +import { getUser } from './slack-users-store.js'; + +const HISTORY_LIMIT = 20; +const SLACK_BLOCK_TEXT_LIMIT = 2900; // leave headroom under Slack's 3000-char section limit + +/** + * Build a plain-text transcript of the recent conversation. Uses thread replies + * when a threadTs is available, otherwise the channel's recent history. + * + * @returns {Promise} Newline-joined "*Who:* text" lines, or '' on failure. + */ +async function fetchTranscript(client, { channelId, threadTs, logger }) { + try { + let messages; + if (threadTs) { + const res = await client.conversations.replies({ channel: channelId, ts: threadTs, limit: 50 }); + messages = res.messages ?? []; + } else { + const res = await client.conversations.history({ channel: channelId, limit: HISTORY_LIMIT }); + messages = (res.messages ?? []).reverse(); // history returns newest-first + } + return messages + .filter((m) => m.text) + .map((m) => { + const who = m.bot_id ? 'Fiona' : m.user ? `<@${m.user}>` : 'User'; + const text = (m.text ?? '').replace(/^(<@[A-Z0-9]+>\s*)+/, '').trim(); + return text ? `*${who}:* ${text}` : null; + }) + .filter(Boolean) + .join('\n'); + } catch (err) { + logger?.warn?.(`Failed to fetch transcript for escalation: ${err.message}`); + return ''; + } +} + +/** + * Post an escalation to the configured channel and record it. Shared by the + * /fiona escalate slash command and the proactive escalation flow. + * + * @param {Object} params + * @param {import("@slack/web-api").WebClient} params.client + * @param {string} params.userId + * @param {string} [params.teamId] + * @param {string} params.channelId + * @param {string|null} [params.threadTs] + * @param {string} params.messageTs + * @param {'slash_escalate'|'auto_escalation'} params.source + * @param {boolean} [params.isDm] + * @param {import("@slack/logger").Logger} [params.logger] + * @returns {Promise<{ ok: boolean, errorType: string|null }>} + */ +export async function postEscalation({ + client, + userId, + teamId, + channelId, + threadTs = null, + messageTs, + source, + isDm = false, + logger, +}) { + const targetChannel = process.env.ESCALATION_CHANNEL; + if (!targetChannel) { + logger?.warn?.('ESCALATION_CHANNEL is not configured; cannot post escalation.'); + return { ok: false, errorType: 'channel_not_configured' }; + } + + const user = await getUser(userId, logger); + const displayName = user?.displayName || user?.realName || user?.name || `<@${userId}>`; + + const transcript = await fetchTranscript(client, { channelId, threadTs, logger }); + const summary = await summarizeForEscalation(transcript, logger); + + let permalink = null; + if (!isDm) { + try { + const res = await client.chat.getPermalink({ channel: channelId, message_ts: threadTs ?? messageTs }); + permalink = res?.permalink ?? null; + } catch (err) { + logger?.warn?.(`Failed to get permalink for escalation: ${err.message}`); + } + } + + const usergroupId = process.env.ESCALATION_USERGROUP_ID; + const mention = usergroupId ? ` ` : ''; + + const locationLink = isDm + ? 'Direct message (no permalink)' + : permalink + ? `<${permalink}|View conversation>` + : `<#${channelId}>`; + const headerLines = [ + `${mention}:rotating_light: *Escalation requested* by *${displayName}*`, + `*Where:* ${locationLink}`, + `*When:* ${new Date().toISOString()}`, + ]; + if (summary) headerLines.push(`*Summary:* ${summary}`); + + let postedTs = null; + try { + const res = await client.chat.postMessage({ + channel: targetChannel, + text: `Escalation requested by ${displayName}`, + blocks: [{ type: 'section', text: { type: 'mrkdwn', text: headerLines.join('\n') } }], + }); + postedTs = res?.ts ?? null; + } catch (err) { + logger?.error?.(`Failed to post escalation to ${targetChannel}: ${err.message}`); + return { ok: false, errorType: 'post_failed' }; + } + + if (transcript && postedTs) { + try { + await client.chat.postMessage({ + channel: targetChannel, + thread_ts: postedTs, + text: 'Conversation transcript', + blocks: [ + { + type: 'section', + text: { type: 'mrkdwn', text: `*Transcript:*\n${transcript}`.slice(0, SLACK_BLOCK_TEXT_LIMIT) }, + }, + ], + }); + } catch (err) { + logger?.warn?.(`Failed to post escalation transcript: ${err.message}`); + } + } + + recordInteraction({ + userId, + teamId, + channelId, + threadTs: threadTs ?? messageTs, + messageTs, + interactionType: source, + status: 'success', + errorType: null, + rateLimited: false, + logger, + }).catch((e) => logger?.warn?.(`Failed to record escalation interaction: ${e.message}`)); + + recordFeedback({ + userId, + channelId, + messageTs, + value: 'escalation', + interactionType: source, + reason: null, + userMessage: transcript || null, + botResponse: summary || null, + logger, + }).catch((e) => logger?.warn?.(`Failed to record escalation feedback: ${e.message}`)); + + return { ok: true, errorType: null }; +} diff --git a/apps/fiona-slack/src/agent/feedback-store.js b/apps/fiona-slack/src/agent/feedback-store.js index a6e62fed..5b0a556d 100644 --- a/apps/fiona-slack/src/agent/feedback-store.js +++ b/apps/fiona-slack/src/agent/feedback-store.js @@ -130,11 +130,12 @@ async function getContainer(logger) { * @param {Object} feedback * @param {string} feedback.userId - Slack user ID * @param {string} feedback.channelId - Slack channel ID - * @param {string} feedback.messageTs - Timestamp of the bot message being rated - * @param {string} feedback.value - 'good-feedback' or 'bad-feedback' + * @param {string} feedback.messageTs - Bot message timestamp or slash-command trigger_id for escalations + * @param {string} feedback.value - 'good-feedback', 'bad-feedback', or 'escalation' * @param {string|null} [feedback.reason] - Optional reason for the feedback * @param {string|null} feedback.userMessage - The user's message that prompted the response * @param {string|null} feedback.botResponse - The bot's response being rated + * @param {string} [feedback.interactionType] - Optional interaction type (e.g., 'slash_escalate') * @param {{ warn?: (msg: string) => void }} [feedback.logger] - Optional logger for warnings */ export async function recordFeedback({ @@ -145,6 +146,7 @@ export async function recordFeedback({ reason, userMessage, botResponse, + interactionType, logger, }) { const c = await getContainer(logger); @@ -162,6 +164,7 @@ export async function recordFeedback({ reason: reason?.trim() ? reason.trim() : null, userMessage, botResponse, + ...(interactionType ? { interactionType } : {}), deploymentType: process.env.DEPLOYMENT_TYPE || 'local', timestamp: new Date().toISOString(), }; diff --git a/apps/fiona-slack/src/agent/interaction-store.js b/apps/fiona-slack/src/agent/interaction-store.js index 63b00f06..3c2001f6 100644 --- a/apps/fiona-slack/src/agent/interaction-store.js +++ b/apps/fiona-slack/src/agent/interaction-store.js @@ -136,7 +136,7 @@ export async function getContainer(logger) { * @param {string} interaction.channelId - Slack channel ID * @param {string} interaction.threadTs - Interaction session identifier (`thread_ts` for message flows, `trigger_id` for slash commands) * @param {string} interaction.messageTs - Interaction event identifier (`message_ts` for message flows, `trigger_id` for slash commands) - * @param {string} interaction.interactionType - 'app_mention', 'assistant_message', 'slash_help', 'slash_ask', 'slash_search', or 'slash_unknown' + * @param {string} interaction.interactionType - 'app_mention', 'assistant_message', 'slash_help', 'slash_ask', 'slash_search', 'slash_escalate', 'auto_escalation', or 'slash_unknown' * @param {string} interaction.status - 'success' or 'error' * @param {string|null} [interaction.errorType] - Error category if status is 'error' * @param {boolean} interaction.rateLimited - true if request was rate-limited diff --git a/apps/fiona-slack/src/agent/llm-caller.js b/apps/fiona-slack/src/agent/llm-caller.js index 1b95ab59..67859190 100644 --- a/apps/fiona-slack/src/agent/llm-caller.js +++ b/apps/fiona-slack/src/agent/llm-caller.js @@ -484,3 +484,39 @@ export async function callLLM(streamer, prompts, logger) { return { metadata, botText, systemPromptVersion: SYSTEM_PROMPT_VERSION }; } + +// ─── Escalation Summary ───────────────────────────────────────────────────── +const ESCALATION_SUMMARY_SYSTEM_PROMPT = + 'You summarize a Slack conversation between a user and Fiona (an Ed-Fi AI assistant) for a human support team. ' + + 'In 2-4 sentences, state what the user is trying to do and where they got stuck. ' + + 'Be factual and concise. Do not add greetings or sign-offs.'; + +/** + * Produce a short human-readable summary of a conversation transcript for an + * escalation post. Non-streaming. Returns null when the LLM is unconfigured, + * the transcript is empty, or the call fails — callers degrade to transcript-only. + * + * @param {string} transcriptText + * @param {{ warn?: (msg: string) => void }} [logger] + * @returns {Promise} + */ +export async function summarizeForEscalation(transcriptText, logger) { + if (!perplexityClient) return null; + if (!transcriptText || !transcriptText.trim()) return null; + + try { + const response = await perplexityClient.chat.completions.create({ + model: PERPLEXITY_API_MODEL, + messages: [ + { role: 'system', content: ESCALATION_SUMMARY_SYSTEM_PROMPT }, + { role: 'user', content: transcriptText }, + ], + stream: false, + }); + const summary = response?.choices?.[0]?.message?.content; + return typeof summary === 'string' && summary.trim() ? summary.trim() : null; + } catch (error) { + logger?.warn?.(`Failed to generate escalation summary: ${error.message}`); + return null; + } +} diff --git a/apps/fiona-slack/src/listeners/commands/fiona.js b/apps/fiona-slack/src/listeners/commands/fiona.js index 3f53503c..be953b5e 100644 --- a/apps/fiona-slack/src/listeners/commands/fiona.js +++ b/apps/fiona-slack/src/listeners/commands/fiona.js @@ -3,7 +3,9 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. +import { postEscalation } from '../../agent/escalation.js'; import { recordInteraction } from '../../agent/interaction-store.js'; +import { checkRateLimit } from '../../agent/rate-limiter.js'; const HELP_TEXT = `*Fiona — your Ed-Fi AI assistant* :wave: Fiona helps you navigate Ed-Fi documentation, standards, and community resources using natural language. @@ -13,6 +15,7 @@ Fiona helps you navigate Ed-Fi documentation, standards, and community resources /fiona help Show this help message /fiona ask Ask Fiona a question about Ed-Fi (coming soon) /fiona search Search Ed-Fi documentation (coming soon) +/fiona escalate Escalate your conversation to a human responder \`\`\` _Tip: You can also @-mention Fiona in any channel, or send her a direct message._`; @@ -24,11 +27,17 @@ const SEARCH_NOT_YET_TEXT = `*/fiona search* is not yet available. ` + `In the meantime, @-mention Fiona in any channel or send her a direct message.`; +const ESCALATE_CONFIRM_TEXT = + '✅ Your conversation has been escalated to #escalation. A team member will follow up shortly.'; +const ESCALATE_DM_TEXT = '✅ A team member will follow up shortly.'; +const ESCALATE_ERROR_TEXT = + ':warning: Sorry, I could not escalate your conversation right now. Please reach out to the team directly.'; + /** * Handles the /fiona slash command. Routes to a sub-command handler or falls * back to help for unrecognized / missing input. Never invokes the LLM. */ -export const fionaCommandCallback = async ({ command, ack, logger }) => { +export const fionaCommandCallback = async ({ command, ack, respond, client, logger }) => { logger?.info?.(`/fiona slash command invoked: ${command.text ?? '(empty)'}`); const subCommand = (command.text ?? '').trim().split(/\s+/)[0].toLowerCase(); @@ -41,13 +50,10 @@ export const fionaCommandCallback = async ({ command, ack, logger }) => { await handleComingSoon({ command, ack, logger, subCommand: 'ask', text: ASK_NOT_YET_TEXT }); break; case 'search': - await handleComingSoon({ - command, - ack, - logger, - subCommand: 'search', - text: SEARCH_NOT_YET_TEXT, - }); + await handleComingSoon({ command, ack, logger, subCommand: 'search', text: SEARCH_NOT_YET_TEXT }); + break; + case 'escalate': + await handleEscalate({ command, ack, respond, client, logger }); break; default: await handleUnknown({ command, ack, logger, subCommand }); @@ -121,3 +127,64 @@ async function handleUnknown({ command, ack, logger, subCommand }) { } fireAndForgetRecord({ command, logger, interactionType: 'slash_unknown' }); } + +function isDmChannel(command) { + return command.channel_name === 'directmessage' || (command.channel_id || '').startsWith('D'); +} + +async function handleEscalate({ command, ack, respond, client, logger }) { + try { + await ack(); + } catch (err) { + logger?.error?.(`Failed to acknowledge /fiona escalate: ${err.name}`); + return; + } + + if (!hasRequiredFields(command)) { + logger?.warn?.('Missing required slash command fields; skipping escalate'); + return; + } + + const { allowed, retryAfterMs } = checkRateLimit(command.user_id); + if (!allowed) { + const minutes = Math.ceil(retryAfterMs / 60000); + await respond({ + response_type: 'ephemeral', + text: `:no_entry: You've reached the request limit. Please wait ${minutes} minute${minutes !== 1 ? 's' : ''} before trying again.`, + }); + recordInteraction({ + ...slashInteractionRecord(command, 'slash_escalate'), + status: 'error', + errorType: 'rate_limited', + rateLimited: true, + logger, + }).catch((err) => logger?.warn?.(`Failed to record slash_escalate interaction: ${err.name}`)); + return; + } + + const dm = isDmChannel(command); + const result = await postEscalation({ + client, + userId: command.user_id, + teamId: command.team_id, + channelId: command.channel_id, + threadTs: null, + messageTs: command.trigger_id, + source: 'slash_escalate', + isDm: dm, + logger, + }); + + if (result.ok) { + await respond({ response_type: 'ephemeral', text: dm ? ESCALATE_DM_TEXT : ESCALATE_CONFIRM_TEXT }); + } else { + await respond({ response_type: 'ephemeral', text: ESCALATE_ERROR_TEXT }); + recordInteraction({ + ...slashInteractionRecord(command, 'slash_escalate'), + status: 'error', + errorType: result.errorType, + rateLimited: false, + logger, + }).catch((err) => logger?.warn?.(`Failed to record slash_escalate interaction: ${err.name}`)); + } +} diff --git a/apps/fiona-slack/tests/agent/escalation.test.js b/apps/fiona-slack/tests/agent/escalation.test.js new file mode 100644 index 00000000..280e6dc0 --- /dev/null +++ b/apps/fiona-slack/tests/agent/escalation.test.js @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockGetUser = jest.fn(); +const mockRecordInteraction = jest.fn().mockResolvedValue(undefined); +const mockRecordFeedback = jest.fn().mockResolvedValue(undefined); +const mockSummarize = jest.fn(); + +jest.unstable_mockModule('../../src/agent/slack-users-store.js', () => ({ getUser: mockGetUser })); +jest.unstable_mockModule('../../src/agent/interaction-store.js', () => ({ recordInteraction: mockRecordInteraction })); +jest.unstable_mockModule('../../src/agent/feedback-store.js', () => ({ recordFeedback: mockRecordFeedback })); +jest.unstable_mockModule('../../src/agent/llm-caller.js', () => ({ summarizeForEscalation: mockSummarize })); + +const { postEscalation } = await import('../../src/agent/escalation.js'); + +function makeClient() { + return { + conversations: { + replies: jest.fn().mockResolvedValue({ messages: [ + { user: 'U1', text: 'I need help with the ODS' }, + { bot_id: 'B1', text: 'Here is some info' }, + ] }), + history: jest.fn().mockResolvedValue({ messages: [ + { bot_id: 'B1', text: 'reply two' }, + { user: 'U1', text: 'message one' }, + ] }), + }, + chat: { + postMessage: jest.fn().mockResolvedValue({ ts: '111.222' }), + getPermalink: jest.fn().mockResolvedValue({ permalink: 'https://slack.test/p1' }), + }, + }; +} + +const baseArgs = () => ({ + client: makeClient(), + userId: 'U1', + teamId: 'T1', + channelId: 'C1', + threadTs: '999.000', + messageTs: '999.000', + source: 'slash_escalate', + logger: { warn: jest.fn(), error: jest.fn() }, +}); + +describe('postEscalation', () => { + beforeEach(() => { + jest.clearAllMocks(); + process.env.ESCALATION_CHANNEL = 'C_ESCALATE'; + delete process.env.ESCALATION_USERGROUP_ID; + mockGetUser.mockResolvedValue({ displayName: 'Ada Lovelace' }); + mockSummarize.mockResolvedValue('User is stuck configuring the ODS.'); + }); + + it('returns channel_not_configured when ESCALATION_CHANNEL is unset', async () => { + delete process.env.ESCALATION_CHANNEL; + const result = await postEscalation(baseArgs()); + expect(result).toEqual({ ok: false, errorType: 'channel_not_configured' }); + }); + + it('posts to the configured channel with the display name and a thread link', async () => { + const args = baseArgs(); + const result = await postEscalation(args); + expect(result.ok).toBe(true); + const post = args.client.chat.postMessage.mock.calls[0][0]; + expect(post.channel).toBe('C_ESCALATE'); + const text = JSON.stringify(post.blocks); + expect(text).toContain('Ada Lovelace'); + expect(text).toContain('https://slack.test/p1'); + }); + + it('pings the user group when ESCALATION_USERGROUP_ID is set', async () => { + process.env.ESCALATION_USERGROUP_ID = 'S123'; + const args = baseArgs(); + await postEscalation(args); + const post = args.client.chat.postMessage.mock.calls[0][0]; + expect(JSON.stringify(post.blocks)).toContain(''); + }); + + it('records to both the interactions and feedback containers', async () => { + await postEscalation(baseArgs()); + await new Promise((resolve) => setImmediate(resolve)); + expect(mockRecordInteraction).toHaveBeenCalledWith( + expect.objectContaining({ interactionType: 'slash_escalate', status: 'success' }), + ); + expect(mockRecordFeedback).toHaveBeenCalledWith( + expect.objectContaining({ interactionType: 'slash_escalate', value: 'escalation' }), + ); + }); + + it('posts the transcript as a threaded reply to the escalation message', async () => { + const args = baseArgs(); + await postEscalation(args); + const primaryTs = (await args.client.chat.postMessage.mock.results[0].value).ts; + const replyCall = args.client.chat.postMessage.mock.calls.find((c) => c[0].thread_ts === primaryTs); + expect(replyCall).toBeDefined(); + }); + + it('still posts (transcript-only) when the summary fails', async () => { + mockSummarize.mockResolvedValue(null); + const args = baseArgs(); + const result = await postEscalation(args); + expect(result.ok).toBe(true); + expect(args.client.chat.postMessage).toHaveBeenCalled(); + expect(JSON.stringify(args.client.chat.postMessage.mock.calls[0][0].blocks)).not.toContain('*Summary:*'); + }); + + it('returns post_failed when chat.postMessage throws', async () => { + const args = baseArgs(); + args.client.chat.postMessage.mockRejectedValueOnce(new Error('not_in_channel')); + const result = await postEscalation(args); + expect(result).toEqual({ ok: false, errorType: 'post_failed' }); + }); + + it('skips the permalink lookup for DM escalations', async () => { + const args = { ...baseArgs(), isDm: true, threadTs: null }; + const result = await postEscalation(args); + expect(result.ok).toBe(true); + expect(args.client.chat.getPermalink).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/fiona-slack/tests/agent/feedback-store-cosmos.test.js b/apps/fiona-slack/tests/agent/feedback-store-cosmos.test.js index e817be44..ae447286 100644 --- a/apps/fiona-slack/tests/agent/feedback-store-cosmos.test.js +++ b/apps/fiona-slack/tests/agent/feedback-store-cosmos.test.js @@ -201,4 +201,32 @@ describe('recordFeedback - with connection string', () => { expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Failed to record feedback')); }); + + it('includes interactionType on the doc when provided', async () => { + await recordFeedback({ + userId: 'U900', + channelId: 'C900', + messageTs: 'trigger-xyz', + value: 'escalation', + interactionType: 'slash_escalate', + userMessage: 'transcript here', + botResponse: 'summary here', + }); + const [doc] = mockUpsert.mock.calls[0]; + expect(doc.interactionType).toBe('slash_escalate'); + expect(doc.value).toBe('escalation'); + }); + + it('omits interactionType from the doc when not provided', async () => { + await recordFeedback({ + userId: 'U901', + channelId: 'C901', + messageTs: '1234567890.000099', + value: 'good-feedback', + userMessage: null, + botResponse: null, + }); + const [doc] = mockUpsert.mock.calls[0]; + expect(doc).not.toHaveProperty('interactionType'); + }); }); diff --git a/apps/fiona-slack/tests/agent/llm-caller.summarize-noclient.test.js b/apps/fiona-slack/tests/agent/llm-caller.summarize-noclient.test.js new file mode 100644 index 00000000..8795fce0 --- /dev/null +++ b/apps/fiona-slack/tests/agent/llm-caller.summarize-noclient.test.js @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockCreate = jest.fn(); +jest.unstable_mockModule('openai', () => ({ + OpenAI: jest.fn().mockImplementation(() => ({ + chat: { completions: { create: mockCreate } }, + })), +})); + +delete process.env.PERPLEXITY_API_KEY; +const { summarizeForEscalation } = await import('../../src/agent/llm-caller.js'); + +describe('summarizeForEscalation (no client)', () => { + beforeEach(() => jest.clearAllMocks()); + + it('returns null when no LLM client is configured, without calling the LLM', async () => { + const result = await summarizeForEscalation('*<@U1>:* hello'); + expect(result).toBeNull(); + expect(mockCreate).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/fiona-slack/tests/agent/llm-caller.summarize.test.js b/apps/fiona-slack/tests/agent/llm-caller.summarize.test.js new file mode 100644 index 00000000..8f45f564 --- /dev/null +++ b/apps/fiona-slack/tests/agent/llm-caller.summarize.test.js @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockCreate = jest.fn(); +jest.unstable_mockModule('openai', () => ({ + OpenAI: jest.fn().mockImplementation(() => ({ + chat: { completions: { create: mockCreate } }, + })), +})); + +process.env.PERPLEXITY_API_KEY = 'test-key'; +const { summarizeForEscalation } = await import('../../src/agent/llm-caller.js'); + +describe('summarizeForEscalation', () => { + beforeEach(() => jest.clearAllMocks()); + + it('returns the trimmed model summary on success', async () => { + mockCreate.mockResolvedValue({ choices: [{ message: { content: ' User wants SIS help. ' } }] }); + const result = await summarizeForEscalation('*<@U1>:* help with SIS'); + expect(result).toBe('User wants SIS help.'); + }); + + it('returns null for empty transcript without calling the LLM', async () => { + const result = await summarizeForEscalation(' '); + expect(result).toBeNull(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it('returns null and warns when the LLM call throws', async () => { + mockCreate.mockRejectedValue(new Error('boom')); + const logger = { warn: jest.fn() }; + const result = await summarizeForEscalation('*<@U1>:* hi', logger); + expect(result).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('escalation summary')); + }); +}); diff --git a/apps/fiona-slack/tests/listeners/commands/fiona.test.js b/apps/fiona-slack/tests/listeners/commands/fiona.test.js index 47134638..9f0450a3 100644 --- a/apps/fiona-slack/tests/listeners/commands/fiona.test.js +++ b/apps/fiona-slack/tests/listeners/commands/fiona.test.js @@ -18,6 +18,11 @@ jest.unstable_mockModule('../../../src/agent/interaction-store.js', () => ({ jest.unstable_mockModule('../../../src/agent/llm-caller.js', () => { throw new Error('llm-caller must not be imported by /fiona slash command handlers'); }); + +const mockPostEscalation = jest.fn().mockResolvedValue({ ok: true, errorType: null }); +jest.unstable_mockModule('../../../src/agent/escalation.js', () => ({ + postEscalation: mockPostEscalation, +})); const { fionaCommandCallback } = await import('../../../src/listeners/commands/fiona.js'); // Flush microtasks and the setImmediate queue so fire-and-forget Promises settle before assertions. @@ -67,6 +72,11 @@ describe('fionaCommandCallback', () => { expect(mockAck).toHaveBeenCalledWith(expect.stringContaining('/fiona search')); }); + it('ack() response includes /fiona escalate', async () => { + await fionaCommandCallback({ command: mockCommand, ack: mockAck, logger: mockLogger }); + expect(mockAck).toHaveBeenCalledWith(expect.stringContaining('/fiona escalate')); + }); + it('calls recordInteraction with interactionType slash_help', async () => { await fionaCommandCallback({ command: mockCommand, ack: mockAck, logger: mockLogger }); await flushMicrotasks(); @@ -322,4 +332,72 @@ describe('fionaCommandCallback', () => { expect(mockAck).toHaveBeenCalledWith(expect.stringContaining('Fiona')); }); }); + + describe('escalate sub-command', () => { + let mockRespond; + let mockClient; + + beforeEach(() => { + jest.clearAllMocks(); + mockRespond = jest.fn().mockResolvedValue(undefined); + mockClient = {}; + mockPostEscalation.mockResolvedValue({ ok: true, errorType: null }); + }); + + const cmd = (over = {}) => ({ + user_id: 'U1', team_id: 'T1', channel_id: 'C1', trigger_id: 'trig-1', text: 'escalate', ...over, + }); + + it('acks and delegates to postEscalation with source slash_escalate', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(ack).toHaveBeenCalledTimes(1); + expect(mockPostEscalation).toHaveBeenCalledWith( + expect.objectContaining({ source: 'slash_escalate', userId: 'U1', channelId: 'C1' }), + ); + }); + + it('sends the channel confirmation on success in a channel', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('escalated to #escalation') }), + ); + }); + + it('sends the DM confirmation and marks isDm when invoked in a DM', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ + command: cmd({ channel_id: 'D9', channel_name: 'directmessage' }), + ack, respond: mockRespond, client: mockClient, logger: mockLogger, + }); + expect(mockPostEscalation).toHaveBeenCalledWith(expect.objectContaining({ isDm: true })); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: '✅ A team member will follow up shortly.' }), + ); + }); + + it('sends an ephemeral error when postEscalation fails', async () => { + mockPostEscalation.mockResolvedValue({ ok: false, errorType: 'post_failed' }); + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('could not escalate') }), + ); + }); + + it('does not call postEscalation and warns the user when rate limited', async () => { + // Exhaust the limiter for this user (default RATE_LIMIT_MAX_REQUESTS=20). + const { checkRateLimit } = await import('../../../src/agent/rate-limiter.js'); + for (let i = 0; i < 25; i++) checkRateLimit('U_RL'); + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ + command: cmd({ user_id: 'U_RL' }), ack, respond: mockRespond, client: mockClient, logger: mockLogger, + }); + expect(mockPostEscalation).not.toHaveBeenCalled(); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('request limit') }), + ); + }); + }); }); diff --git a/apps/usage-report-function/lib/cosmos-queries.js b/apps/usage-report-function/lib/cosmos-queries.js index 6187566b..135e0504 100644 --- a/apps/usage-report-function/lib/cosmos-queries.js +++ b/apps/usage-report-function/lib/cosmos-queries.js @@ -146,7 +146,8 @@ export async function getFeedbackResponseRate(interactionsContainer, feedbackCon `SELECT VALUE COUNT(1) FROM feedback f WHERE f.deploymentType = @deploymentType - AND f.timestamp > @oneWeekAgoISO`, + AND f.timestamp > @oneWeekAgoISO + AND f["value"] IN ('good-feedback', 'bad-feedback')`, deploymentType, oneWeekAgoISO, ), diff --git a/apps/usage-report-function/test/unit/cosmos-queries.test.js b/apps/usage-report-function/test/unit/cosmos-queries.test.js index a693a0c4..2b5f2604 100644 --- a/apps/usage-report-function/test/unit/cosmos-queries.test.js +++ b/apps/usage-report-function/test/unit/cosmos-queries.test.js @@ -196,5 +196,17 @@ describe('cosmos-queries', () => { ); expect(result).toBe(0); }); + + it('feedback count query uses value allow-list to exclude escalation rows', async () => { + mockInteractionsContainer.items.query.mockReturnValue({ + fetchAll: jest.fn().mockResolvedValue({ resources: [100] }), + }); + mockFeedbackContainer.items.query.mockReturnValue({ + fetchAll: jest.fn().mockResolvedValue({ resources: [10] }), + }); + await getFeedbackResponseRate(mockInteractionsContainer, mockFeedbackContainer, deploymentType, oneWeekAgoISO); + const [feedbackQuerySpec] = mockFeedbackContainer.items.query.mock.calls[0]; + expect(feedbackQuerySpec.query).toContain(`f["value"] IN ('good-feedback', 'bad-feedback')`); + }); }); }); diff --git a/docs/fiona-skills-prd.md b/docs/fiona-skills-prd.md index d71b1fe7..20ab86b8 100644 --- a/docs/fiona-skills-prd.md +++ b/docs/fiona-skills-prd.md @@ -270,9 +270,10 @@ Enabling Skills requires the following updates to the Slack app manifest ## 5. Environment Variables -| Variable | Default | Purpose | -| --------------------- | -------------- | -------------------------------------------------- | -| `ESCALATION_CHANNEL` | `#escalation` | Slack channel where escalation posts are created | +| Variable | Default | Purpose | +| ------------------------- | ------- | ---------------------------------------------------------------- | +| `ESCALATION_CHANNEL` | (unset) | Channel **ID** where escalation posts are created (bot must join) | +| `ESCALATION_USERGROUP_ID` | (unset) | Optional Slack user group ID to @-mention on escalation | --- diff --git a/docs/superpowers/plans/2026-06-22-escalate-to-human-design.md b/docs/superpowers/plans/2026-06-22-escalate-to-human-design.md new file mode 100644 index 00000000..371f997e --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-escalate-to-human-design.md @@ -0,0 +1,213 @@ +# Escalate-to-Human — Design + +> **Status:** Draft (design) \ +> **Date:** 2026-06-22 \ +> **Jira:** [AI-122](https://edfi.atlassian.net/browse/AI-122) — `/fiona escalate` — escalate a conversation to a human \ +> **Related PRD:** [Fiona Skills PRD](../../fiona-skills-prd.md) §2.4 (slash command) and §3 (proactive detection) + +## 1. Summary + +Add the ability for a Fiona user to escalate a conversation to a human team +member. Escalation is reachable two ways: + +1. **Explicit slash command** — `/fiona escalate` (this is the scope of + **AI-122**). +2. **Conversational** — while talking with Fiona, the user can *ask* for a + human in plain language (e.g. "can I talk to a person?", "this isn't + helping"). Fiona detects the intent and offers a one-click escalation. This + is a **separate downstream story**; AI-122 only requires that the slash + command extract a shared `postEscalation` helper that the conversational + story will reuse. + +Both paths converge on a single shared helper that posts a notification to a +configured Slack channel, pings an on-call user group, and records the event. + +## 2. Decisions + +These were settled during brainstorming and against the AI-122 acceptance +criteria: + +| Topic | Decision | +| --- | --- | +| Detection (conversational) | **Hybrid** — fast keyword gate, then a cheap LLM confirm before offering. | +| Conversational offer UX | On confirmed intent, Fiona **offers escalation instead of answering** that turn (Yes / No buttons). A single Yes/No offer is used for **all** conversational triggers — explicit asks and inferred frustration alike — so a misread message can never escalate without consent. | +| `/fiona escalate` | Escalates **immediately** (the command is itself explicit intent); no confirmation step. | +| Post content | Short **LLM summary** + the **recent transcript** below it (transcript posted as a threaded reply to keep the channel scannable). | +| Notify | Post to the channel **and** @-mention a configurable user group. *(Beyond AI-122's AC; carried because it also serves the proactive story.)* | +| Channel config | Configured by **channel ID** for reliability, stored in the AC-mandated env var `ESCALATION_CHANNEL` (see §8). | +| Persistence | Record `slash_escalate` to **both** the `interactions` and `feedback` Cosmos containers, per AI-122. | + +## 3. AI-122 Acceptance Criteria (verbatim) → coverage + +| # | Acceptance criterion | Where covered | +| --- | --- | --- | +| 1 | `/fiona escalate` posts to the configured escalation channel (default `#escalation`) with: requesting user's display name, link back to originating channel/thread, and a summary of recent conversation history. | §4, §5 | +| 2 | Ephemeral confirmation to the user: *"✅ Your conversation has been escalated to #escalation. A team member will follow up shortly."* | §5 | +| 3 | In a DM (no thread to escalate), respond ephemerally: *"✅ A team member will follow up shortly."* | §5, §7 | +| 4 | If the channel can't be found or Fiona lacks permission, send an ephemeral error and log the failure. | §7 | +| 5 | Escalation channel configurable via `ESCALATION_CHANNEL` env var (default `#escalation`). | §8 | +| 6 | Rate limiting applies. | §5 | +| 7 | Interaction recorded to Cosmos DB with `interactionType: slash_escalate` to **both** the `interactions` and `feedback` containers. | §6 | +| 8 | A shared `postEscalation` helper is extracted for reuse by the proactive escalation story. | §4 | +| 9 | Unit tests cover: post contains display name + thread link; ephemeral confirmation sent; DM edge case; permission error caught and surfaced ephemerally; both Cosmos containers written; `ESCALATION_CHANNEL` respected. | §10 | + +## 4. Module structure + +The PRD sketches a `src/skills/` tree, but the codebase doesn't use one; this +design follows the **existing** `listeners/` + `agent/` layout. + +```none +src/agent/ + escalation.js # postEscalation(): the shared core flow (AI-122 AC #8) + escalation-intent.js # keyword gate + LLM confirm + maybeOfferEscalation() (proactive story) +src/listeners/ + views/escalation_block.js # the Yes/No offer Block Kit block (proactive story) + actions/escalation.js # escalation_yes / escalation_no button handlers (proactive story) + actions/index.js # (edit) register the two new actions + commands/fiona.js # (edit) add the `escalate` sub-command (AI-122) + events/app_mention.js # (edit) insert detection step before normal answer (proactive story) + assistant/message.js # (edit) insert detection step before normal answer (proactive story) +src/agent/llm-caller.js # (edit) add non-streaming summarizeForEscalation() +``` + +**AI-122 ships:** `escalation.js` (the `postEscalation` helper), the +`fiona.js` `escalate` sub-command, the `summarizeForEscalation()` helper, env +vars/config, and tests. The proactive-story files are listed here for +continuity but are implemented under the separate story. + +### `postEscalation({ client, userId, teamId, channelId, threadTs, source, logger })` + +The shared core (AI-122 AC #8). Steps: + +1. **Resolve display name** via `getUser()` (`slack-users-store`); fall back to + the Slack ID if Cosmos is unconfigured or misses. +2. **Build the transcript** with the existing `buildThreadHistory()`. +3. **Generate a summary** via `summarizeForEscalation()` (new non-streaming + Perplexity call). On failure, **degrade gracefully** to a transcript-only + post — escalation must never fail because the summary did. +4. **Build the Block Kit message**: header → user-group mention + (``) → summary → context block (user display name, permalink to + the source thread via `chat.getPermalink`, timestamp). Post the **full + transcript as a threaded reply** to that message. +5. **Post** to the channel ID from `ESCALATION_CHANNEL` via `chat.postMessage`. +6. **Record** analytics to both Cosmos containers (§6). +7. **Return** `{ ok, error }` so the caller picks the right ephemeral message. + +`source` is `'slash_escalate'` or `'auto_escalation'` so analytics and copy can +differ by entry point. + +## 5. `/fiona escalate` sub-command (AI-122) + +Added as a new `case 'escalate'` in `src/listeners/commands/fiona.js`: + +1. **Rate limit.** The existing slash handlers don't rate-limit; this path adds + a `checkRateLimit(user_id)` guard (the `handleRateLimitedInteraction` wrapper + is built around `say`, so the slash path calls `checkRateLimit` directly and + responds ephemerally via `ack`/`respond`). On limit: ephemeral notice, record + a rate-limited interaction, stop. +2. **DM check.** If the invocation has no escalatable thread (DM / no + `channel`/thread context), send the ephemeral *"✅ A team member will follow + up shortly."* and still record the interaction (AC #3). Whether a DM also + posts to the channel is covered in §7. +3. **Escalate.** Call `postEscalation({ source: 'slash_escalate', ... })`. +4. **Confirm / error.** On success: ephemeral *"✅ Your conversation has been + escalated to #escalation. A team member will follow up shortly."* On failure + (channel missing / no permission): ephemeral error + log (AC #4). + +## 6. Analytics & persistence (AI-122 AC #7) + +`postEscalation` records `interactionType: slash_escalate` (or +`auto_escalation` for the proactive path) to **both** Cosmos containers: + +- **`interactions`** — via the existing `recordInteraction()` (user/team/ + channel/thread/message identifiers, status, timestamp). +- **`feedback`** — via `recordFeedback()` (or a thin escalation variant), + capturing the escalation context (user message / summary, thread link). + +Both writes are best-effort and logged on failure; a Cosmos outage must not +block the user-facing escalation. + +New `interactionType` values: `slash_escalate`, `auto_escalation`, and (for the +proactive buttons) `escalation_confirmed` / `escalation_declined`. + +## 7. Error handling & edge cases + +| Case | Behavior | +| --- | --- | +| `ESCALATION_CHANNEL` unset / channel not found / bot not a member / no permission | Ephemeral error to the user, log the failure (AC #4). | +| Summary LLM call fails | Post with transcript only; do not fail the escalation. | +| Cosmos unavailable | Name lookup falls back to the Slack ID; both records are fire-and-forget. | +| DM with no public thread | Send the AC #3 ephemeral *and* still post to the escalation channel carrying the DM transcript — without it responders have no context to follow up on (the DM isn't visible to them). The permalink/thread link is omitted since the DM isn't linkable for others. | +| LLM-confirm error (proactive path) | Fall back to **offering** escalation (the keyword already matched) and log. *Fail-open chosen so a flagged user is never silently ignored.* | + +## 8. Configuration & Slack app + +- **Env vars:** + - `ESCALATION_CHANNEL` — the destination channel. **Value is a channel ID** + (e.g. `C0123456789`) for reliable posting; the AC's `#escalation` default is + illustrative. Added to `.env.sample` and the container Bicep + (`infra/fiona-slack-container/main.bicep`). + - `ESCALATION_USERGROUP_ID` — the on-call user group to @-mention (e.g. + `S0123456789`). *(Enhancement beyond AI-122.)* +- **Slack scopes:** none new required. A user-group mention is just message text + (``), and `chat:write` already exists. +- **Deploy step:** the bot **must be a member** of the escalation channel — + documented in deploy notes. + +## 9. Conversational path (proactive story — context only) + +Implemented under the separate downstream story; included here to show the +shared helper's reuse. + +- `escalation-intent.js`: `detectEscalationKeyword(text)` (curated phrase list) + → `confirmEscalationIntent(text, logger)` (tiny LLM yes/no) → + `maybeOfferEscalation({...})` which posts the offer block and returns `true` + when intent is confirmed. +- In `app_mention.js` and `message.js`, after the rate-limit check and text + extraction and before `buildThreadHistory`/`callLLM`: + `if (await maybeOfferEscalation({ client, text, ... })) return;` — a single + shared call keeps the two handlers in lockstep. +- `escalation_block.js`: Yes / No buttons (`escalation_yes` / `escalation_no`), + with minimal `{ channelId, threadTs }` context in the button `value`. +- `actions/escalation.js`: `escalation_yes` calls the shared `postEscalation`; + `escalation_no` posts a friendly ephemeral ack. Registered in + `actions/index.js`. + +## 10. Testing (AI-122 AC #9) + +Unit tests mirroring the existing `tests/` layout. For AI-122: + +- Escalation post contains the user **display name** and a **thread link**. +- Ephemeral confirmation is sent to the invoking user. +- DM edge case handled with the AC #3 message. +- Permission/channel error is caught and surfaced ephemerally + logged. +- **Both** Cosmos containers (`interactions` and `feedback`) are written. +- `ESCALATION_CHANNEL` env var is respected (post targets the configured + channel). +- Rate limiting applies on the slash path. +- Summary-failure degradation (transcript-only post). + +Proactive-story tests (separate story): keyword + confirm (mocked LLM), the +"offer instead of answer" flow, and both button handlers. + +## 11. Resolved decisions + +These were the previously-open items; all are now settled and reflected in the +sections above. + +1. **`ESCALATION_CHANNEL` value format** — **Resolved: store a channel ID.** The + AC-named `ESCALATION_CHANNEL` variable holds a channel ID (e.g. + `C0123456789`) rather than a `#name`, avoiding fragile runtime name→ID + lookup. See §8. +2. **User-group ping** — **Resolved: keep it.** `ESCALATION_USERGROUP_ID` is + carried as an enhancement beyond AI-122's AC because it also serves the + proactive story. When unset, escalation still posts to the channel (the ping + is simply omitted). See §2, §8. +3. **DM → channel post** — **Resolved: yes, a DM escalation also posts to the + channel.** A DM escalation sends the AC #3 ephemeral *and* posts the DM + transcript to the escalation channel, because responders otherwise have no + context to follow up on (the DM is invisible to them). The permalink is + omitted since the DM isn't linkable for others. Note this surfaces an + otherwise-private DM transcript into the escalation channel — acceptable + because escalation is an explicit user request for a human to see the + conversation. See §7. diff --git a/docs/superpowers/plans/2026-06-22-escalate-to-human-slash-command.md b/docs/superpowers/plans/2026-06-22-escalate-to-human-slash-command.md new file mode 100644 index 00000000..ee1889ee --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-escalate-to-human-slash-command.md @@ -0,0 +1,970 @@ +# `/fiona escalate` (AI-122) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `/fiona escalate` slash command that posts a summarized escalation (with transcript) to a configured Slack channel, pings an on-call user group, records the event to both Cosmos containers, and confirms to the user — extracting a reusable `postEscalation` helper for the downstream proactive-escalation story. + +**Architecture:** A shared `postEscalation()` helper in `src/agent/escalation.js` performs the channel post + analytics. The slash sub-command in `src/listeners/commands/fiona.js` handles rate limiting, DM detection, and the ephemeral confirmation, delegating the post to `postEscalation`. A new non-streaming `summarizeForEscalation()` in `llm-caller.js` produces the summary; `recordFeedback()` gains an optional `interactionType` so the feedback container can carry `slash_escalate`. + +**Tech Stack:** Node.js 22 (ESM), Slack Bolt 4, OpenAI SDK (Perplexity-compatible), Azure Cosmos SDK, Jest 29 with `jest.unstable_mockModule`, Biome. + +## Global Constraints + +- **Runtime:** Node.js 22+, ES modules (`"type": "module"`); use `import`/`export`, never `require`. +- **License header:** every **new** `.js` file starts with the 4-line Apache-2.0 header (see any existing `src/**/*.js`). +- **Tests:** Jest via `npm test` from `apps/fiona-slack`; ESM mocks use `jest.unstable_mockModule(path, factory)` **before** `await import(...)` of the module under test. +- **Lint:** `npm run lint` (Biome) must pass; `npm run lint:fix` to autofix. +- **Commits:** conventional-commit style; **every commit message ends with the trailer** `Co-Authored-By: Claude Opus 4.8 (1M context) ` (shown in each commit step). Work is on branch `AI-122`. +- **Verbatim user-facing copy (AI-122):** + - Channel confirmation: `✅ Your conversation has been escalated to #escalation. A team member will follow up shortly.` + - DM confirmation: `✅ A team member will follow up shortly.` +- **Env var (AI-122):** `ESCALATION_CHANNEL` — the destination, **value is a channel ID** (e.g. `C0123456789`). Enhancement var: `ESCALATION_USERGROUP_ID` (e.g. `S0123456789`), optional; omit the ping when unset. +- **Cosmos:** record `interactionType: 'slash_escalate'` to **both** the `interactions` and `feedback` containers; all Cosmos writes are best-effort (never block or fail the user path). +- **No regressions:** the existing `tests/listeners/commands/fiona.test.js` guard (llm-caller must not be statically reachable) is preserved by mocking `escalation.js` in that test (Task 4). + +## File Structure + +| File | Responsibility | Task | +| --- | --- | --- | +| `src/agent/llm-caller.js` (modify) | add `summarizeForEscalation(transcriptText, logger)` — non-streaming summary | 1 | +| `src/agent/feedback-store.js` (modify) | add optional `interactionType` to `recordFeedback` doc | 2 | +| `src/agent/escalation.js` (create) | `postEscalation({...})` shared helper + internal transcript builder | 3 | +| `src/listeners/commands/fiona.js` (modify) | `escalate` sub-command: rate limit, DM detection, delegate, confirm | 4 | +| `tests/listeners/commands/fiona.test.js` (modify) | mock `escalation.js`; add escalate tests | 4 | +| `apps/fiona-slack/.env.sample`, `manifest.json`, `infra/fiona-slack-container/main.bicep`, `docs/fiona-skills-prd.md` (modify) | config + docs wiring | 5 | + +Tasks are ordered so each only consumes interfaces produced by earlier tasks. + +--- + +## Task 1: `summarizeForEscalation()` in llm-caller.js + +**Files:** +- Modify: `src/agent/llm-caller.js` +- Test: `tests/agent/llm-caller.summarize.test.js` (create) + +**Interfaces:** +- Produces: `summarizeForEscalation(transcriptText: string, logger?) => Promise` — a 2–4 sentence summary, or `null` when the LLM is unconfigured, the input is empty, or the call fails. + +- [ ] **Step 1: Write the failing test** + +Create `tests/agent/llm-caller.summarize.test.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockCreate = jest.fn(); +jest.unstable_mockModule('openai', () => ({ + OpenAI: jest.fn().mockImplementation(() => ({ + chat: { completions: { create: mockCreate } }, + })), +})); + +process.env.PERPLEXITY_API_KEY = 'test-key'; +const { summarizeForEscalation } = await import('../../src/agent/llm-caller.js'); + +describe('summarizeForEscalation', () => { + beforeEach(() => jest.clearAllMocks()); + + it('returns the trimmed model summary on success', async () => { + mockCreate.mockResolvedValue({ choices: [{ message: { content: ' User wants SIS help. ' } }] }); + const result = await summarizeForEscalation('*<@U1>:* help with SIS'); + expect(result).toBe('User wants SIS help.'); + }); + + it('returns null for empty transcript without calling the LLM', async () => { + const result = await summarizeForEscalation(' '); + expect(result).toBeNull(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it('returns null and warns when the LLM call throws', async () => { + mockCreate.mockRejectedValue(new Error('boom')); + const logger = { warn: jest.fn() }; + const result = await summarizeForEscalation('*<@U1>:* hi', logger); + expect(result).toBeNull(); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('escalation summary')); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd apps/fiona-slack && npm test -- llm-caller.summarize` +Expected: FAIL — `summarizeForEscalation is not a function`. + +- [ ] **Step 3: Add the implementation** + +In `src/agent/llm-caller.js`, after the `callLLM` export, add: + +```javascript +// ─── Escalation Summary ───────────────────────────────────────────────────── +const ESCALATION_SUMMARY_SYSTEM_PROMPT = + 'You summarize a Slack conversation between a user and Fiona (an Ed-Fi AI assistant) for a human support team. ' + + 'In 2-4 sentences, state what the user is trying to do and where they got stuck. ' + + 'Be factual and concise. Do not add greetings or sign-offs.'; + +/** + * Produce a short human-readable summary of a conversation transcript for an + * escalation post. Non-streaming. Returns null when the LLM is unconfigured, + * the transcript is empty, or the call fails — callers degrade to transcript-only. + * + * @param {string} transcriptText + * @param {{ warn?: (msg: string) => void }} [logger] + * @returns {Promise} + */ +export async function summarizeForEscalation(transcriptText, logger) { + if (!perplexityClient) return null; + if (!transcriptText || !transcriptText.trim()) return null; + + try { + const response = await perplexityClient.chat.completions.create({ + model: PERPLEXITY_API_MODEL, + messages: [ + { role: 'system', content: ESCALATION_SUMMARY_SYSTEM_PROMPT }, + { role: 'user', content: transcriptText }, + ], + stream: false, + }); + const summary = response?.choices?.[0]?.message?.content; + return typeof summary === 'string' && summary.trim() ? summary.trim() : null; + } catch (error) { + logger?.warn?.(`Failed to generate escalation summary: ${error.message}`); + return null; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd apps/fiona-slack && npm test -- llm-caller.summarize` +Expected: PASS (3 tests). + +- [ ] **Step 5: Lint and commit** + +```bash +cd apps/fiona-slack && npm run lint +git add apps/fiona-slack/src/agent/llm-caller.js apps/fiona-slack/tests/agent/llm-caller.summarize.test.js +git commit -m "$(cat <<'EOF' +feat(AI-122): add summarizeForEscalation helper + +Non-streaming Perplexity call that summarizes a conversation transcript +for escalation posts. Returns null on empty input, missing client, or +error so callers can degrade to transcript-only. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +## Task 2: optional `interactionType` on `recordFeedback` + +So the `feedback` container row for an escalation can carry `interactionType: 'slash_escalate'` (AI-122 AC #7), without affecting normal thumbs feedback rows. + +**Files:** +- Modify: `src/agent/feedback-store.js:140-167` +- Test: `tests/agent/feedback-store-cosmos.test.js` (add cases) + +**Interfaces:** +- Produces: `recordFeedback({ ..., interactionType?: string })` — when `interactionType` is a non-empty string it is included on the persisted doc; otherwise the doc shape is unchanged. + +- [ ] **Step 1: Write the failing tests** + +Append inside the `describe('recordFeedback - with connection string', ...)` block in `tests/agent/feedback-store-cosmos.test.js`: + +```javascript + it('includes interactionType on the doc when provided', async () => { + await recordFeedback({ + userId: 'U900', + channelId: 'C900', + messageTs: 'trigger-xyz', + value: 'escalation', + interactionType: 'slash_escalate', + userMessage: 'transcript here', + botResponse: 'summary here', + }); + const [doc] = mockUpsert.mock.calls[0]; + expect(doc.interactionType).toBe('slash_escalate'); + expect(doc.value).toBe('escalation'); + }); + + it('omits interactionType from the doc when not provided', async () => { + await recordFeedback({ + userId: 'U901', + channelId: 'C901', + messageTs: '1234567890.000099', + value: 'good-feedback', + userMessage: null, + botResponse: null, + }); + const [doc] = mockUpsert.mock.calls[0]; + expect(doc).not.toHaveProperty('interactionType'); + }); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd apps/fiona-slack && npm test -- feedback-store-cosmos` +Expected: FAIL — first new test sees `doc.interactionType` is `undefined`. + +- [ ] **Step 3: Add the parameter and conditional field** + +In `src/agent/feedback-store.js`, add `interactionType` to the destructured params of `recordFeedback`: + +```javascript +export async function recordFeedback({ + userId, + channelId, + messageTs, + value, + reason, + userMessage, + botResponse, + interactionType, + logger, +}) { +``` + +Then in the `doc` object, add the conditional field immediately after `botResponse,`: + +```javascript + userMessage, + botResponse, + ...(interactionType ? { interactionType } : {}), + deploymentType: process.env.DEPLOYMENT_TYPE || 'local', +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd apps/fiona-slack && npm test -- feedback-store-cosmos` +Expected: PASS (existing + 2 new). + +- [ ] **Step 5: Lint and commit** + +```bash +cd apps/fiona-slack && npm run lint +git add apps/fiona-slack/src/agent/feedback-store.js apps/fiona-slack/tests/agent/feedback-store-cosmos.test.js +git commit -m "$(cat <<'EOF' +feat(AI-122): allow optional interactionType on feedback records + +Lets escalation events be recorded to the feedback container tagged with +interactionType (e.g. slash_escalate) without changing normal feedback rows. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +## Task 3: `postEscalation()` shared helper — `src/agent/escalation.js` + +**Files:** +- Create: `src/agent/escalation.js` +- Test: `tests/agent/escalation.test.js` (create) + +**Interfaces:** +- Consumes: `getUser` (Task pre-existing), `recordInteraction`, `recordFeedback` (Task 2), `summarizeForEscalation` (Task 1). +- Produces: + `postEscalation({ client, userId, teamId, channelId, threadTs?, messageTs, source, isDm?, logger }) => Promise<{ ok: boolean, errorType: string | null }>` + - `source`: `'slash_escalate'` | `'auto_escalation'`. `errorType`: `'channel_not_configured'` | `'post_failed'` | `null`. + - Posts to `process.env.ESCALATION_CHANNEL`; pings `process.env.ESCALATION_USERGROUP_ID` when set; posts the transcript as a threaded reply; records to both Cosmos containers (best-effort). + +- [ ] **Step 1: Write the failing tests** + +Create `tests/agent/escalation.test.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockGetUser = jest.fn(); +const mockRecordInteraction = jest.fn().mockResolvedValue(undefined); +const mockRecordFeedback = jest.fn().mockResolvedValue(undefined); +const mockSummarize = jest.fn(); + +jest.unstable_mockModule('../../src/agent/slack-users-store.js', () => ({ getUser: mockGetUser })); +jest.unstable_mockModule('../../src/agent/interaction-store.js', () => ({ recordInteraction: mockRecordInteraction })); +jest.unstable_mockModule('../../src/agent/feedback-store.js', () => ({ recordFeedback: mockRecordFeedback })); +jest.unstable_mockModule('../../src/agent/llm-caller.js', () => ({ summarizeForEscalation: mockSummarize })); + +const { postEscalation } = await import('../../src/agent/escalation.js'); + +function makeClient() { + return { + conversations: { + replies: jest.fn().mockResolvedValue({ messages: [ + { user: 'U1', text: 'I need help with the ODS' }, + { bot_id: 'B1', text: 'Here is some info' }, + ] }), + history: jest.fn().mockResolvedValue({ messages: [ + { bot_id: 'B1', text: 'reply two' }, + { user: 'U1', text: 'message one' }, + ] }), + }, + chat: { + postMessage: jest.fn().mockResolvedValue({ ts: '111.222' }), + getPermalink: jest.fn().mockResolvedValue({ permalink: 'https://slack.test/p1' }), + }, + }; +} + +const baseArgs = () => ({ + client: makeClient(), + userId: 'U1', + teamId: 'T1', + channelId: 'C1', + threadTs: '999.000', + messageTs: '999.000', + source: 'slash_escalate', + logger: { warn: jest.fn(), error: jest.fn() }, +}); + +describe('postEscalation', () => { + beforeEach(() => { + jest.clearAllMocks(); + process.env.ESCALATION_CHANNEL = 'C_ESCALATE'; + delete process.env.ESCALATION_USERGROUP_ID; + mockGetUser.mockResolvedValue({ displayName: 'Ada Lovelace' }); + mockSummarize.mockResolvedValue('User is stuck configuring the ODS.'); + }); + + it('returns channel_not_configured when ESCALATION_CHANNEL is unset', async () => { + delete process.env.ESCALATION_CHANNEL; + const result = await postEscalation(baseArgs()); + expect(result).toEqual({ ok: false, errorType: 'channel_not_configured' }); + }); + + it('posts to the configured channel with the display name and a thread link', async () => { + const args = baseArgs(); + const result = await postEscalation(args); + expect(result.ok).toBe(true); + const post = args.client.chat.postMessage.mock.calls[0][0]; + expect(post.channel).toBe('C_ESCALATE'); + const text = JSON.stringify(post.blocks); + expect(text).toContain('Ada Lovelace'); + expect(text).toContain('https://slack.test/p1'); + }); + + it('pings the user group when ESCALATION_USERGROUP_ID is set', async () => { + process.env.ESCALATION_USERGROUP_ID = 'S123'; + const args = baseArgs(); + await postEscalation(args); + const post = args.client.chat.postMessage.mock.calls[0][0]; + expect(JSON.stringify(post.blocks)).toContain(''); + }); + + it('records to both the interactions and feedback containers', async () => { + await postEscalation(baseArgs()); + expect(mockRecordInteraction).toHaveBeenCalledWith( + expect.objectContaining({ interactionType: 'slash_escalate', status: 'success' }), + ); + expect(mockRecordFeedback).toHaveBeenCalledWith( + expect.objectContaining({ interactionType: 'slash_escalate', value: 'escalation' }), + ); + }); + + it('posts the transcript as a threaded reply to the escalation message', async () => { + const args = baseArgs(); + await postEscalation(args); + const replyCall = args.client.chat.postMessage.mock.calls.find((c) => c[0].thread_ts === '111.222'); + expect(replyCall).toBeDefined(); + }); + + it('still posts (transcript-only) when the summary fails', async () => { + mockSummarize.mockResolvedValue(null); + const args = baseArgs(); + const result = await postEscalation(args); + expect(result.ok).toBe(true); + expect(args.client.chat.postMessage).toHaveBeenCalled(); + }); + + it('returns post_failed when chat.postMessage throws', async () => { + const args = baseArgs(); + args.client.chat.postMessage.mockRejectedValueOnce(new Error('not_in_channel')); + const result = await postEscalation(args); + expect(result).toEqual({ ok: false, errorType: 'post_failed' }); + }); + + it('skips the permalink lookup for DM escalations', async () => { + const args = { ...baseArgs(), isDm: true, threadTs: null }; + const result = await postEscalation(args); + expect(result.ok).toBe(true); + expect(args.client.chat.getPermalink).not.toHaveBeenCalled(); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd apps/fiona-slack && npm test -- escalation.test` +Expected: FAIL — cannot resolve `../../src/agent/escalation.js`. + +- [ ] **Step 3: Create the implementation** + +Create `src/agent/escalation.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { recordFeedback } from './feedback-store.js'; +import { recordInteraction } from './interaction-store.js'; +import { summarizeForEscalation } from './llm-caller.js'; +import { getUser } from './slack-users-store.js'; + +const HISTORY_LIMIT = 20; +const SLACK_BLOCK_TEXT_LIMIT = 2900; // leave headroom under Slack's 3000-char section limit + +/** + * Build a plain-text transcript of the recent conversation. Uses thread replies + * when a threadTs is available, otherwise the channel's recent history. + * + * @returns {Promise} Newline-joined "*Who:* text" lines, or '' on failure. + */ +async function fetchTranscript(client, { channelId, threadTs, logger }) { + try { + let messages; + if (threadTs) { + const res = await client.conversations.replies({ channel: channelId, ts: threadTs, limit: 50 }); + messages = res.messages ?? []; + } else { + const res = await client.conversations.history({ channel: channelId, limit: HISTORY_LIMIT }); + messages = (res.messages ?? []).reverse(); // history returns newest-first + } + return messages + .filter((m) => m.text) + .map((m) => { + const who = m.bot_id ? 'Fiona' : m.user ? `<@${m.user}>` : 'User'; + const text = (m.text ?? '').replace(/^(<@[A-Z0-9]+>\s*)+/, '').trim(); + return text ? `*${who}:* ${text}` : null; + }) + .filter(Boolean) + .join('\n'); + } catch (err) { + logger?.warn?.(`Failed to fetch transcript for escalation: ${err.message}`); + return ''; + } +} + +/** + * Post an escalation to the configured channel and record it. Shared by the + * /fiona escalate slash command and the proactive escalation flow. + * + * @param {Object} params + * @param {import("@slack/web-api").WebClient} params.client + * @param {string} params.userId + * @param {string} [params.teamId] + * @param {string} params.channelId + * @param {string|null} [params.threadTs] + * @param {string} params.messageTs + * @param {'slash_escalate'|'auto_escalation'} params.source + * @param {boolean} [params.isDm] + * @param {import("@slack/logger").Logger} [params.logger] + * @returns {Promise<{ ok: boolean, errorType: string|null }>} + */ +export async function postEscalation({ + client, + userId, + teamId, + channelId, + threadTs = null, + messageTs, + source, + isDm = false, + logger, +}) { + const targetChannel = process.env.ESCALATION_CHANNEL; + if (!targetChannel) { + logger?.warn?.('ESCALATION_CHANNEL is not configured; cannot post escalation.'); + return { ok: false, errorType: 'channel_not_configured' }; + } + + const user = await getUser(userId, logger); + const displayName = user?.displayName || user?.realName || user?.name || `<@${userId}>`; + + const transcript = await fetchTranscript(client, { channelId, threadTs, logger }); + const summary = await summarizeForEscalation(transcript, logger); + + let permalink = null; + if (!isDm) { + try { + const res = await client.chat.getPermalink({ channel: channelId, message_ts: threadTs ?? messageTs }); + permalink = res?.permalink ?? null; + } catch (err) { + logger?.warn?.(`Failed to get permalink for escalation: ${err.message}`); + } + } + + const usergroupId = process.env.ESCALATION_USERGROUP_ID; + const mention = usergroupId ? ` ` : ''; + const locationLink = permalink ? `<${permalink}|View conversation>` : `<#${channelId}>`; + + const headerLines = [ + `${mention}:rotating_light: *Escalation requested* by *${displayName}*`, + `*Where:* ${locationLink}`, + `*When:* ${new Date().toISOString()}`, + ]; + if (summary) headerLines.push(`*Summary:* ${summary}`); + + let postedTs = null; + try { + const res = await client.chat.postMessage({ + channel: targetChannel, + text: `Escalation requested by ${displayName}`, + blocks: [{ type: 'section', text: { type: 'mrkdwn', text: headerLines.join('\n') } }], + }); + postedTs = res?.ts ?? null; + } catch (err) { + logger?.error?.(`Failed to post escalation to ${targetChannel}: ${err.message}`); + return { ok: false, errorType: 'post_failed' }; + } + + if (transcript && postedTs) { + try { + await client.chat.postMessage({ + channel: targetChannel, + thread_ts: postedTs, + text: 'Conversation transcript', + blocks: [ + { + type: 'section', + text: { type: 'mrkdwn', text: `*Transcript:*\n${transcript}`.slice(0, SLACK_BLOCK_TEXT_LIMIT) }, + }, + ], + }); + } catch (err) { + logger?.warn?.(`Failed to post escalation transcript: ${err.message}`); + } + } + + recordInteraction({ + userId, + teamId, + channelId, + threadTs: threadTs ?? messageTs, + messageTs, + interactionType: source, + status: 'success', + errorType: null, + rateLimited: false, + logger, + }).catch((e) => logger?.warn?.(`Failed to record escalation interaction: ${e.message}`)); + + recordFeedback({ + userId, + channelId, + messageTs, + value: 'escalation', + interactionType: source, + reason: null, + userMessage: transcript || null, + botResponse: summary || null, + logger, + }).catch((e) => logger?.warn?.(`Failed to record escalation feedback: ${e.message}`)); + + return { ok: true, errorType: null }; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd apps/fiona-slack && npm test -- escalation.test` +Expected: PASS (8 tests). + +- [ ] **Step 5: Lint and commit** + +```bash +cd apps/fiona-slack && npm run lint +git add apps/fiona-slack/src/agent/escalation.js apps/fiona-slack/tests/agent/escalation.test.js +git commit -m "$(cat <<'EOF' +feat(AI-122): add shared postEscalation helper + +Builds a summarized escalation message with transcript, posts it to the +configured channel (pinging an optional user group), and records the event +to both the interactions and feedback containers. Reused by the proactive +escalation story. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +## Task 4: `/fiona escalate` sub-command — `src/listeners/commands/fiona.js` + +**Files:** +- Modify: `src/listeners/commands/fiona.js` +- Test: `tests/listeners/commands/fiona.test.js` (add escalation mock + escalate suite) + +**Interfaces:** +- Consumes: `postEscalation` (Task 3), `checkRateLimit` (pre-existing), `recordInteraction` (pre-existing). +- Produces: a new `case 'escalate'` in the dispatcher that calls `handleEscalate({ command, ack, respond, client, logger })`. + +> **Why the test changes:** `fiona.js` now statically imports `escalation.js`, which imports `llm-caller.js`. The existing test mocks `llm-caller` to throw on import. Mocking `escalation.js` in the test prevents the real (llm-caller-importing) module from loading, so that guard never fires and existing tests stay green. + +- [ ] **Step 1: Update the dispatcher and add the handler** + +In `src/listeners/commands/fiona.js`, add imports near the top (after the existing `recordInteraction` import): + +```javascript +import { checkRateLimit } from '../../agent/rate-limiter.js'; +import { postEscalation } from '../../agent/escalation.js'; +``` + +Add the copy constants near the other `*_TEXT` constants: + +```javascript +const ESCALATE_CONFIRM_TEXT = + '✅ Your conversation has been escalated to #escalation. A team member will follow up shortly.'; +const ESCALATE_DM_TEXT = '✅ A team member will follow up shortly.'; +const ESCALATE_ERROR_TEXT = + ':warning: Sorry, I could not escalate your conversation right now. Please reach out to the team directly.'; +``` + +Change the dispatcher signature and add the `escalate` case: + +```javascript +export const fionaCommandCallback = async ({ command, ack, respond, client, logger }) => { + logger?.info?.(`/fiona slash command invoked: ${command.text ?? '(empty)'}`); + const subCommand = (command.text ?? '').trim().split(/\s+/)[0].toLowerCase(); + + switch (subCommand) { + case 'help': + case '': + await handleHelp({ command, ack, logger }); + break; + case 'ask': + await handleComingSoon({ command, ack, logger, subCommand: 'ask', text: ASK_NOT_YET_TEXT }); + break; + case 'search': + await handleComingSoon({ command, ack, logger, subCommand: 'search', text: SEARCH_NOT_YET_TEXT }); + break; + case 'escalate': + await handleEscalate({ command, ack, respond, client, logger }); + break; + default: + await handleUnknown({ command, ack, logger, subCommand }); + break; + } +}; +``` + +Add the handler and a DM helper at the end of the file: + +```javascript +function isDmChannel(command) { + return command.channel_name === 'directmessage' || (command.channel_id || '').startsWith('D'); +} + +async function handleEscalate({ command, ack, respond, client, logger }) { + try { + await ack(); + } catch (err) { + logger?.error?.(`Failed to acknowledge /fiona escalate: ${err.name}`); + return; + } + + if (!hasRequiredFields(command)) { + logger?.warn?.('Missing required slash command fields; skipping escalate'); + return; + } + + const { allowed, retryAfterMs } = checkRateLimit(command.user_id); + if (!allowed) { + const minutes = Math.ceil(retryAfterMs / 60000); + await respond({ + response_type: 'ephemeral', + text: `:no_entry: You've reached the request limit. Please wait ${minutes} minute${minutes !== 1 ? 's' : ''} before trying again.`, + }); + recordInteraction({ + ...slashInteractionRecord(command, 'slash_escalate'), + status: 'error', + errorType: 'rate_limited', + rateLimited: true, + logger, + }).catch((err) => logger?.warn?.(`Failed to record slash_escalate interaction: ${err.name}`)); + return; + } + + const dm = isDmChannel(command); + const result = await postEscalation({ + client, + userId: command.user_id, + teamId: command.team_id, + channelId: command.channel_id, + threadTs: null, + messageTs: command.trigger_id, + source: 'slash_escalate', + isDm: dm, + logger, + }); + + if (result.ok) { + await respond({ response_type: 'ephemeral', text: dm ? ESCALATE_DM_TEXT : ESCALATE_CONFIRM_TEXT }); + } else { + await respond({ response_type: 'ephemeral', text: ESCALATE_ERROR_TEXT }); + recordInteraction({ + ...slashInteractionRecord(command, 'slash_escalate'), + status: 'error', + errorType: result.errorType, + rateLimited: false, + logger, + }).catch((err) => logger?.warn?.(`Failed to record slash_escalate interaction: ${err.name}`)); + } +} +``` + +> Note: `slashInteractionRecord(command, type)` already returns `status: 'success'`; the spread is overridden by the explicit `status`/`errorType`/`rateLimited` that follow it. On success, `postEscalation` itself records the interaction, so `handleEscalate` does not re-record. + +Also update the JSDoc `interactionType` enum in `src/agent/interaction-store.js:139` to include the new values (documentation only): + +```javascript + * @param {string} interaction.interactionType - 'app_mention', 'assistant_message', 'slash_help', 'slash_ask', 'slash_search', 'slash_escalate', 'auto_escalation', or 'slash_unknown' +``` + +- [ ] **Step 2: Update the test file's mocks** + +In `tests/listeners/commands/fiona.test.js`, add an `escalation.js` mock alongside the existing mocks (after the `llm-caller` mock, before the `await import`): + +```javascript +const mockPostEscalation = jest.fn().mockResolvedValue({ ok: true, errorType: null }); +jest.unstable_mockModule('../../../src/agent/escalation.js', () => ({ + postEscalation: mockPostEscalation, +})); +``` + +- [ ] **Step 3: Run the existing suite to confirm no regression** + +Run: `cd apps/fiona-slack && npm test -- listeners/commands/fiona` +Expected: PASS — all existing tests still green (the `escalation.js` mock keeps `llm-caller` from loading). + +- [ ] **Step 4: Write the failing escalate tests** + +Append a new `describe` block in `tests/listeners/commands/fiona.test.js`: + +```javascript +describe('escalate sub-command', () => { + let mockRespond; + let mockClient; + + beforeEach(() => { + jest.clearAllMocks(); + mockRespond = jest.fn().mockResolvedValue(undefined); + mockClient = {}; + mockPostEscalation.mockResolvedValue({ ok: true, errorType: null }); + }); + + const cmd = (over = {}) => ({ + user_id: 'U1', team_id: 'T1', channel_id: 'C1', trigger_id: 'trig-1', text: 'escalate', ...over, + }); + + it('acks and delegates to postEscalation with source slash_escalate', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(ack).toHaveBeenCalledTimes(1); + expect(mockPostEscalation).toHaveBeenCalledWith( + expect.objectContaining({ source: 'slash_escalate', userId: 'U1', channelId: 'C1' }), + ); + }); + + it('sends the channel confirmation on success in a channel', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('escalated to #escalation') }), + ); + }); + + it('sends the DM confirmation and marks isDm when invoked in a DM', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ + command: cmd({ channel_id: 'D9', channel_name: 'directmessage' }), + ack, respond: mockRespond, client: mockClient, logger: mockLogger, + }); + expect(mockPostEscalation).toHaveBeenCalledWith(expect.objectContaining({ isDm: true })); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: '✅ A team member will follow up shortly.' }), + ); + }); + + it('sends an ephemeral error when postEscalation fails', async () => { + mockPostEscalation.mockResolvedValue({ ok: false, errorType: 'post_failed' }); + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('could not escalate') }), + ); + }); + + it('does not call postEscalation and warns the user when rate limited', async () => { + // Exhaust the limiter for this user (default RATE_LIMIT_MAX_REQUESTS=20). + const { checkRateLimit } = await import('../../../src/agent/rate-limiter.js'); + for (let i = 0; i < 25; i++) checkRateLimit('U_RL'); + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ + command: cmd({ user_id: 'U_RL' }), ack, respond: mockRespond, client: mockClient, logger: mockLogger, + }); + expect(mockPostEscalation).not.toHaveBeenCalled(); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('request limit') }), + ); + }); +}); +``` + +> The rate-limit test imports the **real** `rate-limiter.js` (it is not mocked) and exhausts the window for a dedicated user id so it does not affect other tests. + +- [ ] **Step 5: Run the escalate tests to verify pass** + +Run: `cd apps/fiona-slack && npm test -- listeners/commands/fiona` +Expected: PASS (existing + 5 new). + +- [ ] **Step 6: Lint and commit** + +```bash +cd apps/fiona-slack && npm run lint +git add apps/fiona-slack/src/listeners/commands/fiona.js apps/fiona-slack/src/agent/interaction-store.js apps/fiona-slack/tests/listeners/commands/fiona.test.js +git commit -m "$(cat <<'EOF' +feat(AI-122): add /fiona escalate sub-command + +Routes /fiona escalate through rate limiting and DM detection, delegates +the channel post to postEscalation, and returns the AC-mandated ephemeral +confirmations and error message. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +--- + +## Task 5: Configuration & docs + +No new behavior — wires up env vars, the slash usage hint, deployment config, and docs. Verified by lint + the full test suite + JSON validity. + +**Files:** +- Modify: `apps/fiona-slack/.env.sample` +- Modify: `apps/fiona-slack/manifest.json` +- Modify: `infra/fiona-slack-container/main.bicep` +- Modify: `docs/fiona-skills-prd.md:271-276` + +- [ ] **Step 1: Add env vars to `.env.sample`** + +Append to `apps/fiona-slack/.env.sample`: + +```bash + +# Escalation (/fiona escalate). ESCALATION_CHANNEL is the destination channel ID +# (e.g. C0123456789) — the bot must be a member of that channel to post. +# ESCALATION_CHANNEL=C0123456789 +# Optional: a Slack user group ID (e.g. S0123456789) to @-mention on escalation. +# ESCALATION_USERGROUP_ID=S0123456789 +``` + +- [ ] **Step 2: Update the slash command usage hint** + +In `apps/fiona-slack/manifest.json`, change the `/fiona` `usage_hint`: + +```json + "usage_hint": "[help | ask | search | escalate]", +``` + +- [ ] **Step 3: Wire env vars into the container Bicep** + +In `infra/fiona-slack-container/main.bicep`, add two params after the `captureAllConversations` param (line ~80): + +```bicep +@description('Slack channel ID where /fiona escalate posts (bot must be a member)') +param escalationChannel string = '' + +@description('Optional Slack user group ID to @-mention on escalation') +param escalationUsergroupId string = '' +``` + +Then add two entries to the container `env` array (after the `COSMOS_INTERACTIONS_CONTAINER` entry, ~line 388): + +```bicep + { + name: 'ESCALATION_CHANNEL' + value: escalationChannel + } + { + name: 'ESCALATION_USERGROUP_ID' + value: escalationUsergroupId + } +``` + +- [ ] **Step 4: Update the PRD env table** + +In `docs/fiona-skills-prd.md`, replace the §5 Environment Variables table with: + +```markdown +| Variable | Default | Purpose | +| ------------------------- | ------- | ---------------------------------------------------------------- | +| `ESCALATION_CHANNEL` | (unset) | Channel **ID** where escalation posts are created (bot must join) | +| `ESCALATION_USERGROUP_ID` | (unset) | Optional Slack user group ID to @-mention on escalation | +``` + +- [ ] **Step 5: Verify and commit** + +```bash +cd apps/fiona-slack && npm run lint && npm test +node -e "JSON.parse(require('fs').readFileSync('manifest.json','utf8')); console.log('manifest OK')" +git add apps/fiona-slack/.env.sample apps/fiona-slack/manifest.json infra/fiona-slack-container/main.bicep docs/fiona-skills-prd.md +git commit -m "$(cat <<'EOF' +chore(AI-122): wire escalation config and docs + +Adds ESCALATION_CHANNEL / ESCALATION_USERGROUP_ID to .env.sample and the +container Bicep, updates the /fiona usage hint, and documents the vars. + +Co-Authored-By: Claude Opus 4.8 (1M context) +EOF +)" +``` + +Expected: lint clean, full suite PASS, `manifest OK`. + +--- + +## Self-Review + +**1. Spec coverage (AI-122 ACs → tasks):** + +| AC | Covered by | +| --- | --- | +| Post with display name + thread link + summary of recent history | Task 3 (post + transcript + summary), Task 1 (summary) | +| Ephemeral channel confirmation (exact copy) | Task 4 (`ESCALATE_CONFIRM_TEXT`) | +| DM ephemeral (exact copy) | Task 4 (`isDmChannel` → `ESCALATE_DM_TEXT`) | +| Channel-not-found/no-permission → ephemeral error + log | Task 3 (`post_failed`, `error` log) + Task 4 (`ESCALATE_ERROR_TEXT`) | +| `ESCALATION_CHANNEL` env var | Tasks 3, 5 | +| Rate limiting | Task 4 (`checkRateLimit`) | +| Record `slash_escalate` to both containers | Task 3 (interactions + feedback), Task 2 (feedback `interactionType`) | +| Shared `postEscalation` helper | Task 3 | +| Unit tests (name+link, confirmation, DM, permission error, both containers, env respected) | Tasks 3 & 4 test suites | + +No gaps. + +**2. Placeholder scan:** No `TBD`/`TODO`/"handle edge cases"/"similar to Task N"; every code step shows full code. ✅ + +**3. Type consistency:** `postEscalation(...) => { ok, errorType }` is produced in Task 3 and consumed identically in Task 4. `summarizeForEscalation(transcriptText, logger) => string|null` produced in Task 1, called in Task 3. `recordFeedback({ ..., interactionType })` extended in Task 2, called in Task 3. `source: 'slash_escalate'` consistent across Tasks 3–4. ✅ + +**4. Known assumptions to verify during execution:** +- Bolt provides `respond` and `client` to slash-command listeners (used in Task 4). +- The bot has `im:history` (DM transcript) and `channels:history`/`groups:history` (channel transcript) — already present in `manifest.json`. +- A user-group mention `` needs no extra scope (it is message text). + diff --git a/docs/superpowers/plans/2026-06-23-escalation-redesign.md b/docs/superpowers/plans/2026-06-23-escalation-redesign.md new file mode 100644 index 00000000..df12ba22 --- /dev/null +++ b/docs/superpowers/plans/2026-06-23-escalation-redesign.md @@ -0,0 +1,1522 @@ +# Escalation Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Re-scope `/fiona escalate` so conversation context comes only from the user's Fiona thread, add an always-present "Get Live Help" thread button and an async LLM-suggested escalation, all feeding one shared `postEscalation` helper. + +**Architecture:** Three entry points — slash command (no transcript), an always-present thread button, and a proactive LLM-suggested button — call a single `postEscalation` helper. Transcripts are built only from a real Slack thread via the existing `buildThreadHistory()`, never from channel history. + +**Tech Stack:** Node.js ESM, Slack Bolt 4, Jest 29 (`jest.unstable_mockModule`), Perplexity via the OpenAI SDK, Azure Cosmos. + +## Global Constraints + +- Every new `.js` file starts with the Apache-2.0 license header (4 comment lines) used across the repo. +- ESM only (`"type": "module"`); import paths end in `.js`. +- Tests run from `apps/fiona-slack/` with `npm test -- ` (wrapper: `node --experimental-vm-modules`). +- Mock ESM dependencies with `jest.unstable_mockModule(...)` **before** `await import(...)` of the module under test. +- User-facing escalation label/CTA is exactly **"Get Live Help"**. +- Classifier signal first line is exactly `Get Live Help: yes` or `Get Live Help: no` (parsed case-insensitively; non-`yes` ⇒ no suggestion). +- Proactive suggestions are gated solely on `ESCALATION_CHANNEL` being set (no separate flag). +- `ESCALATION_CHANNEL` is a Slack channel **ID**; there is no name default. +- Escalation record values come from `escalation-constants.js`, never inline literals. + +--- + +## File Structure + +**New:** +- `src/agent/escalation-constants.js` — `ESCALATION_SOURCES`, `FEEDBACK_VALUE_ESCALATION`. +- `src/listeners/views/escalate_block.js` — `escalateButtonBlock`, `buildSuggestionBlocks(reason)`. +- `src/listeners/actions/escalate.js` — `escalateActionCallback`. +- Tests: `tests/agent/llm-caller.classify.test.js`, `tests/listeners/views/escalate-block.test.js`, `tests/listeners/actions/escalate.test.js`. + +**Modified:** +- `src/agent/rate-limiter.js` — add `rateLimitMessage(retryAfterMs)`. +- `src/agent/rate-limited-handler.js` — use `rateLimitMessage`. +- `src/agent/thread-history.js` — add `neutralizeBroadcasts`, `renderTranscript`. +- `src/agent/llm-caller.js` — add `classifyForEscalation`. +- `src/agent/escalation.js` — refactor `postEscalation`; add `isEscalationConfigured`, `maybeSuggestEscalation`. +- `src/listeners/commands/fiona.js` — help text, shared rate-limit message, confirm wording. +- `src/listeners/actions/index.js` — register `escalate_conversation`. +- `src/listeners/events/app_mention.js`, `src/listeners/assistant/message.js` — attach button + fire-and-forget suggestion. +- `src/agent/interaction-store.js` — JSDoc interactionType list. +- Existing tests: `tests/agent/escalation.test.js`, `tests/listeners/commands/fiona.test.js`. + +--- + +### Task 1: Shared `rateLimitMessage` helper + +**Files:** +- Modify: `src/agent/rate-limiter.js` +- Modify: `src/agent/rate-limited-handler.js:55-58` +- Test: `tests/agent/rate-limiter.test.js` + +**Interfaces:** +- Produces: `rateLimitMessage(retryAfterMs: number) => string` + +- [ ] **Step 1: Write the failing test** — append to `tests/agent/rate-limiter.test.js`: + +```javascript +import { rateLimitMessage } from '../../src/agent/rate-limiter.js'; + +describe('rateLimitMessage', () => { + it('formats a single minute without pluralizing', () => { + expect(rateLimitMessage(60000)).toBe( + ":no_entry: You've reached the request limit. Please wait 1 minute before trying again.", + ); + }); + + it('pluralizes and rounds up partial minutes', () => { + expect(rateLimitMessage(90000)).toContain('2 minutes'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/agent/rate-limiter.test.js` +Expected: FAIL — `rateLimitMessage is not a function`. + +- [ ] **Step 3: Add the helper** — append to `src/agent/rate-limiter.js`: + +```javascript +/** + * Build the standard user-facing rate-limit message. + * + * @param {number} retryAfterMs - Milliseconds until the user may retry. + * @returns {string} + */ +export function rateLimitMessage(retryAfterMs) { + const minutes = Math.ceil(retryAfterMs / 60000); + return `:no_entry: You've reached the request limit. Please wait ${minutes} minute${minutes !== 1 ? 's' : ''} before trying again.`; +} +``` + +- [ ] **Step 4: Refactor `rate-limited-handler.js`** to reuse it. Change the import on line 7 and the message block at lines 55-58: + +```javascript +import { checkRateLimit, rateLimitMessage } from './rate-limiter.js'; +``` + +Replace lines 55-58 (the `const minutes = ...` and `await say(...)`) with: + +```javascript + await say(rateLimitMessage(retryAfterMs)); +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `npm test -- tests/agent/rate-limiter.test.js tests/listeners/events/app-mention.test.js` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/agent/rate-limiter.js src/agent/rate-limited-handler.js tests/agent/rate-limiter.test.js +git commit -m "refactor: extract shared rateLimitMessage helper [AI-122]" +``` + +--- + +### Task 2: Transcript rendering in `thread-history.js` + +**Files:** +- Modify: `src/agent/thread-history.js` +- Test: `tests/agent/thread-history.test.js` + +**Interfaces:** +- Consumes: `buildThreadHistory` output shape `Array<{role:'user'|'assistant', content:string}>`. +- Produces: + - `neutralizeBroadcasts(text: string) => string` + - `renderTranscript(messages: Array<{role,content}>) => string` — newline-joined `*User:*`/`*Fiona:*` lines, broadcasts neutralized; `''` for empty input. + +- [ ] **Step 1: Write the failing test** — append to `tests/agent/thread-history.test.js`: + +```javascript +import { neutralizeBroadcasts, renderTranscript } from '../../src/agent/thread-history.js'; + +describe('neutralizeBroadcasts', () => { + it('defuses channel/here broadcasts and subteam pings', () => { + expect(neutralizeBroadcasts('hey and ')).toBe('hey channel and oncall'); + }); + it('passes through plain text unchanged', () => { + expect(neutralizeBroadcasts('just text')).toBe('just text'); + }); +}); + +describe('renderTranscript', () => { + it('labels roles and joins lines', () => { + const out = renderTranscript([ + { role: 'user', content: 'I need ODS help' }, + { role: 'assistant', content: 'Here is info' }, + ]); + expect(out).toBe('*User:* I need ODS help\n*Fiona:* Here is info'); + }); + it('neutralizes broadcasts inside content', () => { + expect(renderTranscript([{ role: 'user', content: 'ping ' }])).toBe('*User:* ping here'); + }); + it('returns empty string for empty input', () => { + expect(renderTranscript([])).toBe(''); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/agent/thread-history.test.js` +Expected: FAIL — exports not defined. + +- [ ] **Step 3: Add the helpers** — append to `src/agent/thread-history.js`: + +```javascript +/** + * Defuse Slack broadcast/subteam tokens so transcript content cannot mass-ping + * or spoof when re-posted into the escalation channel. `` -> `channel`, + * `` -> `oncall`. + * + * @param {string} text + * @returns {string} + */ +export function neutralizeBroadcasts(text) { + return (text ?? '').replace(/|]+)(?:\|([^>]+))?>/g, (_match, trigger, label) => label || trigger); +} + +/** + * Render a buildThreadHistory message array into a plain-text transcript. + * + * @param {Array<{role: string, content: string}>} messages + * @returns {string} Newline-joined "*Who:* text" lines, or '' when empty. + */ +export function renderTranscript(messages) { + return (messages ?? []) + .map((m) => { + const who = m.role === 'assistant' ? 'Fiona' : 'User'; + const text = neutralizeBroadcasts(m.content).trim(); + return text ? `*${who}:* ${text}` : null; + }) + .filter(Boolean) + .join('\n'); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npm test -- tests/agent/thread-history.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/thread-history.js tests/agent/thread-history.test.js +git commit -m "feat: add transcript renderer and broadcast neutralizer [AI-122]" +``` + +--- + +### Task 3: Shared escalation constants + +**Files:** +- Create: `src/agent/escalation-constants.js` + +**Interfaces:** +- Produces: + - `ESCALATION_SOURCES = { SLASH: 'slash_escalate', BUTTON: 'button_escalate', SUGGESTED: 'suggested_escalate' }` + - `FEEDBACK_VALUE_ESCALATION = 'escalation'` + +- [ ] **Step 1: Create the module** (no dedicated test — covered by consumers): + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +/** + * Canonical escalation interaction-type sources, recorded to Cosmos. + * @readonly + */ +export const ESCALATION_SOURCES = { + SLASH: 'slash_escalate', + BUTTON: 'button_escalate', + SUGGESTED: 'suggested_escalate', +}; + +/** Feedback-container `value` written for escalation rows. */ +export const FEEDBACK_VALUE_ESCALATION = 'escalation'; +``` + +- [ ] **Step 2: Commit** + +```bash +git add src/agent/escalation-constants.js +git commit -m "feat: add shared escalation constants [AI-122]" +``` + +--- + +### Task 4: Escalation block builders + +**Files:** +- Create: `src/listeners/views/escalate_block.js` +- Test: `tests/listeners/views/escalate-block.test.js` + +**Interfaces:** +- Consumes: `ESCALATION_SOURCES` from `src/agent/escalation-constants.js`. +- Produces: + - `escalateButtonBlock` — an `actions` block; single button, `action_id: 'escalate_conversation'`, `value: 'button_escalate'`. + - `buildSuggestionBlocks(reason?: string) => Array` — section (explanatory text incl. reason) + actions block; button `action_id: 'escalate_conversation'`, `value: 'suggested_escalate'`. + +- [ ] **Step 1: Write the failing test** — create `tests/listeners/views/escalate-block.test.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect } from '@jest/globals'; +import { escalateButtonBlock, buildSuggestionBlocks } from '../../../src/listeners/views/escalate_block.js'; + +describe('escalateButtonBlock', () => { + it('is an actions block with a Get Live Help button carrying the button source', () => { + expect(escalateButtonBlock.type).toBe('actions'); + const btn = escalateButtonBlock.elements[0]; + expect(btn.action_id).toBe('escalate_conversation'); + expect(btn.text.text).toBe('Get Live Help'); + expect(btn.value).toBe('button_escalate'); + }); +}); + +describe('buildSuggestionBlocks', () => { + it('includes the reason and a suggested-source button', () => { + const blocks = buildSuggestionBlocks('You seem stuck on ODS setup.'); + const json = JSON.stringify(blocks); + expect(json).toContain('You seem stuck on ODS setup.'); + expect(json).toContain('Get Live Help'); + const actions = blocks.find((b) => b.type === 'actions'); + expect(actions.elements[0].value).toBe('suggested_escalate'); + expect(actions.elements[0].action_id).toBe('escalate_conversation'); + }); + + it('omits a reason sentence when none is given but still renders the button', () => { + const blocks = buildSuggestionBlocks(null); + expect(blocks.find((b) => b.type === 'actions').elements[0].value).toBe('suggested_escalate'); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/listeners/views/escalate-block.test.js` +Expected: FAIL — module not found. + +- [ ] **Step 3: Create the module** `src/listeners/views/escalate_block.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { ESCALATION_SOURCES } from '../../agent/escalation-constants.js'; + +const ESCALATE_ACTION_ID = 'escalate_conversation'; + +/** + * Always-present "Get Live Help" button attached to every Fiona response. + * @type {import("@slack/types").ActionsBlock} + */ +export const escalateButtonBlock = { + type: 'actions', + elements: [ + { + type: 'button', + action_id: ESCALATE_ACTION_ID, + text: { type: 'plain_text', text: 'Get Live Help', emoji: true }, + value: ESCALATION_SOURCES.BUTTON, + }, + ], +}; + +/** + * Build the proactive suggestion message: explanatory text plus a prominent + * "Get Live Help" button that records `suggested_escalate` when clicked. + * + * @param {string|null} [reason] - Short classifier reason, shown to the user. + * @returns {Array} Slack blocks. + */ +export function buildSuggestionBlocks(reason) { + const lead = reason + ? `It looks like I may not have fully solved this — ${reason}` + : 'It looks like I may not have fully solved this.'; + return [ + { type: 'section', text: { type: 'mrkdwn', text: `${lead}\nWant me to connect you with a human?` } }, + { + type: 'actions', + elements: [ + { + type: 'button', + action_id: ESCALATE_ACTION_ID, + text: { type: 'plain_text', text: 'Get Live Help', emoji: true }, + value: ESCALATION_SOURCES.SUGGESTED, + style: 'primary', + }, + ], + }, + ]; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npm test -- tests/listeners/views/escalate-block.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/listeners/views/escalate_block.js tests/listeners/views/escalate-block.test.js +git commit -m "feat: add Get Live Help button and suggestion blocks [AI-122]" +``` + +--- + +### Task 5: `classifyForEscalation` in `llm-caller.js` + +**Files:** +- Modify: `src/agent/llm-caller.js` (append after `summarizeForEscalation`) +- Test: `tests/agent/llm-caller.classify.test.js` + +**Interfaces:** +- Consumes: module-level `perplexityClient`, `PERPLEXITY_API_MODEL`. +- Produces: `classifyForEscalation(transcriptText: string, logger?) => Promise<{ suggest: boolean, reason: string|null }>` + +- [ ] **Step 1: Write the failing test** — create `tests/agent/llm-caller.classify.test.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockCreate = jest.fn(); +jest.unstable_mockModule('openai', () => ({ + OpenAI: jest.fn().mockImplementation(() => ({ + chat: { completions: { create: mockCreate } }, + })), +})); + +process.env.PERPLEXITY_API_KEY = 'test-key'; +const { classifyForEscalation } = await import('../../src/agent/llm-caller.js'); + +describe('classifyForEscalation', () => { + beforeEach(() => jest.clearAllMocks()); + + it('returns suggest:true and the reason when the first line says yes', async () => { + mockCreate.mockResolvedValue({ + choices: [{ message: { content: 'Get Live Help: yes\nUser is blocked on ODS install.' } }], + }); + const result = await classifyForEscalation('*User:* help'); + expect(result).toEqual({ suggest: true, reason: 'User is blocked on ODS install.' }); + }); + + it('returns suggest:false when the first line says no', async () => { + mockCreate.mockResolvedValue({ choices: [{ message: { content: 'Get Live Help: no' } }] }); + expect(await classifyForEscalation('*User:* thanks!')).toEqual({ suggest: false, reason: null }); + }); + + it('treats anything that is not an explicit yes as no (case-insensitive)', async () => { + mockCreate.mockResolvedValue({ choices: [{ message: { content: 'maybe later' } }] }); + expect((await classifyForEscalation('*User:* hmm')).suggest).toBe(false); + }); + + it('returns false for empty transcript without calling the LLM', async () => { + const result = await classifyForEscalation(' '); + expect(result).toEqual({ suggest: false, reason: null }); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it('returns false and warns when the LLM throws', async () => { + mockCreate.mockRejectedValue(new Error('boom')); + const logger = { warn: jest.fn() }; + expect(await classifyForEscalation('*User:* hi', logger)).toEqual({ suggest: false, reason: null }); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('escalation classification')); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/agent/llm-caller.classify.test.js` +Expected: FAIL — `classifyForEscalation is not a function`. + +- [ ] **Step 3: Append the helper** to `src/agent/llm-caller.js`: + +```javascript +// ─── Escalation Suggestion Classifier ─────────────────────────────────────── +const ESCALATION_CLASSIFIER_SYSTEM_PROMPT = + 'You decide whether a user in a Slack conversation with Fiona (an Ed-Fi AI assistant) ' + + 'should be offered live help from a human. Consider whether the user appears stuck, ' + + 'unresolved, repeating themselves, or frustrated. ' + + 'Respond with a first line that is EXACTLY "Get Live Help: yes" or "Get Live Help: no". ' + + 'If yes, add a second line with a brief (max 20 words) reason. No other text.'; + +/** + * Classify whether Fiona should proactively offer live help. Non-streaming. + * Returns { suggest:false } when the LLM is unconfigured, the transcript is + * empty, the call fails, or the response is not an explicit yes. + * + * @param {string} transcriptText + * @param {{ warn?: (msg: string) => void }} [logger] + * @returns {Promise<{ suggest: boolean, reason: string|null }>} + */ +export async function classifyForEscalation(transcriptText, logger) { + if (!perplexityClient) return { suggest: false, reason: null }; + if (!transcriptText || !transcriptText.trim()) return { suggest: false, reason: null }; + + try { + const response = await perplexityClient.chat.completions.create({ + model: PERPLEXITY_API_MODEL, + messages: [ + { role: 'system', content: ESCALATION_CLASSIFIER_SYSTEM_PROMPT }, + { role: 'user', content: transcriptText }, + ], + stream: false, + }); + const content = response?.choices?.[0]?.message?.content; + if (typeof content !== 'string') return { suggest: false, reason: null }; + + const lines = content.split('\n').map((l) => l.trim()).filter(Boolean); + const firstLine = lines[0] ?? ''; + const match = firstLine.match(/get live help:\s*(yes|no)/i); + const suggest = Boolean(match) && match[1].toLowerCase() === 'yes'; + const reason = suggest && lines[1] ? lines[1] : null; + return { suggest, reason }; + } catch (error) { + logger?.warn?.(`Failed to generate escalation classification: ${error.message}`); + return { suggest: false, reason: null }; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npm test -- tests/agent/llm-caller.classify.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/llm-caller.js tests/agent/llm-caller.classify.test.js +git commit -m "feat: add classifyForEscalation LLM self-signal [AI-122]" +``` + +--- + +### Task 6: Refactor `postEscalation` + add `isEscalationConfigured` + +**Files:** +- Modify (rewrite): `src/agent/escalation.js` +- Test (rewrite): `tests/agent/escalation.test.js` + +**Interfaces:** +- Consumes: `buildThreadHistory`, `renderTranscript`, `neutralizeBroadcasts` (`thread-history.js`); `summarizeForEscalation` (`llm-caller.js`); `getUser` (`slack-users-store.js`); `recordInteraction`, `recordFeedback`; `ESCALATION_SOURCES`, `FEEDBACK_VALUE_ESCALATION`. +- Produces: + - `isEscalationConfigured() => boolean` + - `postEscalation({ client, userId, teamId, channelId, threadTs=null, messageTs, source, isDm=false, logger }) => Promise<{ ok, errorType }>` — transcript built only when `threadTs` matches `^\d+\.\d+$`; permalink only for real ts + non-DM; slash path posts a nudge line, no transcript. + +- [ ] **Step 1: Rewrite the test** — replace the contents of `tests/agent/escalation.test.js` with: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach, afterEach } from '@jest/globals'; + +const mockGetUser = jest.fn(); +const mockRecordInteraction = jest.fn().mockResolvedValue(undefined); +const mockRecordFeedback = jest.fn().mockResolvedValue(undefined); +const mockSummarize = jest.fn(); +const mockBuildThreadHistory = jest.fn(); + +jest.unstable_mockModule('../../src/agent/slack-users-store.js', () => ({ getUser: mockGetUser })); +jest.unstable_mockModule('../../src/agent/interaction-store.js', () => ({ recordInteraction: mockRecordInteraction })); +jest.unstable_mockModule('../../src/agent/feedback-store.js', () => ({ recordFeedback: mockRecordFeedback })); +jest.unstable_mockModule('../../src/agent/llm-caller.js', () => ({ + summarizeForEscalation: mockSummarize, + classifyForEscalation: jest.fn(), +})); +jest.unstable_mockModule('../../src/agent/thread-history.js', () => ({ + buildThreadHistory: mockBuildThreadHistory, + renderTranscript: (msgs) => (msgs ?? []).map((m) => `*${m.role === 'assistant' ? 'Fiona' : 'User'}:* ${m.content}`).join('\n'), + neutralizeBroadcasts: (t) => t, +})); + +const { postEscalation, isEscalationConfigured } = await import('../../src/agent/escalation.js'); + +function makeClient() { + return { + chat: { + postMessage: jest.fn().mockResolvedValue({ ts: '111.222' }), + getPermalink: jest.fn().mockResolvedValue({ permalink: 'https://slack.test/p1' }), + }, + }; +} + +const REAL_TS = '1700000000.000100'; + +const threadArgs = () => ({ + client: makeClient(), + userId: 'U1', + teamId: 'T1', + channelId: 'C1', + threadTs: REAL_TS, + messageTs: REAL_TS, + source: 'button_escalate', + logger: { warn: jest.fn(), error: jest.fn() }, +}); + +const slashArgs = () => ({ + client: makeClient(), + userId: 'U1', + teamId: 'T1', + channelId: 'C1', + threadTs: null, + messageTs: 'trigger-abc', + source: 'slash_escalate', + logger: { warn: jest.fn(), error: jest.fn() }, +}); + +describe('escalation', () => { + beforeEach(() => { + jest.clearAllMocks(); + process.env.ESCALATION_CHANNEL = 'C_ESCALATE'; + delete process.env.ESCALATION_USERGROUP_ID; + mockGetUser.mockResolvedValue({ displayName: 'Ada Lovelace' }); + mockSummarize.mockResolvedValue('User is stuck configuring the ODS.'); + mockBuildThreadHistory.mockResolvedValue([ + { role: 'user', content: 'I need help with the ODS' }, + { role: 'assistant', content: 'Here is some info' }, + ]); + }); + afterEach(() => { + delete process.env.ESCALATION_CHANNEL; + delete process.env.ESCALATION_USERGROUP_ID; + }); + + describe('isEscalationConfigured', () => { + it('reflects whether ESCALATION_CHANNEL is set', () => { + expect(isEscalationConfigured()).toBe(true); + delete process.env.ESCALATION_CHANNEL; + expect(isEscalationConfigured()).toBe(false); + }); + }); + + it('returns channel_not_configured when ESCALATION_CHANNEL is unset', async () => { + delete process.env.ESCALATION_CHANNEL; + expect(await postEscalation(threadArgs())).toEqual({ ok: false, errorType: 'channel_not_configured' }); + }); + + describe('thread path (real threadTs)', () => { + it('builds a transcript via buildThreadHistory and posts summary + permalink', async () => { + const args = threadArgs(); + const result = await postEscalation(args); + expect(result.ok).toBe(true); + expect(mockBuildThreadHistory).toHaveBeenCalledWith(args.client, 'C1', REAL_TS, expect.any(Object)); + expect(args.client.chat.getPermalink).toHaveBeenCalledWith({ channel: 'C1', message_ts: REAL_TS }); + const header = JSON.stringify(args.client.chat.postMessage.mock.calls[0][0].blocks); + expect(header).toContain('Ada Lovelace'); + expect(header).toContain('https://slack.test/p1'); + expect(header).toContain('User is stuck configuring the ODS.'); + }); + + it('posts the transcript as a threaded reply to the escalation message', async () => { + const args = threadArgs(); + await postEscalation(args); + const reply = args.client.chat.postMessage.mock.calls.find((c) => c[0].thread_ts === '111.222'); + expect(reply).toBeDefined(); + expect(JSON.stringify(reply[0].blocks)).toContain('I need help with the ODS'); + }); + + it('pings the user group when ESCALATION_USERGROUP_ID is set', async () => { + process.env.ESCALATION_USERGROUP_ID = 'S123'; + const args = threadArgs(); + await postEscalation(args); + expect(JSON.stringify(args.client.chat.postMessage.mock.calls[0][0].blocks)).toContain(''); + }); + + it('still posts (transcript-only) when the summary fails', async () => { + mockSummarize.mockResolvedValue(null); + const args = threadArgs(); + expect((await postEscalation(args)).ok).toBe(true); + expect(JSON.stringify(args.client.chat.postMessage.mock.calls[0][0].blocks)).not.toContain('*Summary:*'); + }); + + it('records to both containers with the given source and escalation value', async () => { + await postEscalation(threadArgs()); + await new Promise((r) => setImmediate(r)); + expect(mockRecordInteraction).toHaveBeenCalledWith( + expect.objectContaining({ interactionType: 'button_escalate', status: 'success' }), + ); + expect(mockRecordFeedback).toHaveBeenCalledWith( + expect.objectContaining({ interactionType: 'button_escalate', value: 'escalation' }), + ); + }); + }); + + describe('slash path (threadTs null)', () => { + it('does not build a transcript, fetch a permalink, or post a summary', async () => { + const args = slashArgs(); + const result = await postEscalation(args); + expect(result.ok).toBe(true); + expect(mockBuildThreadHistory).not.toHaveBeenCalled(); + expect(args.client.chat.getPermalink).not.toHaveBeenCalled(); + const blocks = JSON.stringify(args.client.chat.postMessage.mock.calls[0][0].blocks); + expect(blocks).not.toContain('*Summary:*'); + expect(blocks).toContain('Get Live Help'); + expect(blocks).toContain('<#C1>'); + }); + + it('does not post a transcript reply when there is no transcript', async () => { + const args = slashArgs(); + await postEscalation(args); + expect(args.client.chat.postMessage).toHaveBeenCalledTimes(1); + }); + }); + + it('uses a DM placeholder and skips permalink for DM escalations', async () => { + const args = { ...slashArgs(), isDm: true }; + expect((await postEscalation(args)).ok).toBe(true); + expect(args.client.chat.getPermalink).not.toHaveBeenCalled(); + expect(JSON.stringify(args.client.chat.postMessage.mock.calls[0][0].blocks)).toContain('Direct message'); + }); + + it('returns post_failed when chat.postMessage throws', async () => { + const args = threadArgs(); + args.client.chat.postMessage.mockRejectedValueOnce(new Error('not_in_channel')); + expect(await postEscalation(args)).toEqual({ ok: false, errorType: 'post_failed' }); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/agent/escalation.test.js` +Expected: FAIL — current `postEscalation` calls `conversations.replies`/`history` and `getUser` is real; new exports/behavior absent. + +- [ ] **Step 3: Rewrite `src/agent/escalation.js`** entirely: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { FEEDBACK_VALUE_ESCALATION } from './escalation-constants.js'; +import { recordFeedback } from './feedback-store.js'; +import { recordInteraction } from './interaction-store.js'; +import { summarizeForEscalation } from './llm-caller.js'; +import { getUser } from './slack-users-store.js'; +import { buildThreadHistory, neutralizeBroadcasts, renderTranscript } from './thread-history.js'; + +const SLACK_BLOCK_TEXT_LIMIT = 2900; // headroom under Slack's 3000-char section limit +const SLASH_NUDGE = + '_No conversation context — requested via `/fiona escalate`. ' + + 'For full context, use the *Get Live Help* button in your conversation with Fiona._'; + +/** A real Slack message timestamp looks like "1700000000.000100". */ +const isRealTs = (ts) => typeof ts === 'string' && /^\d+\.\d+$/.test(ts); + +/** @returns {boolean} Whether an escalation destination channel is configured. */ +export function isEscalationConfigured() { + return Boolean(process.env.ESCALATION_CHANNEL); +} + +/** + * Post an escalation to the configured channel and record it. Shared by the + * /fiona escalate slash command, the always-present button, and proactive + * suggestions. A transcript is built only when a real thread timestamp exists. + * + * @param {Object} params + * @param {import("@slack/web-api").WebClient} params.client + * @param {string} params.userId + * @param {string} [params.teamId] + * @param {string} params.channelId + * @param {string|null} [params.threadTs] + * @param {string} params.messageTs + * @param {'slash_escalate'|'button_escalate'|'suggested_escalate'} params.source + * @param {boolean} [params.isDm] + * @param {import("@slack/logger").Logger} [params.logger] + * @returns {Promise<{ ok: boolean, errorType: string|null }>} + */ +export async function postEscalation({ + client, + userId, + teamId, + channelId, + threadTs = null, + messageTs, + source, + isDm = false, + logger, +}) { + const targetChannel = process.env.ESCALATION_CHANNEL; + if (!targetChannel) { + logger?.warn?.('ESCALATION_CHANNEL is not configured; cannot post escalation.'); + return { ok: false, errorType: 'channel_not_configured' }; + } + + const user = await getUser(userId, logger); + const rawName = user?.displayName || user?.realName || user?.name || `<@${userId}>`; + const displayName = neutralizeBroadcasts(rawName); + + // Transcript + summary only when a genuine thread exists. + let transcript = ''; + let summary = null; + if (isRealTs(threadTs)) { + const history = await buildThreadHistory(client, channelId, threadTs, { logger }); + transcript = renderTranscript(history); + summary = transcript ? await summarizeForEscalation(transcript, logger) : null; + } + + // Permalink only when a real thread ts exists in a non-DM context. + let permalink = null; + if (!isDm && isRealTs(threadTs)) { + try { + const res = await client.chat.getPermalink({ channel: channelId, message_ts: threadTs }); + permalink = res?.permalink ?? null; + } catch (err) { + logger?.warn?.(`Failed to get permalink for escalation: ${err.message}`); + } + } + + const usergroupId = process.env.ESCALATION_USERGROUP_ID; + const mention = usergroupId ? ` ` : ''; + let locationLink; + if (permalink) locationLink = `<${permalink}|View conversation>`; + else if (isDm) locationLink = 'Direct message (transcript below)'; + else locationLink = `<#${channelId}>`; + + const unixSeconds = Math.floor(Date.now() / 1000); + const iso = new Date().toISOString(); + const headerLines = [ + `${mention}:rotating_light: *Escalation requested* by *${displayName}*`, + `*Where:* ${locationLink}`, + `*When:* `, + ]; + if (summary) headerLines.push(`*Summary:* ${summary}`); + else if (!transcript) headerLines.push(SLASH_NUDGE); + + let postedTs = null; + try { + const res = await client.chat.postMessage({ + channel: targetChannel, + text: `Escalation requested by ${displayName}`, + blocks: [{ type: 'section', text: { type: 'mrkdwn', text: headerLines.join('\n') } }], + }); + postedTs = res?.ts ?? null; + } catch (err) { + logger?.error?.(`Failed to post escalation to ${targetChannel}: ${err.message}`); + return { ok: false, errorType: 'post_failed' }; + } + + if (transcript && postedTs) { + try { + await client.chat.postMessage({ + channel: targetChannel, + thread_ts: postedTs, + text: 'Conversation transcript', + blocks: [ + { + type: 'section', + text: { type: 'mrkdwn', text: `*Transcript:*\n${transcript}`.slice(0, SLACK_BLOCK_TEXT_LIMIT) }, + }, + ], + }); + } catch (err) { + logger?.warn?.(`Failed to post escalation transcript: ${err.message}`); + } + } + + recordInteraction({ + userId, + teamId, + channelId, + threadTs: threadTs ?? messageTs, + messageTs, + interactionType: source, + status: 'success', + errorType: null, + rateLimited: false, + logger, + }).catch((e) => logger?.warn?.(`Failed to record escalation interaction: ${e.message}`)); + + recordFeedback({ + userId, + channelId, + messageTs, + value: FEEDBACK_VALUE_ESCALATION, + interactionType: source, + reason: null, + userMessage: transcript || null, + botResponse: summary || null, + logger, + }).catch((e) => logger?.warn?.(`Failed to record escalation feedback: ${e.message}`)); + + return { ok: true, errorType: null }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npm test -- tests/agent/escalation.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/escalation.js tests/agent/escalation.test.js +git commit -m "refactor: build escalation transcript only from real threads [AI-122]" +``` + +--- + +### Task 7: `maybeSuggestEscalation` (proactive flow) + +**Files:** +- Modify: `src/agent/escalation.js` (add export + imports) +- Test: `tests/agent/escalation.test.js` (add a describe block) + +**Interfaces:** +- Consumes: `classifyForEscalation` (`llm-caller.js`), `renderTranscript` (`thread-history.js`), `buildSuggestionBlocks` (`escalate_block.js`), `isEscalationConfigured`. +- Produces: `maybeSuggestEscalation({ client, channelId, threadTs, history, botText, logger }) => Promise` — returns true when a suggestion was posted. No-op (false) when not configured, classifier says no, or on error. + +- [ ] **Step 1: Add the failing test** — append a new describe block to `tests/agent/escalation.test.js`. Also extend the `llm-caller.js` mock at the top of that file so `classifyForEscalation` is controllable; replace the existing `llm-caller.js` mock line with: + +```javascript +const mockClassify = jest.fn(); +jest.unstable_mockModule('../../src/agent/llm-caller.js', () => ({ + summarizeForEscalation: mockSummarize, + classifyForEscalation: mockClassify, +})); +``` + +Update the import line to also pull in the new export: + +```javascript +const { postEscalation, isEscalationConfigured, maybeSuggestEscalation } = await import('../../src/agent/escalation.js'); +``` + +Then append: + +```javascript +describe('maybeSuggestEscalation', () => { + const baseArgs = () => ({ + client: makeClient(), + channelId: 'C1', + threadTs: REAL_TS, + history: [{ role: 'user', content: 'still broken' }], + botText: 'Sorry, I am not sure.', + logger: { warn: jest.fn() }, + }); + + beforeEach(() => { + process.env.ESCALATION_CHANNEL = 'C_ESCALATE'; + mockClassify.mockReset(); + }); + + it('posts a suggestion in-thread when the classifier says yes', async () => { + mockClassify.mockResolvedValue({ suggest: true, reason: 'User still blocked.' }); + const args = baseArgs(); + const posted = await maybeSuggestEscalation(args); + expect(posted).toBe(true); + const call = args.client.chat.postMessage.mock.calls[0][0]; + expect(call.channel).toBe('C1'); + expect(call.thread_ts).toBe(REAL_TS); + expect(JSON.stringify(call.blocks)).toContain('Get Live Help'); + expect(JSON.stringify(call.blocks)).toContain('User still blocked.'); + }); + + it('does nothing when the classifier says no', async () => { + mockClassify.mockResolvedValue({ suggest: false, reason: null }); + const args = baseArgs(); + expect(await maybeSuggestEscalation(args)).toBe(false); + expect(args.client.chat.postMessage).not.toHaveBeenCalled(); + }); + + it('does nothing when ESCALATION_CHANNEL is unset (without calling the classifier)', async () => { + delete process.env.ESCALATION_CHANNEL; + const args = baseArgs(); + expect(await maybeSuggestEscalation(args)).toBe(false); + expect(mockClassify).not.toHaveBeenCalled(); + }); + + it('swallows post errors and returns false', async () => { + mockClassify.mockResolvedValue({ suggest: true, reason: null }); + const args = baseArgs(); + args.client.chat.postMessage.mockRejectedValueOnce(new Error('down')); + expect(await maybeSuggestEscalation(args)).toBe(false); + expect(args.logger.warn).toHaveBeenCalled(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/agent/escalation.test.js` +Expected: FAIL — `maybeSuggestEscalation is not a function`. + +- [ ] **Step 3: Implement** — in `src/agent/escalation.js`, add to the imports: + +```javascript +import { classifyForEscalation, summarizeForEscalation } from './llm-caller.js'; +import { buildSuggestionBlocks } from '../listeners/views/escalate_block.js'; +``` + +(Replace the existing single `summarizeForEscalation` import with the combined line above.) Then append the function: + +```javascript +/** + * After a Fiona answer, ask the classifier whether to proactively offer live + * help; if so, post a suggestion (with a Get Live Help button) in the thread. + * Fire-and-forget friendly: never throws, returns whether a suggestion posted. + * + * @param {Object} params + * @param {import("@slack/web-api").WebClient} params.client + * @param {string} params.channelId + * @param {string} params.threadTs + * @param {Array<{role:string, content:string}>} params.history - buildThreadHistory output for the turn. + * @param {string} params.botText - The answer Fiona just sent. + * @param {import("@slack/logger").Logger} [params.logger] + * @returns {Promise} + */ +export async function maybeSuggestEscalation({ client, channelId, threadTs, history, botText, logger }) { + if (!isEscalationConfigured()) return false; + + const transcript = [renderTranscript(history), botText ? `*Fiona:* ${neutralizeBroadcasts(botText)}` : ''] + .filter(Boolean) + .join('\n'); + + const { suggest, reason } = await classifyForEscalation(transcript, logger); + if (!suggest) return false; + + try { + await client.chat.postMessage({ + channel: channelId, + thread_ts: threadTs, + text: 'Want me to connect you with a human?', + blocks: buildSuggestionBlocks(reason), + }); + return true; + } catch (err) { + logger?.warn?.(`Failed to post escalation suggestion: ${err.message}`); + return false; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `npm test -- tests/agent/escalation.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/escalation.js tests/agent/escalation.test.js +git commit -m "feat: add proactive maybeSuggestEscalation flow [AI-122]" +``` + +--- + +### Task 8: Escalate button action handler + registration + +**Files:** +- Create: `src/listeners/actions/escalate.js` +- Modify: `src/listeners/actions/index.js` +- Test: `tests/listeners/actions/escalate.test.js` + +**Interfaces:** +- Consumes: `postEscalation` (`escalation.js`), `checkRateLimit` + `rateLimitMessage` (`rate-limiter.js`), `recordInteraction`, `ESCALATION_SOURCES`. +- Produces: `escalateActionCallback({ ack, body, client, respond, logger })`; registered on `action('escalate_conversation', ...)`. + +- [ ] **Step 1: Write the failing test** — create `tests/listeners/actions/escalate.test.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { describe, it, expect, jest, beforeEach } from '@jest/globals'; + +const mockPostEscalation = jest.fn(); +const mockCheckRateLimit = jest.fn(); +const mockRecordInteraction = jest.fn().mockResolvedValue(undefined); + +jest.unstable_mockModule('../../../src/agent/escalation.js', () => ({ postEscalation: mockPostEscalation })); +jest.unstable_mockModule('../../../src/agent/rate-limiter.js', () => ({ + checkRateLimit: mockCheckRateLimit, + rateLimitMessage: (ms) => `wait ${Math.ceil(ms / 60000)}`, +})); +jest.unstable_mockModule('../../../src/agent/interaction-store.js', () => ({ recordInteraction: mockRecordInteraction })); + +const { escalateActionCallback } = await import('../../../src/listeners/actions/escalate.js'); + +const baseBody = (over = {}) => ({ + type: 'block_actions', + user: { id: 'U1' }, + team: { id: 'T1' }, + channel: { id: 'C1', type: 'channel' }, + message: { ts: '111.000', thread_ts: '100.000' }, + actions: [{ action_id: 'escalate_conversation', value: 'button_escalate' }], + ...over, +}); + +describe('escalateActionCallback', () => { + let ack; + let respond; + let client; + const logger = { warn: jest.fn(), error: jest.fn(), info: jest.fn() }; + + beforeEach(() => { + jest.clearAllMocks(); + ack = jest.fn().mockResolvedValue(undefined); + respond = jest.fn().mockResolvedValue(undefined); + client = {}; + mockCheckRateLimit.mockReturnValue({ allowed: true, retryAfterMs: 0 }); + mockPostEscalation.mockResolvedValue({ ok: true, errorType: null }); + }); + + it('acks and delegates with the real thread_ts and button source', async () => { + await escalateActionCallback({ ack, body: baseBody(), client, respond, logger }); + expect(ack).toHaveBeenCalledTimes(1); + expect(mockPostEscalation).toHaveBeenCalledWith( + expect.objectContaining({ threadTs: '100.000', messageTs: '111.000', source: 'button_escalate', channelId: 'C1' }), + ); + expect(respond).toHaveBeenCalledWith(expect.objectContaining({ response_type: 'ephemeral' })); + }); + + it('passes source suggested_escalate when the button value says so', async () => { + const body = baseBody({ actions: [{ action_id: 'escalate_conversation', value: 'suggested_escalate' }] }); + await escalateActionCallback({ ack, body, client, respond, logger }); + expect(mockPostEscalation).toHaveBeenCalledWith(expect.objectContaining({ source: 'suggested_escalate' })); + }); + + it('marks isDm and skips when in an im channel', async () => { + const body = baseBody({ channel: { id: 'D9', type: 'im' } }); + await escalateActionCallback({ ack, body, client, respond, logger }); + expect(mockPostEscalation).toHaveBeenCalledWith(expect.objectContaining({ isDm: true })); + }); + + it('responds with the rate-limit message and does not escalate when limited', async () => { + mockCheckRateLimit.mockReturnValue({ allowed: false, retryAfterMs: 120000 }); + await escalateActionCallback({ ack, body: baseBody(), client, respond, logger }); + expect(mockPostEscalation).not.toHaveBeenCalled(); + expect(respond).toHaveBeenCalledWith(expect.objectContaining({ text: expect.stringContaining('wait 2') })); + }); + + it('surfaces an error response when postEscalation fails', async () => { + mockPostEscalation.mockResolvedValue({ ok: false, errorType: 'post_failed' }); + await escalateActionCallback({ ack, body: baseBody(), client, respond, logger }); + expect(respond).toHaveBeenCalledWith(expect.objectContaining({ text: expect.stringContaining('could not') })); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `npm test -- tests/listeners/actions/escalate.test.js` +Expected: FAIL — module not found. + +- [ ] **Step 3: Create `src/listeners/actions/escalate.js`:** + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { ESCALATION_SOURCES } from '../../agent/escalation-constants.js'; +import { postEscalation } from '../../agent/escalation.js'; +import { recordInteraction } from '../../agent/interaction-store.js'; +import { checkRateLimit, rateLimitMessage } from '../../agent/rate-limiter.js'; + +const CONFIRM_TEXT = '✅ Your conversation has been escalated. A team member will follow up shortly.'; +const DM_CONFIRM_TEXT = '✅ A team member will follow up shortly.'; +const ERROR_TEXT = + ':warning: Sorry, I could not escalate your conversation right now. Please reach out to the team directly.'; + +/** + * Handles the "Get Live Help" button on Fiona responses and proactive + * suggestions. Escalates the thread the button lives in (real thread_ts). + * + * @param {import("@slack/bolt").SlackActionMiddlewareArgs & { client: import("@slack/web-api").WebClient, logger: import("@slack/logger").Logger }} args + */ +export const escalateActionCallback = async ({ ack, body, client, respond, logger }) => { + await ack(); + + const action = Array.isArray(body.actions) ? body.actions[0] : null; + const source = action?.value === ESCALATION_SOURCES.SUGGESTED ? ESCALATION_SOURCES.SUGGESTED : ESCALATION_SOURCES.BUTTON; + const userId = body.user?.id; + const teamId = body.team?.id; + const channelId = body.channel?.id; + const messageTs = body.message?.ts; + const threadTs = body.message?.thread_ts ?? body.message?.ts; + const isDm = body.channel?.type === 'im' || (channelId || '').startsWith('D'); + + if (!userId || !channelId || !messageTs) { + logger?.warn?.('Missing fields on escalate action; skipping'); + return; + } + + const { allowed, retryAfterMs } = checkRateLimit(userId); + if (!allowed) { + await respond({ response_type: 'ephemeral', replace_original: false, text: rateLimitMessage(retryAfterMs) }); + recordInteraction({ + userId, + teamId, + channelId, + threadTs, + messageTs, + interactionType: source, + status: 'error', + errorType: 'rate_limited', + rateLimited: true, + logger, + }).catch((err) => logger?.warn?.(`Failed to record ${source} interaction: ${err.name}`)); + return; + } + + const result = await postEscalation({ client, userId, teamId, channelId, threadTs, messageTs, source, isDm, logger }); + + if (result.ok) { + await respond({ response_type: 'ephemeral', replace_original: false, text: isDm ? DM_CONFIRM_TEXT : CONFIRM_TEXT }); + } else { + await respond({ response_type: 'ephemeral', replace_original: false, text: ERROR_TEXT }); + recordInteraction({ + userId, + teamId, + channelId, + threadTs, + messageTs, + interactionType: source, + status: 'error', + errorType: result.errorType, + rateLimited: false, + logger, + }).catch((err) => logger?.warn?.(`Failed to record ${source} interaction: ${err.name}`)); + } +}; +``` + +- [ ] **Step 4: Register the action** — edit `src/listeners/actions/index.js`: + +```javascript +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { escalateActionCallback } from './escalate.js'; +import { feedbackActionCallback } from './feedback.js'; + +/** + * @param {import("@slack/bolt").App} app + */ +export const register = (app) => { + app.action('feedback', feedbackActionCallback); + app.action('escalate_conversation', escalateActionCallback); +}; +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `npm test -- tests/listeners/actions/escalate.test.js tests/listeners/index.test.js` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/listeners/actions/escalate.js src/listeners/actions/index.js tests/listeners/actions/escalate.test.js +git commit -m "feat: add Get Live Help button action handler [AI-122]" +``` + +--- + +### Task 9: Slash command polish (`fiona.js`) + +**Files:** +- Modify: `src/listeners/commands/fiona.js` +- Test: `tests/listeners/commands/fiona.test.js` + +**Interfaces:** +- Consumes: `rateLimitMessage` (`rate-limiter.js`), `ESCALATION_SOURCES` (`escalation-constants.js`). +- Produces: unchanged exports; `HELP_TEXT` lists escalate; confirmation no longer hardcodes `#escalation`; rate-limit text shared. + +- [ ] **Step 1: Update the existing escalate tests** in `tests/listeners/commands/fiona.test.js`. Change the success-confirmation assertion so it no longer expects `#escalation`: + +```javascript + it('sends the channel confirmation on success in a channel', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: cmd(), ack, respond: mockRespond, client: mockClient, logger: mockLogger }); + expect(mockRespond).toHaveBeenCalledWith( + expect.objectContaining({ text: expect.stringContaining('escalated') }), + ); + expect(mockRespond.mock.calls[0][0].text).not.toContain('#escalation'); + }); +``` + +Add a help-text discoverability test inside the existing `describe('fionaCommandCallback', ...)`: + +```javascript + it('lists escalate in the help text', async () => { + const ack = jest.fn().mockResolvedValue(undefined); + await fionaCommandCallback({ command: { user_id: 'U1', channel_id: 'C1', trigger_id: 't', text: 'help' }, ack, logger: mockLogger }); + expect(ack).toHaveBeenCalledWith(expect.stringContaining('/fiona escalate')); + }); +``` + +- [ ] **Step 2: Run tests to verify the new expectations fail** + +Run: `npm test -- tests/listeners/commands/fiona.test.js` +Expected: FAIL — help text lacks `escalate`; confirmation still contains `#escalation`. + +- [ ] **Step 3: Edit `src/listeners/commands/fiona.js`:** + +Update the imports (lines 6-8): + +```javascript +import { postEscalation } from '../../agent/escalation.js'; +import { ESCALATION_SOURCES } from '../../agent/escalation-constants.js'; +import { recordInteraction } from '../../agent/interaction-store.js'; +import { checkRateLimit, rateLimitMessage } from '../../agent/rate-limiter.js'; +``` + +Add an escalate line to `HELP_TEXT` (inside the code-fence block, after the search line): + +``` +/fiona escalate Get live help from a human +``` + +Change `ESCALATE_CONFIRM_TEXT` (line 29-30) to drop the channel name: + +```javascript +const ESCALATE_CONFIRM_TEXT = '✅ Your conversation has been escalated. A team member will follow up shortly.'; +``` + +In `handleEscalate`, replace the inline rate-limit message block with the shared helper: + +```javascript + const { allowed, retryAfterMs } = checkRateLimit(command.user_id); + if (!allowed) { + await respond({ response_type: 'ephemeral', text: rateLimitMessage(retryAfterMs) }); + recordInteraction({ + ...slashInteractionRecord(command, ESCALATION_SOURCES.SLASH), + status: 'error', + errorType: 'rate_limited', + rateLimited: true, + logger, + }).catch((err) => logger?.warn?.(`Failed to record slash_escalate interaction: ${err.name}`)); + return; + } +``` + +Replace the two remaining literal `'slash_escalate'` occurrences (the `postEscalation({ ... source: 'slash_escalate' ... })` call and the error-path `slashInteractionRecord(command, 'slash_escalate')`) with `ESCALATION_SOURCES.SLASH`. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `npm test -- tests/listeners/commands/fiona.test.js` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/listeners/commands/fiona.js tests/listeners/commands/fiona.test.js +git commit -m "feat: list escalate in help, share rate-limit text, drop channel name [AI-122]" +``` + +--- + +### Task 10: Wire the button + proactive suggestion into the response handlers + +**Files:** +- Modify: `src/listeners/events/app_mention.js` +- Modify: `src/listeners/assistant/message.js` +- Test: `tests/listeners/events/app-mention.test.js`, `tests/listeners/assistant/message.test.js` + +**Interfaces:** +- Consumes: `isEscalationConfigured`, `maybeSuggestEscalation` (`escalation.js`); `escalateButtonBlock` (`escalate_block.js`). +- Produces: response `stop()` blocks include the escalate button when configured; a fire-and-forget `maybeSuggestEscalation` runs after each answer. + +- [ ] **Step 1: Add failing tests.** In `tests/listeners/events/app-mention.test.js`, find where the test file mocks agent modules and add a mock for `escalation.js` (or extend the existing one) exposing `isEscalationConfigured` and `maybeSuggestEscalation`: + +```javascript +const mockIsEscalationConfigured = jest.fn().mockReturnValue(true); +const mockMaybeSuggest = jest.fn().mockResolvedValue(false); +jest.unstable_mockModule('../../../src/agent/escalation.js', () => ({ + isEscalationConfigured: mockIsEscalationConfigured, + maybeSuggestEscalation: mockMaybeSuggest, +})); +``` + +Add assertions to an existing successful-response test (after `streamer.stop` is asserted): + +```javascript + // escalate button is attached when escalation is configured + const stopArg = mockStreamer.stop.mock.calls.at(-1)[0]; + expect(JSON.stringify(stopArg.blocks)).toContain('Get Live Help'); + // proactive suggestion runs after the answer + await new Promise((r) => setImmediate(r)); + expect(mockMaybeSuggest).toHaveBeenCalledWith( + expect.objectContaining({ channelId: expect.any(String), threadTs: expect.any(String) }), + ); +``` + +> Mirror the same mock + assertions in `tests/listeners/assistant/message.test.js` against its streamer/stop mock. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npm test -- tests/listeners/events/app-mention.test.js tests/listeners/assistant/message.test.js` +Expected: FAIL — blocks lack the button; `maybeSuggestEscalation` not called. + +- [ ] **Step 3: Edit `src/listeners/events/app_mention.js`.** Add imports near the existing ones: + +```javascript +import { isEscalationConfigured, maybeSuggestEscalation } from '../../agent/escalation.js'; +import { escalateButtonBlock } from '../views/escalate_block.js'; +``` + +Replace the finalize line `await streamer.stop({ blocks: [feedbackBlock] });` with: + +```javascript + const responseBlocks = isEscalationConfigured() ? [feedbackBlock, escalateButtonBlock] : [feedbackBlock]; + await streamer.stop({ blocks: responseBlocks }); +``` + +After the `await captureConversation({ ... })` call, add the fire-and-forget suggestion: + +```javascript + // Proactively offer live help if Fiona judges it did not resolve the ask. + // Fire-and-forget: never blocks or fails the response. + void maybeSuggestEscalation({ + client, + channelId: channel, + threadTs: thread_ts, + history: prompts, + botText, + logger, + }); +``` + +- [ ] **Step 4: Edit `src/listeners/assistant/message.js`** identically: add the same two imports; in the **text-response branch** (the `else` branch, around line 234) replace `await streamer.stop({ blocks: [feedbackBlock] });` with the same `responseBlocks` construction; after its `captureConversation` call add the same `void maybeSuggestEscalation({ client, channelId: channel, threadTs: thread_ts, history: prompts, botText, logger });` block. (Leave the demo `'Wonder a few deep thoughts.'` branch unchanged.) + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `npm test -- tests/listeners/events/app-mention.test.js tests/listeners/assistant/message.test.js` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add src/listeners/events/app_mention.js src/listeners/assistant/message.js tests/listeners/events/app-mention.test.js tests/listeners/assistant/message.test.js +git commit -m "feat: attach Get Live Help button and run proactive suggestion [AI-122]" +``` + +--- + +### Task 11: Docs/JSDoc cleanup and cross-app verification + +**Files:** +- Modify: `src/agent/interaction-store.js:139` +- Modify: `src/agent/feedback-store.js` (JSDoc for `value`) +- Verify: `apps/usage-report-function/lib/cosmos-queries.js:150` + +- [ ] **Step 1: Update `interaction-store.js` JSDoc** (line 139) to the final taxonomy (drop `auto_escalation`, add button/suggested): + +```javascript + * @param {string} interaction.interactionType - 'app_mention', 'assistant_message', 'slash_help', 'slash_ask', 'slash_search', 'slash_escalate', 'button_escalate', 'suggested_escalate', or 'slash_unknown' +``` + +- [ ] **Step 2: Update `feedback-store.js` JSDoc** for the `value` param so future query code knows escalation rows exist. Find the `@param {string} feedback.value` line and change it to: + +```javascript + * @param {string} feedback.value - 'good-feedback', 'bad-feedback', or 'escalation' +``` + +- [ ] **Step 3: Verify the usage-report allow-list still excludes escalation.** Confirm `apps/usage-report-function/lib/cosmos-queries.js:150` reads `f["value"] IN ('good-feedback', 'bad-feedback')` (so `'escalation'` rows are excluded from the feedback response-rate). Add a clarifying comment above that query line: + +```javascript + // Response-rate counts only thumbs feedback; 'escalation' rows are excluded by design. +``` + +> Scope note: the usage-report function is a separate npm package. Rather than couple it to fiona-slack's `escalation-constants.js`, the allow-list keeps its literal with this comment. No behavior change — `'escalation'` is already excluded. + +- [ ] **Step 4: Run the affected suites** + +Run (from `apps/fiona-slack/`): `npm test -- tests/agent/interaction-store.test.js tests/agent/feedback-store-cosmos.test.js` +Run (from `apps/usage-report-function/`): `npm test` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add apps/fiona-slack/src/agent/interaction-store.js apps/fiona-slack/src/agent/feedback-store.js apps/usage-report-function/lib/cosmos-queries.js +git commit -m "docs: align interaction/feedback taxonomy with escalation sources [AI-122]" +``` + +--- + +### Task 12: Full suite + lint gate + +**Files:** none (verification only) + +- [ ] **Step 1: Run the whole fiona-slack suite** + +Run (from `apps/fiona-slack/`): `npm test` +Expected: PASS (all suites). + +- [ ] **Step 2: Lint** + +Run (from `apps/fiona-slack/`): `npm run lint` +Expected: no errors. Fix any with `npm run lint:fix`, then re-run tests. + +- [ ] **Step 3: Confirm no stale references remain** + +Run: `git grep -n "fetchTranscript\|auto_escalation\|conversations.history" apps/fiona-slack/src` +Expected: no matches (the channel-history path and the removed source are gone). + +- [ ] **Step 4: Commit any lint fixes** + +```bash +git add -A +git commit -m "chore: lint and cleanup for escalation redesign [AI-122]" +``` + +--- + +## Self-Review + +**Spec coverage:** +- Shared `postEscalation`, optional transcript, no channel scraping → Tasks 6. +- `buildThreadHistory` reuse + transcript renderer → Tasks 2, 6. +- Slash command (no transcript, nudge, channel link, DM placeholder, drop `#escalation`, rate-limit share, help text) → Tasks 1, 9, 6. +- Always-present "Get Live Help" button → Tasks 4, 10. +- Proactive async classifier + suggestion → Tasks 5, 7, 10. +- Working permalink guard (`^\d+\.\d+$`) → Task 6. +- Shared constants + taxonomy → Tasks 3, 8, 9, 11. +- Injection hardening (broadcast neutralization, display name) → Tasks 2, 6. +- Slack `` timestamp → Task 6. +- Config gated solely on `ESCALATION_CHANNEL` → Tasks 6, 7, 10. +- Usage-report exclusion preserved → Task 11. +- Process-artifact docs from PR #63 (move out): handled separately at PR cleanup — noted below. + +**Placeholder scan:** No TBD/TODO; every code step shows full code. + +**Type consistency:** `postEscalation` params, `ESCALATION_SOURCES.*` values, `classifyForEscalation` return shape, and `maybeSuggestEscalation` params are consistent across Tasks 5–10. + +**Open follow-up (not a code task):** remove the two large `docs/superpowers/plans/2026-06-22-*.md` PR-#63 artifacts (or move to the PR description) before merge, per the review. diff --git a/docs/superpowers/specs/2026-06-23-escalation-redesign-design.md b/docs/superpowers/specs/2026-06-23-escalation-redesign-design.md new file mode 100644 index 00000000..267485e2 --- /dev/null +++ b/docs/superpowers/specs/2026-06-23-escalation-redesign-design.md @@ -0,0 +1,242 @@ +# Escalation Redesign — escalate a conversation to a human + +- **Date:** 2026-06-23 +- **Jira:** [AI-122](https://edfi.atlassian.net/browse/AI-122) +- **Supersedes:** the escalation approach in PR #63 (`AI-122`) +- **Status:** Design approved; pending spec review + +## Background + +PR #63 implemented `/fiona escalate` as a Slack slash command. Slash commands +carry no `thread_ts`, so the handler always passed `threadTs: null`, and the +transcript builder fell back to scraping the channel's recent **top-level** +messages — not the thread where the user's Fiona conversation lives. That both +failed the core requirement ("summarize the conversation from that thread") and +created a data-egress problem (one user copying other users' channel messages +into the escalation channel and to Perplexity). The green test suite masked the +gap by injecting a synthetic `threadTs` the real slash path never produces. + +This redesign reframes the feature around where the conversation context +actually exists: the **Fiona thread**. + +## Goals + +1. A user in a conversation with Fiona can escalate **that conversation** to a + human, with a transcript and summary of the thread. +2. Fiona can **proactively suggest** escalation when it judges it has not helped. +3. `/fiona escalate` still works outside a conversation, but **without a + transcript** — it is an explicit "I need a human" signal, not a conversation + capture. +4. No channel-history scraping anywhere. Transcripts come only from the user's + own Fiona thread. + +## Non-goals + +- Fully automatic escalation with no human action. Every escalation in this + design is user-initiated (slash command, explicit button, or clicking a + suggested button). The model only *suggests*; it never escalates on its own. +- Group-DM (mpim) special handling beyond safe degradation. +- Routing/assignment logic in the human channel (out of scope; humans triage). + +## Architecture + +Three entry points feed one shared core. The transcript is **optional** and is +built only when a real Slack thread timestamp is available. + +``` +/fiona escalate (slash, no thread) ─┐ +explicit button (Fiona thread) ─┼─► postEscalation() ─► human channel +suggested button (Fiona thread, LLM) ─┘ + Cosmos records +``` + +### Shared core: `postEscalation()` (`src/agent/escalation.js`, refactored) + +Single responsibility: post an escalation to the configured human channel and +record it to Cosmos. Signature (unchanged shape, clarified semantics): + +``` +postEscalation({ + client, userId, teamId, channelId, + threadTs = null, // real Slack ts when escalating a thread; null for slash + messageTs, // real message ts when available; trigger_id for slash + source, // 'slash_escalate' | 'button_escalate' | 'suggested_escalate' + isDm = false, + logger, +}) -> { ok: boolean, errorType: string|null } +``` + +Behavior: + +- If `ESCALATION_CHANNEL` is unset → return `{ ok: false, errorType: 'channel_not_configured' }`. +- **Transcript (only when `threadTs` is a real Slack ts):** build via the shared + thread-history core (see below). When `threadTs` is null, there is no + transcript and no summary. +- **Summary:** `summarizeForEscalation(transcript)` only when a transcript + exists; degrade to transcript-only on failure. +- **Permalink:** attempt `chat.getPermalink` only when the candidate ts matches + `^\d+\.\d+$` and `isDm` is false. Otherwise skip straight to the fallback + location text (no failed API call, no misleading warning). +- **Post** the escalation header block to `ESCALATION_CHANNEL` (optional on-call + user-group ping), then the transcript as a threaded reply when present. +- **Record** to the interactions container and the feedback container + (best-effort, fire-and-forget) using the shared `source`/value constants. + +### Shared thread-history core + +`fetchTranscript` is removed. Transcript text is derived from the same +`buildThreadHistory()` used by `app_mention.js` and `message.js` +(`src/agent/thread-history.js`), so the escalation transcript matches the +context Fiona actually answered from (subtype filtering, mention stripping, +consecutive-turn merging, char-budget truncation). + +`buildThreadHistory` returns an OpenAI-style `[{ role, content }]` array. A +small adapter renders that to the plain-text "`*Who:* text`" transcript used in +the escalation post and passed to `summarizeForEscalation`. The adapter lives in +`thread-history.js` (e.g. `renderTranscript(messages)`) so the message-shaping +logic stays in one module. + +### Entry point 1 — `/fiona escalate` (slash, no transcript) + +`src/listeners/commands/fiona.js`: + +- Ack, validate required fields, rate-limit (per user) using the shared + rate-limit-message helper. +- Call `postEscalation({ threadTs: null, messageTs: trigger_id, + source: 'slash_escalate', isDm })`. +- Human-channel post (no transcript, no permalink): + > :rotating_light: *Escalation requested* by ** from `<#channel>` + > _Requested via `/fiona escalate` — no conversation context. For full + > context, escalate from your conversation with Fiona using the **Get Live + > Help** button._ +- Ephemeral confirmation to the user. Do **not** hardcode "#escalation"; either + resolve the channel's real name via `conversations.info` (cached) or omit the + name ("…has been escalated. A team member will follow up shortly."). +- Add an `/fiona escalate` line to `HELP_TEXT`. + +### Entry point 2 — explicit button (always present, with transcript) + +- A small `actions` block with a single "Get Live Help" button is appended + to the blocks passed to `streamer.stop()` in `app_mention.js` and `message.js`, + alongside the existing `feedbackBlock`. (The feedback buttons live in a + `context_actions` block; the escalate button uses a separate `actions` block + with a `button` element and a dedicated `action_id`, e.g. `escalate_conversation`.) +- New action handler `src/listeners/actions/escalate.js`, registered in + `src/listeners/actions/index.js`. The action body provides `body.message.thread_ts` + (real), `body.channel.id`, `body.user.id`, `body.trigger_id`. +- Handler: ack, rate-limit (reuse `checkRateLimit`), then `postEscalation({ + threadTs: body.message.thread_ts ?? body.message.ts, messageTs: body.message.ts, + source: 'button_escalate', isDm })`. Ephemeral confirmation via `respond`. + +### Entry point 3 — proactive suggestion (async LLM self-signal) + +- New non-streaming helper `classifyForEscalation(history, latestAnswer, logger)` + in `src/agent/llm-caller.js`: + - Returns `{ suggest: boolean, reason: string|null }`. + - Returns `{ suggest: false, reason: null }` when the LLM client is + unconfigured, the call fails, or the response can't be parsed (degrade + quietly). + - System prompt instructs the model to judge whether the user appears stuck, + unresolved, or frustrated, and to answer with a fixed first line — + `Get Live Help: yes` or `Get Live Help: no` — optionally followed by a short + reason line. Parsed defensively (case-insensitive; anything that is not an + explicit `yes` defaults to `no`). +- After `streamer.stop()` and `captureConversation()` in both `app_mention.js` + and `message.js`, run the classifier **fire-and-forget / non-blocking** (does + not delay the answer). Runs on **every** turn, gated only by: + - `ESCALATION_CHANNEL` configured — this is also the single on/off switch for + suggestions (no separate flag); nowhere to escalate otherwise, AND + - the turn did not error / degrade. +- If `suggest`, post a follow-up block in the thread: explanatory text plus a + prominent "Get Live Help" button (same handler, `action_id` variant or + metadata marking `source: 'suggested_escalate'`, carrying the classifier + `reason`). Clicking it runs the same transcript-capturing escalation. +- The suggestion itself is not an escalation. Only a button click records an + escalation. (Optionally record a lightweight "suggestion shown" telemetry + event; not required for v1.) + +## Data / records + +- **Interactions container:** `interactionType` ∈ existing set plus + `slash_escalate`, `button_escalate`, `suggested_escalate`. (Drop the unused + `auto_escalation` type — there is no automatic escalation in this design.) +- **Feedback container:** `value: 'escalation'`, with `interactionType` set to + the source. `userMessage` = transcript (or null for slash), `botResponse` = + summary (or null). +- **Usage report** (`apps/usage-report-function/lib/cosmos-queries.js`): the + feedback response-rate allow-list must continue to exclude escalation rows; + the source/value strings come from a shared constants module rather than + hardcoded literals duplicated across apps. + +## Shared constants + +Introduce a small constants module (e.g. `src/agent/escalation-constants.js`) +exporting the escalation `source` values and the `'escalation'` feedback value, +imported by `escalation.js`, the command/action handlers, and referenced by the +usage-report allow-list, to remove the cross-file magic-string coupling. + +## Configuration + +- `ESCALATION_CHANNEL` (existing) — destination channel **ID**; bot must be a + member. +- `ESCALATION_USERGROUP_ID` (existing, optional) — on-call user group to ping. +- Proactive suggestions have **no separate flag**: they are active whenever + `ESCALATION_CHANNEL` is set and inactive when it is not. +- All read at module-load into consts (no inline `process.env` reads in + functions), consistent with the rest of the codebase. + +## Security / privacy + +- **No new egress.** Transcripts come only from the user's own Fiona thread, + whose content was already sent to Perplexity to generate the answers. The + slash path sends no transcript. The PR-#63 channel-history scraping is removed. +- **Injection hardening:** render the requesting user's display name and the + transcript so that Slack control tokens cannot mass-ping or spoof headers — + prefer `plain_text` for the display name and neutralize ``, + ``, `` and `<@…>`-style tokens in transcript text before + posting. +- **Summarizer prompt injection:** wrap the transcript passed to + `summarizeForEscalation` in clear delimiters and render the returned summary as + non-interpolated text. + +## Error handling & degradation + +- Channel not configured → `{ ok:false, errorType:'channel_not_configured' }`; + handlers show `ESCALATE_ERROR_TEXT`. +- Post failure → `{ ok:false, errorType:'post_failed' }`; user is told and the + error interaction is recorded. +- Transcript fetch / summary / permalink / classifier failures degrade + gracefully (transcript-only, no summary, fallback location link, no + suggestion) and warn; the escalation still succeeds where the channel post + succeeds. +- `hasRequiredFields` failure on the slash path sends `ESCALATE_ERROR_TEXT` + rather than a silent no-op. + +## Testing + +- **`postEscalation`:** real assertions on the posted block content for both the + with-transcript (real `threadTs`) and no-transcript (`threadTs: null`) paths; + permalink attempted only for real ts + non-DM; summary-failure degradation; + post-failure error; recording to both containers; usergroup ping. +- **Transcript rendering:** assert ordering, who-labeling, mention/token + stripping, and truncation against `buildThreadHistory` output. +- **Slash handler:** ack, rate-limit branch (with limiter state cleanup), + delegation with `threadTs: null` and `source: 'slash_escalate'`, + required-fields failure path, DM vs channel confirmation text. +- **Button handler:** delegation with real `thread_ts` and + `source: 'button_escalate'`; rate-limit; confirmation. +- **`classifyForEscalation`:** suggest true/false parsing, unconfigured-client + returns false without calling the LLM, error path returns false and warns. +- **Proactive flow:** suggestion posted when classifier returns true and + `ESCALATION_CHANNEL` is set; not posted when the channel is unset or the turn + errored. Classifier first line parsed case-insensitively; non-`yes` → no + suggestion. +- Tests must use real entry-point arguments (no synthetic `threadTs` on the + slash path) and must not leak shared limiter / `process.env` state. + +## Migration / cleanup from PR #63 + +- Remove `fetchTranscript` and its channel-history branch. +- Remove the `auto_escalation` interaction type. +- Move the two large `docs/superpowers/plans/*.md` process artifacts out of the + PR (PR description or local), per the review. +- Replace hardcoded source/value literals with the shared constants module. diff --git a/infra/fiona-slack-container/main.bicep b/infra/fiona-slack-container/main.bicep index 0f9d1668..a398534d 100644 --- a/infra/fiona-slack-container/main.bicep +++ b/infra/fiona-slack-container/main.bicep @@ -79,6 +79,12 @@ param conversationsContainerName string = 'conversations' @description('Enable capturing all conversations for human evaluation (default: false)') param captureAllConversations bool = false +@description('Slack channel ID where /fiona escalate posts (bot must be a member)') +param escalationChannel string = '' + +@description('Optional Slack user group ID to @-mention on escalation') +param escalationUsergroupId string = '' + // --- Reference shared resources --- resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-07-01' existing = { @@ -386,6 +392,14 @@ resource slackContainerApp 'Microsoft.App/containerApps@2022-03-01' = { name: 'COSMOS_INTERACTIONS_CONTAINER' value: interactionsContainerName } + { + name: 'ESCALATION_CHANNEL' + value: escalationChannel + } + { + name: 'ESCALATION_USERGROUP_ID' + value: escalationUsergroupId + } ] // No probes -- no ingress port for HTTP health checks // Container restart policy handles crash recovery