From ca938f978b56ebaf89ef06f50ede3a42bb01b956 Mon Sep 17 00:00:00 2001 From: Melon80 <2597090588@qq.com> Date: Wed, 13 May 2026 13:20:32 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=8F=AF=E4=BB=A5=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E5=9B=BE=E7=89=87=20=EF=BC=8Cpdf=EF=BC=8Cdoc=EF=BC=8Cdocx?= =?UTF-8?q?=E7=AD=89=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clawpet/app/desktop-pet-bubble/page.tsx | 4 +- .../clawpet/app/desktop-pet/desktop-pet.css | 71 ++++ .../clawpet/app/desktop-pet/page.tsx | 135 +++++++- .../clawpet/electron/file-prompt-preload.js | 6 + clawpet-frontend/clawpet/electron/main.js | 306 +++++++++++++++++- clawpet-frontend/clawpet/electron/preload.js | 67 ++++ clawpet-frontend/clawpet/hooks/use-chat.ts | 50 ++- clawpet-frontend/clawpet/lib/utils.ts | 12 + go.mod | 1 + go.sum | 2 + pkg/agent/loop_mcp.go | 7 +- pkg/channels/pet/channel.go | 1 + pkg/mcp/manager.go | 32 +- pkg/mcp/uv.go | 105 ++++++ pkg/pet/service.go | 303 +++++++++++++++++ pkg/pet/types.go | 12 + pkg/providers/anthropic_messages/provider.go | 71 +++- pkg/providers/factory_provider.go | 48 ++- pkg/providers/openai_compat/provider.go | 58 +++- 19 files changed, 1242 insertions(+), 49 deletions(-) create mode 100644 clawpet-frontend/clawpet/electron/file-prompt-preload.js create mode 100644 pkg/mcp/uv.go diff --git a/clawpet-frontend/clawpet/app/desktop-pet-bubble/page.tsx b/clawpet-frontend/clawpet/app/desktop-pet-bubble/page.tsx index 07826bbe..89452b12 100644 --- a/clawpet-frontend/clawpet/app/desktop-pet-bubble/page.tsx +++ b/clawpet-frontend/clawpet/app/desktop-pet-bubble/page.tsx @@ -1,6 +1,7 @@ "use client" import { useEffect, useMemo, useRef, useState } from "react" +import { extractPetText } from "@/lib/utils" import "./bubble.css" @@ -72,7 +73,8 @@ export default function DesktopPetBubblePage() { useEffect(() => { const handleBubbleShow = (payload: BubbleData) => { - const nextText = typeof payload?.text === "string" ? payload.text.trim() : "" + const rawText = typeof payload?.text === "string" ? payload.text.trim() : "" + const nextText = extractPetText(rawText) if (!nextText) { clearHideTimer() setVisible(false) diff --git a/clawpet-frontend/clawpet/app/desktop-pet/desktop-pet.css b/clawpet-frontend/clawpet/app/desktop-pet/desktop-pet.css index 76936ea5..a5608b0e 100644 --- a/clawpet-frontend/clawpet/app/desktop-pet/desktop-pet.css +++ b/clawpet-frontend/clawpet/app/desktop-pet/desktop-pet.css @@ -113,6 +113,25 @@ pointer-events: auto; width: fit-content; height: fit-content; + outline: 2px solid transparent; + outline-offset: 4px; + border-radius: 12px; +} + +.desktop-pet-area--drag-over { + outline: 3px dashed #667eea; + outline-offset: 12px; + animation: drop-pulse 0.8s ease-in-out infinite alternate; +} + +.desktop-pet-area--drag-over .desktop-pet-image { + filter: brightness(0.85); + transition: filter 0.15s ease; +} + +@keyframes drop-pulse { + from { outline-color: rgba(102, 126, 234, 0.3); } + to { outline-color: rgba(102, 126, 234, 0.9); } } .desktop-pet-area:active { @@ -145,3 +164,55 @@ .desktop-pet-state { display: none; } + +.desktop-pet-btn--active { + background: rgba(102, 126, 234, 0.9); + color: #fff; +} + +.desktop-pet-btn--active:hover { + background: rgba(102, 126, 234, 1); + transform: scale(1.1); +} + +.desktop-pet-dropzone { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 6px; + width: 200px; + min-height: 80px; + border: 2px dashed #667eea; + border-radius: 12px; + background: rgba(102, 126, 234, 0.06); + cursor: pointer; + padding: 12px 16px; + transition: all 0.15s ease; + pointer-events: auto; + -webkit-app-region: no-drag; +} + +.desktop-pet-dropzone:hover { + background: rgba(102, 126, 234, 0.12); + border-color: #5a6fd6; +} + +.desktop-pet-dropzone--drag-over { + background: rgba(102, 126, 234, 0.18); + border-color: #4a5fc6; + border-style: solid; + animation: drop-pulse 0.8s ease-in-out infinite alternate; +} + +.desktop-pet-dropzone-icon { + color: #667eea; + opacity: 0.7; +} + +.desktop-pet-dropzone-text { + font-size: 11px; + color: #667eea; + text-align: center; + line-height: 1.4; +} diff --git a/clawpet-frontend/clawpet/app/desktop-pet/page.tsx b/clawpet-frontend/clawpet/app/desktop-pet/page.tsx index aa8f354e..fadea102 100644 --- a/clawpet-frontend/clawpet/app/desktop-pet/page.tsx +++ b/clawpet-frontend/clawpet/app/desktop-pet/page.tsx @@ -1,7 +1,8 @@ "use client" -import { MessageCircle, X } from "lucide-react" +import { MessageCircle, X, Paperclip } from "lucide-react" import { useCallback, useEffect, useRef, useState, type MouseEvent } from "react" +import { extractPetText } from "@/lib/utils" import "./desktop-pet.css" @@ -212,6 +213,9 @@ export default function DesktopPetPage() { const [runtimeHint, setRuntimeHint] = useState("") const [showControls, setShowControls] = useState(false) const [voicePhase, setVoicePhase] = useState<"idle" | "recording" | "recognizing" | "error">("idle") + const [isDragOver, setIsDragOver] = useState(false) + const [showDropZone, setShowDropZone] = useState(false) + const dropZoneActiveRef = useRef(false) const stackRef = useRef(null) const controlsVisibleRef = useRef(false) const currentImageRef = useRef(currentImage) @@ -369,7 +373,7 @@ export default function DesktopPetPage() { } if (data.text !== null) { - setBubble(data.text || "") + setBubble(extractPetText(data.text || "")) } const resolvedState = resolvePetState(data) @@ -719,6 +723,96 @@ export default function DesktopPetPage() { window.close() } + const allowedExtensions = [ + ".txt", ".md", ".json", ".xml", ".yaml", ".yml", ".csv", + ".js", ".ts", ".jsx", ".tsx", ".py", ".go", ".rs", ".java", ".c", ".cpp", ".h", + ".css", ".scss", ".html", ".htm", ".sh", ".bash", ".zsh", ".ps1", ".bat", ".cmd", + ".sql", ".rb", ".php", ".swift", ".kt", ".scala", ".dart", + ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp", ".svg", ".ico", + ] + + const handleDragEnter = (e: React.DragEvent) => { + e.preventDefault() + setIsDragOver(true) + } + + const handleDragOver = (e: React.DragEvent) => { + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + } + + const handleDragLeave = (e: React.DragEvent) => { + e.preventDefault() + setIsDragOver(false) + } + + const handleDrop = (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(false) + closeDropZone() + const files = e.dataTransfer?.files + if (files && files.length > 0) { + for (let i = 0; i < files.length; i++) { + const f = files[i] + if ((f as any).path) { + window.electronAPI?.sendFileDropped?.((f as any).path, f.name) + } + } + } + transitionTo("think") + } + + const handleDropZoneDrop = (e: React.DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(false) + closeDropZone() + const files = e.dataTransfer?.files + if (files && files.length > 0) { + for (let i = 0; i < files.length; i++) { + const f = files[i] + if ((f as any).path) { + window.electronAPI?.sendFileDropped?.((f as any).path, f.name) + } + } + } + transitionTo("think") + } + + const handleDropZoneClick = () => { + window.electronAPI?.openFileDialog?.() + closeDropZone() + } + + const openDropZone = useCallback(() => { + setShowDropZone(true) + dropZoneActiveRef.current = true + window.electronAPI?.toggleDropZone?.(true) + }, []) + + const closeDropZone = useCallback(() => { + setShowDropZone(false) + setIsDragOver(false) + dropZoneActiveRef.current = false + window.electronAPI?.toggleDropZone?.(false) + }, []) + + const handleFileDialogClick = () => { + window.electronAPI?.openFileDialog?.() + closeDropZone() + } + + // 文件拖放后可开始处理后自动关闭拖放区 + useEffect(() => { + const unlisten = window.electronAPI?.onDropZoneAutoClose?.(() => { + if (dropZoneActiveRef.current) { + closeDropZone() + } + }) + return () => unlisten?.() + }, [closeDropZone]) + const handleOpenSettingsMouseDown = (event: MouseEvent) => { event.preventDefault() event.stopPropagation() @@ -734,7 +828,10 @@ export default function DesktopPetPage() { return (
-
+
+
Pet {petState}
+ {showDropZone && ( +
+ + + + + {isDragOver ? "释放文件" : "拖入文件 / 点击浏览"} + +
+ )} {(bubble || runtimeHint) ? (
ipcRenderer.send('file-prompt-submit', text || ''), + cancelPrompt: () => ipcRenderer.send('file-prompt-cancel'), +}) diff --git a/clawpet-frontend/clawpet/electron/main.js b/clawpet-frontend/clawpet/electron/main.js index 75c25ee3..109b4f07 100644 --- a/clawpet-frontend/clawpet/electron/main.js +++ b/clawpet-frontend/clawpet/electron/main.js @@ -14,7 +14,7 @@ * - 前端面板：通过环境变量 GOCLAW_DASHBOARD_URL 连接（默认 3000） */ -const { app, BrowserWindow, ipcMain, screen, globalShortcut } = require('electron'); +const { app, BrowserWindow, ipcMain, screen, globalShortcut, dialog } = require('electron'); const path = require('path'); const os = require('os'); const fs = require('fs'); @@ -47,6 +47,10 @@ let petTopCorrectionTimer = null; let onboardingLocked = false; let lastBubbleFingerprint = ''; let lastBubbleAt = 0; + +// 文件拖放 +let filePromptWindow = null; +let pendingFileInfo = null; let bubbleWindowReady = false; // 语音输入快捷键 @@ -68,6 +72,8 @@ const PET_HEIGHT = 180; const PET_WINDOW_MARGIN = 16; const PET_RENDERER_RETRY_DELAY_MS = 1200; const PET_RENDERER_MAX_RETRIES = 60; +const PET_EXPANDED_WIDTH = 250; +const PET_EXPANDED_HEIGHT = 290; const BUBBLE_WINDOW_DEFAULT_WIDTH = 320; const BUBBLE_WINDOW_DEFAULT_HEIGHT = 140; const BUBBLE_WINDOW_MIN_WIDTH = 140; @@ -1340,6 +1346,20 @@ function createPetWindow() { attachRendererCrashDiagnostics(petWindow, 'PET WINDOW'); blockCtrlPDefault(petWindow); + // 防止文件拖放时自动导航到 file://，改为触发文件处理 + petWindow.webContents.on('will-navigate', (event, url) => { + if (url && url.startsWith('file://')) { + event.preventDefault(); + let filePath = decodeURIComponent(url.replace(/^file:\/\//, '')); + if (process.platform === 'win32') { + filePath = filePath.replace(/^\//, ''); + } + const fileName = path.basename(filePath); + logToFile(`[pet] will-navigate intercepted file drop: ${fileName}`); + handleFileDropEvent(filePath, fileName); + } + }); + // Re-apply bottom-right placement to avoid OS window policy override. placePetWindowBottomRight(petWindow); createBubbleWindow(); @@ -1884,6 +1904,92 @@ function startStartupFlow() { }, 1000); } +/** + * 创建文件提示输入弹窗 + */ +function createFilePromptWindow(fileName, mimeType, isImage) { + if (filePromptWindow && !filePromptWindow.isDestroyed()) { + filePromptWindow.focus(); + return; + } + + const typeLabel = isImage ? '📷 图片' : '📄 文件'; + + filePromptWindow = new BrowserWindow({ + width: 420, + height: 280, + resizable: false, + frame: true, + title: '发送文件给 AI', + webPreferences: { + preload: path.join(__dirname, 'file-prompt-preload.js'), + contextIsolation: true, + nodeIntegration: false, + }, + }); + + const escapedName = fileName.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + const escapedMime = mimeType.replace(/&/g, '&').replace(//g, '>'); + + const html = ` + + + + + + +
\u270f\ufe0f 输入提示语（可选）
+
+ ${typeLabel} ${escapedName}
+ ${escapedMime} +
+ +
+ + +
+ + +`; + + filePromptWindow.loadURL('data:text/html;charset=utf-8,' + encodeURIComponent(html)); + + filePromptWindow.on('closed', () => { + filePromptWindow = null; + if (pendingFileInfo) { + pendingFileInfo = null; + if (petWindow && !petWindow.isDestroyed()) { + petWindow.webContents.send('bubble-show', { text: null, animation: 'standby' }); + } + } + }); +} + /** * IPC 通信处理器 * 处理渲染进程发来的各种请求 @@ -2083,6 +2189,59 @@ ipcMain.on('chat-history', (_event, history) => { } }); +// ==================== 拖放区展开/收起 ==================== + +ipcMain.on('toggle-drop-zone', (_event, enabled) => { + if (petWindow && !petWindow.isDestroyed()) { + const display = screen.getPrimaryDisplay(); + const area = display.workArea; + const w = enabled ? PET_EXPANDED_WIDTH : PET_WIDTH; + const h = enabled ? PET_EXPANDED_HEIGHT : PET_HEIGHT; + const x = Math.max(area.x, area.x + area.width - w - PET_WINDOW_MARGIN); + const y = Math.max(area.y, area.y + area.height - h - PET_WINDOW_MARGIN); + logToFile(`[drop-zone] ${enabled ? 'expand' : 'collapse'} to ${w}x${h} @ (${x},${y})`); + petWindow.setBounds({ width: w, height: h, x, y }); + } +}); + +// 打开原生文件选择器 +ipcMain.on('open-file-dialog', async () => { + logToFile('[drop-zone] opening file dialog'); + try { + const result = await dialog.showOpenDialog({ + properties: ['openFile'], + filters: [ + { + name: 'Supported files', + extensions: [ + 'txt', 'md', 'json', 'xml', 'yaml', 'yml', 'csv', + 'js', 'ts', 'jsx', 'tsx', 'py', 'go', 'rs', 'java', 'c', 'cpp', 'h', + 'css', 'scss', 'html', 'htm', 'sh', 'bash', 'zsh', 'ps1', 'bat', 'cmd', + 'sql', 'rb', 'php', 'swift', 'kt', 'scala', 'dart', + 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'webp', 'svg', 'ico', + ], + }, + ], + }); + + if (!result.canceled && result.filePaths.length > 0) { + const filePath = result.filePaths[0]; + const fileName = path.basename(filePath); + logToFile(`[drop-zone] file selected: ${fileName}`); + handleFileDropEvent(filePath, fileName); + } + } catch (err) { + logToFile(`[drop-zone] dialog failed: ${err.message}`); + } +}); + +// 当文件开始处理后通过此 IPC 通知桌宠关闭拖放区 +ipcMain.on('file-processing-start', () => { + if (petWindow && !petWindow.isDestroyed()) { + petWindow.webContents.send('drop-zone-auto-close'); + } +}); + // 显示气泡消息（桌宠说话） ipcMain.on('show-bubble', (_event, data) => { const audio = typeof data?.audio === 'string' ? data.audio.trim() : ''; @@ -2136,6 +2295,151 @@ ipcMain.on('connection-alive', () => { } }); +// ==================== 文件拖放 ==================== + +// 去重缓存：同一文件 3 秒内不处理第二次（preload + will-navigate 双触发保护） +const recentFileDrops = new Map(); + +/** + * 将文件消息转发给 settings window（WebSocket 发送）， + * 如果 settings window 不存在则自动重建并等待加载后发送 + */ +function forwardFileToSettings(info, prompt) { + function sendPayload() { + if (!settingsWindow || settingsWindow.isDestroyed()) { + logToFile('[file-drop] settings window still unavailable after recreate'); + return; + } + settingsWindow.webContents.send('incoming-file-message', { + fileName: info.fileName, + mimeType: info.mimeType, + textContent: info.textContent || '', + base64Content: info.base64Content || '', + isImage: info.isImage, + isBinary: info.isBinary || false, + prompt: prompt, + }); + } + + if (settingsWindow && !settingsWindow.isDestroyed()) { + sendPayload(); + } else { + logToFile('[file-drop] settings window not found, recreating...'); + // 主窗口 URL（不带 onboarding） + const url = buildSettingsWindowUrl({ onboarding: false }); + createSettingsWindow(url); + // 等页面加载完成后再发送 + if (settingsWindow && !settingsWindow.isDestroyed()) { + settingsWindow.webContents.once('did-finish-load', sendPayload); + // 兜底：5秒后如果还没触发，也尝试发送 + setTimeout(() => { + if (settingsWindow && !settingsWindow.isDestroyed()) { + sendPayload(); + } + }, 5000); + } + } +} + +/** + * 处理文件拖放（读文件 + 弹窗输入） + * 同时被 file-dropped IPC 和 will-navigate 防护调用 + */ +function handleFileDropEvent(filePath, fileName) { + logToFile(`[file-drop] handling: ${fileName} path=${filePath}`); + + // 去重：同一文件 3 秒内不处理第二次 + const now = Date.now(); + const lastDrop = recentFileDrops.get(filePath); + if (lastDrop && now - lastDrop < 3000) { + logToFile(`[file-drop] skipped duplicate: ${fileName}`); + return; + } + recentFileDrops.set(filePath, now); + + try { + if (!filePath || !fileName) { + logToFile('[file-drop] invalid filePath or fileName'); + return; + } + + const ext = path.extname(fileName).toLowerCase(); + const imageExts = ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp', '.svg', '.ico']; + const binaryExts = ['.pdf', '.docx', '.doc', '.xlsx', '.xls', '.pptx', '.ppt', '.zip', '.rar', '.7z', '.tar', '.gz']; + const isImage = imageExts.includes(ext); + const isBinary = !isImage && binaryExts.includes(ext); + + let textContent = ''; + let base64Content = ''; + + if (isImage) { + const buffer = fs.readFileSync(filePath); + base64Content = buffer.toString('base64'); + } else if (isBinary) { + const buffer = fs.readFileSync(filePath); + base64Content = buffer.toString('base64'); + } else { + textContent = fs.readFileSync(filePath, 'utf-8'); + } + + const mimeType = isImage ? 'image/' + ext.replace('.', '') : 'text/plain'; + pendingFileInfo = { fileName, mimeType, textContent, base64Content, isImage, isBinary }; + + // 通知桌宠关闭拖放区 + if (petWindow && !petWindow.isDestroyed()) { + petWindow.webContents.send('drop-zone-auto-close'); + } + + createFilePromptWindow(fileName, mimeType, isImage); + } catch (err) { + logToFile(`[file-drop] FAILED: ${err.message}`); + if (petWindow && !petWindow.isDestroyed()) { + petWindow.webContents.send('error-notification', { + level: 'error', + code: 'file_read_error', + message: '无法读取文件: ' + err.message, + }); + } + } +} + +// 文件拖放 — 从桌宠 IPC 触发 +ipcMain.on('file-dropped', (_event, { filePath, fileName }) => { + logToFile(`[file-drop] IPC received: ${fileName}`); + handleFileDropEvent(filePath, fileName); +}); + +// 文件弹窗提交 +ipcMain.on('file-prompt-submit', (_event, prompt) => { + logToFile(`[file-drop] prompt submitted: "${prompt ? prompt.slice(0, 50) : ''}"`); + + const info = pendingFileInfo; + pendingFileInfo = null; + + if (filePromptWindow && !filePromptWindow.isDestroyed()) { + filePromptWindow.close(); + } + + if (!info) { + logToFile('[file-drop] no pending file info'); + return; + } + + forwardFileToSettings(info, prompt || ''); +}); + +// 文件弹窗取消 +ipcMain.on('file-prompt-cancel', () => { + logToFile('[file-drop] cancelled'); + pendingFileInfo = null; + if (filePromptWindow && !filePromptWindow.isDestroyed()) { + filePromptWindow.close(); + } + if (petWindow && !petWindow.isDestroyed()) { + petWindow.webContents.send('bubble-show', { text: null, animation: 'standby' }); + } +}); + // 获取启动状态 ipcMain.handle('startup-state', async () => startupState); diff --git a/clawpet-frontend/clawpet/electron/preload.js b/clawpet-frontend/clawpet/electron/preload.js index f93b5c53..33f4cb9b 100644 --- a/clawpet-frontend/clawpet/electron/preload.js +++ b/clawpet-frontend/clawpet/electron/preload.js @@ -17,6 +17,21 @@ const { contextBridge, ipcRenderer } = require('electron'); +// ====== 文件拖放（document 级，绕开 drag-region 拦截) ====== +document.addEventListener('dragover', (e) => { + e.preventDefault(); +}); + +document.addEventListener('drop', (e) => { + e.preventDefault(); + const files = e.dataTransfer?.files; + if (files && files.length > 0) { + const file = files[0]; + ipcRenderer.send('show-bubble', { text: null, animation: 'think' }); + ipcRenderer.send('file-dropped', { filePath: file.path, fileName: file.name }); + } +}); + /** * 暴露 electronAPI 到全局 window 对象 * 渲染进程（网页）可以通过 window.electronAPI 调用这些方法 @@ -315,6 +330,58 @@ contextBridge.exposeInMainWorld('electronAPI', { ipcRenderer.send('show-error-notification', data); }, + // ==================== 文件拖放区 ==================== + + /** + * 展开/收起桌宠拖放区（调整窗口大小） + * @param {boolean} enabled - true 展开，false 收起 + */ + toggleDropZone: (enabled) => { + ipcRenderer.send('toggle-drop-zone', Boolean(enabled)); + }, + + /** + * 打开原生文件选择器对话框 + */ + openFileDialog: () => { + ipcRenderer.send('open-file-dialog'); + }, + + /** + * 监听拖放区自动关闭事件（文件开始处理后） + * @param {function} callback + */ + onDropZoneAutoClose: (callback) => { + ipcRenderer.on('drop-zone-auto-close', () => callback()); + }, + + // ==================== 文件拖放 ==================== + + /** + * 桌宠收到文件拖放后发送到主进程 + * @param {string} filePath - 完整文件路径 + * @param {string} fileName - 文件名 + */ + sendFileDropped: (filePath, fileName) => { + ipcRenderer.send('file-dropped', { filePath, fileName }); + }, + + /** + * 监听文件消息（主进程 → 对话窗口） + * @param {function} callback - (data) => void + */ + onIncomingFileMessage: (callback) => { + ipcRenderer.on('incoming-file-message', (_event, data) => callback(data)); + }, + + /** + * 监听文件拖放取消 + * @param {function} callback + */ + onFileDropCancel: (callback) => { + ipcRenderer.on('file-drop-cancel', () => callback()); + }, + /** * ?????? * @param {function} callback - ???? diff --git a/clawpet-frontend/clawpet/hooks/use-chat.ts b/clawpet-frontend/clawpet/hooks/use-chat.ts index 306b2919..5155317e 100644 --- a/clawpet-frontend/clawpet/hooks/use-chat.ts +++ b/clawpet-frontend/clawpet/hooks/use-chat.ts @@ -13,6 +13,7 @@ import { deleteSession as deleteSessionOnServer, getSessionHistory, } from "@/lib/api/sessions" +import { extractPetText } from "@/lib/utils" const SESSIONS_STORAGE_KEY = "petclaw_sessions" const ACTIVE_SESSION_KEY = "petclaw_active_session" @@ -1084,6 +1085,7 @@ export function useChat(options: UseChatOptions = {}): UseChatResult { case "message": if (typeof event.data === "object" && event.data) { const message = event.data as ChatMessage + message.content = extractPetText(message.content || "") updateSessionMessages(activeSessionIdRef.current, (prev) => mergeMessage(prev, message), ) @@ -1675,6 +1677,51 @@ export function useChat(options: UseChatOptions = {}): UseChatResult { }, [clearPendingBubbleTimer, connectWithBootstrap, resetAudioQueue]) useEffect(() => { + const unlistenFile = window.electronAPI?.onIncomingFileMessage?.((data: { + fileName: string + mimeType: string + textContent?: string + base64Content?: string + isImage: boolean + isBinary?: boolean + prompt: string + }) => { + const sessionKey = activeSessionIdRef.current + if (!sessionKey || !wsRef.current?.isConnected) { + window.electronAPI?.showErrorNotification?.({ + level: "warn", + code: "no_connection", + message: "当前没有可用连接，无法发送文件", + }) + return + } + + const label = data.isImage ? "图片" : "文件" + const userContent = data.prompt + ? `[${label} ${data.fileName}] ${data.prompt}` + : `[${label} ${data.fileName}] 请分析这个文件` + + wsRef.current.sendAction("file_chat", { + session_key: sessionKey, + prompt: data.prompt, + file_name: data.fileName, + file_mime: data.mimeType, + file_content: (data.textContent || data.base64Content || ""), + file_is_image: data.isImage, + file_is_binary: data.isBinary || false, + }) + + updateSessionMessages(sessionKey, (prev) => [ + ...prev, + { + id: `user-file-${Date.now()}`, + role: "user", + content: userContent, + timestamp: Date.now(), + } as ChatMessage, + ]) + }) + const unlisten = window.electronAPI?.onForceStopMedia?.(() => { if (currentAudioRef.current) { currentAudioRef.current.pause() @@ -1697,9 +1744,10 @@ export function useChat(options: UseChatOptions = {}): UseChatResult { }) return () => { + unlistenFile?.() unlisten?.() } - }, [resetAudioQueue]) + }, [resetAudioQueue, updateSessionMessages]) const activeSession = sessionsState.find((session) => session.id === activeSessionId) ?? diff --git a/clawpet-frontend/clawpet/lib/utils.ts b/clawpet-frontend/clawpet/lib/utils.ts index fed2fe91..12e08bb5 100644 --- a/clawpet-frontend/clawpet/lib/utils.ts +++ b/clawpet-frontend/clawpet/lib/utils.ts @@ -4,3 +4,15 @@ import { twMerge } from 'tailwind-merge' export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)) } + +const petTagRe = /\[(?:text|emotion|action|mbti|memory_type)[^\]]*\]/g + +export function stripPetTags(text: string): string { + return text.replace(petTagRe, "").trim() +} + +export function extractPetText(text: string): string { + const m = text.match(/\[text:([^\]]*)\]/) + if (m) return m[1] + return stripPetTags(text) +} diff --git a/go.mod b/go.mod index 6daa9e7c..eeee8e8a 100644 --- a/go.mod +++ b/go.mod @@ -76,6 +76,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index 03526f37..fe54e7c7 100644 --- a/go.sum +++ b/go.sum @@ -173,6 +173,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.5.3 h1:xvf8Dv29kBXC5/DNDCLhHkAFW8l/0LlQJimO5Zn+JUk= github.com/larksuite/oapi-sdk-go/v3 v3.5.3/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8= +github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= diff --git a/pkg/agent/loop_mcp.go b/pkg/agent/loop_mcp.go index b9c844d1..5ac371c8 100644 --- a/pkg/agent/loop_mcp.go +++ b/pkg/agent/loop_mcp.go @@ -9,6 +9,7 @@ package agent import ( "context" "fmt" + "path/filepath" "sync" "github.com/sipeed/picoclaw/pkg/config" @@ -81,13 +82,15 @@ func (al *AgentLoop) ensureMCPInitialized(ctx context.Context) error { } al.mcp.initOnce.Do(func() { - mcpManager := mcp.NewManager() - + // binDir: workspace/../bin/ (e.g. .goclaw-runtime/bin/) defaultAgent := al.registry.GetDefaultAgent() workspacePath := al.cfg.WorkspacePath() if defaultAgent != nil && defaultAgent.Workspace != "" { workspacePath = defaultAgent.Workspace } + binDir := filepath.Join(filepath.Dir(workspacePath), "bin") + + mcpManager := mcp.NewManagerWithBinDir(binDir) if err := mcpManager.LoadFromMCPConfig(ctx, al.cfg.Tools.MCP, workspacePath); err != nil { logger.WarnCF("agent", "Failed to load MCP servers, MCP tools will not be available", diff --git a/pkg/channels/pet/channel.go b/pkg/channels/pet/channel.go index ffa6b629..79a6183d 100644 --- a/pkg/channels/pet/channel.go +++ b/pkg/channels/pet/channel.go @@ -174,6 +174,7 @@ func NewPetChannel(cfg config.PetConfig, msgBus *bus.MessageBus, workspacePath s WorkspacePath: workspacePath, Config: systemConfig, ConfigPath: configPath, + MediaStore: pc.GetMediaStore(), }) if err != nil { logger.Errorf("pet: failed to create PetService: %v", err) diff --git a/pkg/mcp/manager.go b/pkg/mcp/manager.go index 8d7c0995..ffa10c01 100644 --- a/pkg/mcp/manager.go +++ b/pkg/mcp/manager.go @@ -112,6 +112,7 @@ type Manager struct { mu sync.RWMutex closed atomic.Bool // changed from bool to atomic.Bool to avoid TOCTOU race wg sync.WaitGroup // tracks in-flight CallTool calls + binDir string // directory for auto-downloaded binaries (e.g. uvx) } // NewManager creates a new MCP manager @@ -121,6 +122,15 @@ func NewManager() *Manager { } } +// NewManagerWithBinDir creates a new MCP manager with a binary download directory. +// binDir is used to auto-download tools like uvx when they are not found in PATH. +func NewManagerWithBinDir(binDir string) *Manager { + return &Manager{ + servers: make(map[string]*ServerConnection), + binDir: binDir, + } +} + // LoadFromConfig loads MCP servers from configuration func (m *Manager) LoadFromConfig(ctx context.Context, cfg *config.Config) error { return m.LoadFromMCPConfig(ctx, cfg.Tools.MCP, cfg.WorkspacePath()) @@ -237,6 +247,24 @@ func (m *Manager) LoadFromMCPConfig( return nil } +// resolveCommand 解析命令路径，自动下载缺失的工具（如 uvx） +func (m *Manager) resolveCommand(command string) string { + if strings.Contains(command, string(os.PathSeparator)) { + return command // 已有完整路径，直接用 + } + if p, err := exec.LookPath(command); err == nil { + return p // PATH 中找到 + } + if (command == "uvx" || command == "uv") && m.binDir != "" { + p, dlErr := ensureUVX(m.binDir) + if dlErr == nil { + return p + } + logger.WarnCF("mcp", "Failed to auto-download uvx", map[string]any{"error": dlErr.Error()}) + } + return command // 回退原始值 +} + // ConnectServer connects to a single MCP server func (m *Manager) ConnectServer( ctx context.Context, @@ -324,8 +352,10 @@ func (m *Manager) ConnectServer( "server": name, "command": cfg.Command, }) + // 解析命令（自动下载 uvx 等工具） + resolvedCmd := m.resolveCommand(cfg.Command) // Create command with context - cmd := exec.CommandContext(ctx, cfg.Command, cfg.Args...) + cmd := exec.CommandContext(ctx, resolvedCmd, cfg.Args...) processutil.PrepareBackgroundCommand(cmd) // Build environment variables with proper override semantics diff --git a/pkg/mcp/uv.go b/pkg/mcp/uv.go new file mode 100644 index 00000000..3f690695 --- /dev/null +++ b/pkg/mcp/uv.go @@ -0,0 +1,105 @@ +package mcp + +import ( + "archive/zip" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/sipeed/picoclaw/pkg/logger" +) + +const ( + uvVersion = "0.11.14" + uvDownloadURL = "https://github.com/astral-sh/uv/releases/download/" + uvVersion + "/uv-{arch}-pc-windows-msvc.zip" +) + +// ensureUVX 确保 uvx 可用，返回 uvx.exe 的完整路径。 +// 先检查系统 PATH，再检查 installDir，最后从 GitHub 下载。 +func ensureUVX(installDir string) (string, error) { + // 1. 检查系统 PATH + if p, err := exec.LookPath("uvx"); err == nil { + return p, nil + } + + // 2. 检查安装目录 + os.MkdirAll(installDir, 0755) + uvxPath := filepath.Join(installDir, "uvx.exe") + if info, err := os.Stat(uvxPath); err == nil && !info.IsDir() { + return uvxPath, nil + } + + // 3. 从 GitHub 下载 + logger.InfoCF("mcp", "Downloading uv "+uvVersion, map[string]any{ + "install_dir": installDir, + }) + + arch := "x86_64" + if runtime.GOARCH == "arm64" { + arch = "aarch64" + } + url := strings.ReplaceAll(uvDownloadURL, "{arch}", arch) + + tmpZip := filepath.Join(os.TempDir(), "uv-download.zip") + defer os.Remove(tmpZip) + + // 下载 + out, err := os.Create(tmpZip) + if err != nil { + return "", fmt.Errorf("create temp file: %w", err) + } + resp, err := http.Get(url) + if err != nil { + out.Close() + return "", fmt.Errorf("download uv: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + out.Close() + return "", fmt.Errorf("download uv: HTTP %d", resp.StatusCode) + } + _, err = io.Copy(out, resp.Body) + out.Close() + if err != nil { + return "", fmt.Errorf("save uv zip: %w", err) + } + + // 解压 + zr, err := zip.OpenReader(tmpZip) + if err != nil { + return "", fmt.Errorf("open uv zip: %w", err) + } + defer zr.Close() + for _, f := range zr.File { + if f.Name == "uv.exe" || f.Name == "uvx.exe" { + dst := filepath.Join(installDir, f.Name) + rc, openErr := f.Open() + if openErr != nil { + continue + } + dstF, createErr := os.Create(dst) + if createErr != nil { + rc.Close() + continue + } + io.Copy(dstF, rc) + dstF.Close() + rc.Close() + } + } + + // 验证 + if info, err := os.Stat(uvxPath); err == nil && !info.IsDir() { + logger.InfoCF("mcp", "uv downloaded successfully", map[string]any{ + "path": uvxPath, + }) + return uvxPath, nil + } + + return "", fmt.Errorf("uvx not found after download") +} diff --git a/pkg/pet/service.go b/pkg/pet/service.go index 6e29afea..0ea5690f 100644 --- a/pkg/pet/service.go +++ b/pkg/pet/service.go @@ -1,19 +1,27 @@ package pet import ( + "archive/zip" + "bytes" "context" "encoding/base64" "encoding/json" + "encoding/xml" "errors" "fmt" + "io" + "os" "path/filepath" + "strings" "sync" "time" + "github.com/ledongthuc/pdf" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/cron" "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/media" "github.com/sipeed/picoclaw/pkg/pet/action" "github.com/sipeed/picoclaw/pkg/pet/activity" "github.com/sipeed/picoclaw/pkg/pet/asr" @@ -53,6 +61,7 @@ type PetService struct { skillsMgr *skills.Manager activityStore *activity.Store proactiveManager *proactive.Manager + mediaStore media.MediaStore connSessions map[string]string activeSessionID string @@ -70,6 +79,7 @@ type PetServiceConfig struct { WorkspacePath string Config *config.Config ConfigPath string + MediaStore media.MediaStore } func NewPetService(msgBus *bus.MessageBus, cfg PetServiceConfig) (*PetService, error) { @@ -81,6 +91,7 @@ func NewPetService(msgBus *bus.MessageBus, cfg PetServiceConfig) (*PetService, e connSessions: make(map[string]string), ctx: ctx, cancel: cancel, + mediaStore: cfg.MediaStore, } workspacePath := cfg.WorkspacePath if workspacePath != "" { @@ -742,6 +753,8 @@ func (s *PetService) HandleRequest(connID string, req Request) error { return s.handleSkillRemove(sessionID, req) case ActionSkillGet: return s.handleSkillGet(sessionID, req) + case ActionFileChat: + return s.handleFileChat(sessionID, req) case ActionAudioFrame: return s.handleAudioFrame(sessionID, req) case ActionVoiceConfigGet: @@ -826,6 +839,215 @@ func (s *PetService) handleChat(sessionID string, req Request) error { return s.sendResponse(sessionID, req.Action, map[string]string{"session_key": chatReq.SessionKey}) } +func (s *PetService) handleFileChat(sessionID string, req Request) error { + var fileReq FileChatRequest + if err := json.Unmarshal(req.Data, &fileReq); err != nil { + logger.Errorf("pet: handleFileChat invalid data: %v", err) + return s.sendError(sessionID, req.Action, "invalid file chat data") + } + + if fileReq.FileName == "" { + return s.sendError(sessionID, req.Action, "file name is required") + } + + if !fileReq.FileIsImage && fileReq.FileContent == "" { + return s.sendError(sessionID, req.Action, "file content is empty") + } + + char := s.charManager.GetCurrent() + if char == nil { + return s.sendError(sessionID, req.Action, "no active character") + } + + if fileReq.Prompt == "" { + fileReq.Prompt = "请分析这个文件" + } + + s.recordUserActivity(sessionID, fileReq.FileName) + if s.proactiveManager != nil { + s.proactiveManager.Trigger("user_message") + } + + logger.Warnf("pet: handleFileChat file=%s mime=%s isImage=%v prompt=%s", + fileReq.FileName, fileReq.FileMime, fileReq.FileIsImage, fileReq.Prompt) + + var content string + var mediaRefs []string + const maxContentLen = 50000 + + if fileReq.FileIsImage { + // 图片：尝试通过 MediaStore 存储，用 media:// 引用 + rawData, err := base64.StdEncoding.DecodeString(fileReq.FileContent) + if err != nil { + logger.Warnf("pet: handleFileChat base64 decode failed: %v", err) + } + + if err == nil && s.mediaStore != nil { + scope := fmt.Sprintf("file_chat:%s:%s", char.ID, sessionID) + // 写临时文件再存到 MediaStore + tmpDir := filepath.Join(os.TempDir(), "picoclaw-media") + if mkErr := os.MkdirAll(tmpDir, 0755); mkErr == nil { + tmpPath := filepath.Join(tmpDir, fmt.Sprintf("fc_%s_%s", sessionID, fileReq.FileName)) + if writeErr := os.WriteFile(tmpPath, rawData, 0644); writeErr == nil { + ref, storeErr := s.mediaStore.Store(tmpPath, media.MediaMeta{ + ContentType: fileReq.FileMime, + Filename: fileReq.FileName, + Source: "file_chat", + CleanupPolicy: media.CleanupPolicyDeleteOnCleanup, + }, scope) + if storeErr == nil { + mediaRefs = append(mediaRefs, fmt.Sprintf("media://%s", ref)) + // 同时写入工作区，方便 LLM 用 load_image 访问 + wsPath := "" + if s.config.WorkspacePath != "" { + wsDir := filepath.Join(s.config.WorkspacePath, ".uploads") + if mkErr2 := os.MkdirAll(wsDir, 0755); mkErr2 == nil { + wsP := filepath.Join(wsDir, fileReq.FileName) + if wsErr := os.WriteFile(wsP, rawData, 0644); wsErr == nil { + wsPath = wsP + } + } + } + if wsPath != "" { + content = fmt.Sprintf( + "用户上传了图片「%s」，并说：%s\n请使用 understand_image 工具分析这张图片，参数 image_source 设为文件路径。\n文件路径: %s", + fileReq.FileName, fileReq.Prompt, wsPath, + ) + } else { + content = fmt.Sprintf( + "用户上传了图片「%s」，并说：%s\n[image: media://%s]", + fileReq.FileName, fileReq.Prompt, ref, + ) + } + } else { + logger.Warnf("pet: handleFileChat Store failed: %v", storeErr) + _ = os.Remove(tmpPath) + } + } + } + } + + // 降级：如果 MediaStore 不可用或保存失败，写入工作区 + if content == "" { + workspace := s.config.WorkspacePath + if workspace != "" { + tmpDir := filepath.Join(workspace, ".uploads") + if mkErr := os.MkdirAll(tmpDir, 0755); mkErr == nil { + tmpPath := filepath.Join(tmpDir, fileReq.FileName) + if len(rawData) > 0 { + if writeErr := os.WriteFile(tmpPath, rawData, 0644); writeErr == nil { + content = fmt.Sprintf( + "用户上传了图片「%s」，并说：%s\n请使用 understand_image 工具分析这张图片，参数 image_source 设为文件路径。\n文件路径: %s", + fileReq.FileName, fileReq.Prompt, tmpPath, + ) + } + } + } + } + } + + // 最终降级：仅通知文件名 + if content == "" { + content = fmt.Sprintf( + "用户上传了图片「%s」，并说：%s\n（图片数据未加载，类型：%s）", + fileReq.FileName, fileReq.Prompt, fileReq.FileMime, + ) + } + } else if fileReq.FileIsBinary { + // 二进制文件（PDF、Word 等）：尝试提取纯文本 + rawData, err := base64.StdEncoding.DecodeString(fileReq.FileContent) + if err != nil { + logger.Warnf("pet: handleFileChat base64 decode failed for binary: %v", err) + } + + // 写入工作区 + wsPath := "" + if s.config.WorkspacePath != "" { + wsDir := filepath.Join(s.config.WorkspacePath, ".uploads") + if mkErr := os.MkdirAll(wsDir, 0755); mkErr == nil { + wsP := filepath.Join(wsDir, fileReq.FileName) + if len(rawData) > 0 { + if writeErr := os.WriteFile(wsP, rawData, 0644); writeErr == nil { + wsPath = wsP + } + } + } + } + + // 尝试提取文本 + extracted := "" + ext := strings.ToLower(filepath.Ext(fileReq.FileName)) + switch ext { + case ".docx": + extracted = extractDocxText(rawData) + case ".pdf": + if wsPath != "" { + extracted = extractPdfText(wsPath) + } + } + + if extracted != "" { + truncated := "" + if len(extracted) > maxContentLen { + extracted = extracted[:maxContentLen] + truncated = " (文件较大，已截取前50000字符)" + } + content = fmt.Sprintf( + "用户上传了文件「%s」%s，内容如下：\n---文件内容---\n%s\n---\n用户说：%s", + fileReq.FileName, truncated, extracted, fileReq.Prompt, + ) + } else if wsPath != "" { + content = fmt.Sprintf( + "用户上传了文件「%s」，并说：%s\n文件已保存到路径: %s\n可以使用 read_file 工具读取原始内容。", + fileReq.FileName, fileReq.Prompt, wsPath, + ) + } else { + content = fmt.Sprintf( + "用户上传了文件「%s」，并说：%s\n（无法读取文件）", + fileReq.FileName, fileReq.Prompt, + ) + } + } else { + // 纯文本文件：直接嵌入内容 + fileContent := fileReq.FileContent + truncated := "" + if len(fileContent) > maxContentLen { + fileContent = fileContent[:maxContentLen] + truncated = " (文件较大，已截取前50000字符)" + } + content = fmt.Sprintf( + "用户上传了文件「%s」%s，内容如下：\n---文件内容---\n%s\n---\n用户说：%s", + fileReq.FileName, truncated, fileContent, fileReq.Prompt, + ) + } + + inbound := bus.InboundMessage{ + Channel: "pet", + ChatID: sessionID, + SessionKey: fileReq.SessionKey, + Peer: bus.Peer{ + Kind: char.ID, + ID: fileReq.SessionKey, + }, + Content: content, + Media: mediaRefs, + Metadata: map[string]string{"type": "file_chat", "conn_id": req.RequestID, "file_name": fileReq.FileName}, + } + + if err := s.msgBus.PublishInbound(context.Background(), inbound); err != nil { + logger.Errorf("pet: handleFileChat PublishInbound failed: %v", err) + return s.sendError(sessionID, req.Action, err.Error()) + } + + logger.Warnf("pet: handleFileChat published session_key=%s chat_id=%s content_len=%d", + fileReq.SessionKey, sessionID, len(content)) + + return s.sendResponse(sessionID, req.Action, map[string]string{ + "session_key": fileReq.SessionKey, + "file_name": fileReq.FileName, + }) +} + func (s *PetService) handleOnboardingConfig(sessionID string, req Request) error { var data OnboardingConfigRequest if err := json.Unmarshal(req.Data, &data); err != nil { @@ -3022,3 +3244,84 @@ func ensureDefaultActions(mgr *action.ActionManager) { _ = mgr.Register(a) // 同名已存在时 Register 返回 error，直接忽略 } } + +// docx 内部结构（不含 w: 命名空间，解析前先替换） +type docxFlatDocument struct { + Body docxFlatBody `xml:"body"` +} + +type docxFlatBody struct { + Paragraphs []docxFlatParagraph `xml:"p"` +} + +type docxFlatParagraph struct { + Runs []docxFlatRun `xml:"r"` +} + +type docxFlatRun struct { + Text string `xml:"t"` +} + +// extractDocxText 从 .docx 文件的原始字节中提取纯文本 +func extractDocxText(rawData []byte) string { + r, err := zip.NewReader(bytes.NewReader(rawData), int64(len(rawData))) + if err != nil { + return "" + } + for _, f := range r.File { + if f.Name == "word/document.xml" { + rc, openErr := f.Open() + if openErr != nil { + return "" + } + data, readErr := io.ReadAll(rc) + rc.Close() + if readErr != nil { + return "" + } + // 剥离 w: 命名空间前缀 + data = bytes.ReplaceAll(data, []byte(" 0 { timeout = time.Duration(timeoutSeconds) * time.Second } - return &Provider{ + p := &Provider{ apiKey: apiKey, apiBase: baseURL, userAgent: userAgent, @@ -65,6 +77,14 @@ func NewProviderWithTimeout(apiKey, apiBase, userAgent string, timeoutSeconds in Timeout: timeout, }, } + + for _, opt := range opts { + if opt != nil { + opt(p) + } + } + + return p } // Chat sends messages to the Anthropic Messages API and returns the response. @@ -79,6 +99,16 @@ func (p *Provider) Chat( return nil, fmt.Errorf("API key not configured") } + // Strip image media when the API doesn't support it (e.g. MiniMax) + if p.disableImages { + cleaned := make([]Message, len(messages)) + for i, m := range messages { + m.Media = nil + cleaned[i] = m + } + messages = cleaned + } + // Build request body requestBody, err := buildRequestBody(messages, tools, model, options) if err != nil { @@ -199,17 +229,22 @@ func buildRequestBody( "tool_use_id": msg.ToolCallID, "content": msg.Content, } + merged := false if len(apiMessages) > 0 { if prev, ok := apiMessages[len(apiMessages)-1].(map[string]any); ok && prev["role"] == "user" { if content, ok := prev["content"].([]map[string]any); ok { prev["content"] = append(content, toolResultBlock) - continue + merged = true } } } + if merged { + continue + } + // Orphaned tool_result — fall back to plain text apiMessages = append(apiMessages, map[string]any{ "role": "user", - "content": []map[string]any{toolResultBlock}, + "content": msg.Content, }) } else { // Regular user message @@ -251,10 +286,13 @@ func buildRequestBody( content = append(content, toolUse) } - apiMessages = append(apiMessages, map[string]any{ - "role": "assistant", - "content": content, - }) + // Skip assistant messages with neither text nor tool calls + if len(content) > 0 { + apiMessages = append(apiMessages, map[string]any{ + "role": "assistant", + "content": content, + }) + } case "tool": // Tool result (alternative format) — merge into previous user message if it contains tool_results @@ -263,17 +301,22 @@ func buildRequestBody( "tool_use_id": msg.ToolCallID, "content": msg.Content, } + merged := false if len(apiMessages) > 0 { if prev, ok := apiMessages[len(apiMessages)-1].(map[string]any); ok && prev["role"] == "user" { if content, ok := prev["content"].([]map[string]any); ok { prev["content"] = append(content, toolResultBlock) - continue + merged = true } } } + if merged { + continue + } + // Orphaned tool_result — fall back to plain text apiMessages = append(apiMessages, map[string]any{ "role": "user", - "content": []map[string]any{toolResultBlock}, + "content": msg.Content, }) } } diff --git a/pkg/providers/factory_provider.go b/pkg/providers/factory_provider.go index ab7277fa..7c84d509 100644 --- a/pkg/providers/factory_provider.go +++ b/pkg/providers/factory_provider.go @@ -15,6 +15,7 @@ import ( anthropicmessages "github.com/sipeed/picoclaw/pkg/providers/anthropic_messages" "github.com/sipeed/picoclaw/pkg/providers/azure" "github.com/sipeed/picoclaw/pkg/providers/bedrock" + "github.com/sipeed/picoclaw/pkg/providers/openai_compat" ) type protocolMeta struct { @@ -241,7 +242,6 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err ), modelID, nil case "minimax": - // Minimax requires reasoning_split: true in the request body if cfg.APIKey() == "" && cfg.APIBase == "" { return nil, "", fmt.Errorf("api_key or api_base is required for HTTP-based protocol %q", protocol) } @@ -249,22 +249,19 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err if apiBase == "" { apiBase = getDefaultAPIBase(protocol) } - extraBody := cfg.ExtraBody - if extraBody == nil { - extraBody = make(map[string]any) - } - if _, ok := extraBody["reasoning_split"]; !ok { - extraBody["reasoning_split"] = true - } - return NewHTTPProviderWithMaxTokensFieldAndRequestTimeout( - cfg.APIKey(), - apiBase, - cfg.Proxy, - cfg.MaxTokensField, - userAgent, - cfg.RequestTimeout, - extraBody, - ), modelID, nil + return &HTTPProvider{ + delegate: openai_compat.NewProvider( + cfg.APIKey(), + apiBase, + cfg.Proxy, + openai_compat.WithMaxTokensField(cfg.MaxTokensField), + openai_compat.WithRequestTimeout(time.Duration(cfg.RequestTimeout)*time.Second), + openai_compat.WithExtraBody(cfg.ExtraBody), + openai_compat.WithUserAgent(userAgent), + openai_compat.WithSkipTools(true), + openai_compat.WithMergeSystemMessages(true), + ), + }, modelID, nil case "anthropic": if cfg.AuthMethod == "oauth" || cfg.AuthMethod == "token" { @@ -309,6 +306,23 @@ func CreateProviderFromConfig(cfg *config.ModelConfig) (LLMProvider, string, err cfg.RequestTimeout, ), modelID, nil + case "minimax-anthropic": + // MiniMax via Anthropic-compatible API (supports tools/function calling) + apiBase := cfg.APIBase + if apiBase == "" { + apiBase = "https://api.minimaxi.com/anthropic" + } + if cfg.APIKey() == "" { + return nil, "", fmt.Errorf("api_key is required for minimax-anthropic protocol (model: %s)", cfg.Model) + } + return anthropicmessages.NewProviderWithTimeout( + cfg.APIKey(), + apiBase, + userAgent, + cfg.RequestTimeout, + anthropicmessages.WithDisableImages(true), + ), modelID, nil + case "coding-plan-anthropic", "alibaba-coding-anthropic": // Alibaba Coding Plan with Anthropic-compatible API apiBase := cfg.APIBase diff --git a/pkg/providers/openai_compat/provider.go b/pkg/providers/openai_compat/provider.go index 7cda033a..bd121523 100644 --- a/pkg/providers/openai_compat/provider.go +++ b/pkg/providers/openai_compat/provider.go @@ -31,12 +31,14 @@ type ( ) type Provider struct { - apiKey string - apiBase string - maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models) - httpClient *http.Client - extraBody map[string]any // Additional fields to inject into request body - userAgent string + apiKey string + apiBase string + maxTokensField string // Field name for max tokens (e.g., "max_completion_tokens" for o1/glm models) + httpClient *http.Client + extraBody map[string]any // Additional fields to inject into request body + userAgent string + skipTools bool // When true, tools/tool_choice are omitted from the request body + mergeSystemMessages bool // When true, multiple system messages are merged into one } type Option func(*Provider) @@ -87,6 +89,18 @@ func WithExtraBody(extraBody map[string]any) Option { } } +func WithSkipTools(skip bool) Option { + return func(p *Provider) { + p.skipTools = skip + } +} + +func WithMergeSystemMessages(merge bool) Option { + return func(p *Provider) { + p.mergeSystemMessages = merge + } +} + func NewProvider(apiKey, apiBase, proxy string, opts ...Option) *Provider { p := &Provider{ apiKey: apiKey, @@ -126,6 +140,10 @@ func (p *Provider) buildRequestBody( ) map[string]any { model = normalizeModel(model, p.apiBase) + if p.mergeSystemMessages { + messages = mergeSystemMessages(messages) + } + requestBody := map[string]any{ "model": model, "messages": common.SerializeMessages(messages), @@ -135,8 +153,10 @@ func (p *Provider) buildRequestBody( nativeSearch, _ := options["native_search"].(bool) nativeSearch = nativeSearch && isNativeSearchHost(p.apiBase) if len(tools) > 0 || nativeSearch { - requestBody["tools"] = buildToolsList(tools, nativeSearch) - requestBody["tool_choice"] = "auto" + if !p.skipTools { + requestBody["tools"] = buildToolsList(tools, nativeSearch) + requestBody["tool_choice"] = "auto" + } } if maxTokens, ok := common.AsInt(options["max_tokens"]); ok { @@ -471,3 +491,25 @@ func supportsPromptCacheKey(apiBase string) bool { host := u.Hostname() return host == "api.openai.com" || strings.HasSuffix(host, ".openai.azure.com") } + +// mergeSystemMessages consolidates consecutive system messages into one. +// Some providers (e.g. MiniMax) reject multiple system messages. +func mergeSystemMessages(messages []Message) []Message { + var merged []Message + var parts []string + for _, msg := range messages { + if msg.Role == "system" { + parts = append(parts, msg.Content) + } else { + if len(parts) > 0 { + merged = append(merged, Message{Role: "system", Content: strings.Join(parts, "\n\n")}) + parts = nil + } + merged = append(merged, msg) + } + } + if len(parts) > 0 { + merged = append(merged, Message{Role: "system", Content: strings.Join(parts, "\n\n")}) + } + return merged +}