From a43a23bde242e4adfd1ae261f0b73e18ba8c8ccb Mon Sep 17 00:00:00 2001 From: Vivek Agarwal Date: Mon, 13 Jul 2026 15:31:09 +0530 Subject: [PATCH] feat(cad-agent-plugin): add Google Gemini provider support Add @ai-sdk/google integration with Gemini 2.5/2.0/1.5 models to the model catalog, model factory, settings store, drawing verifier, and i18n. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/cad-agent-plugin/package.json | 1 + .../src/agent/createCadAgent.ts | 36 +++++++++++- .../cad-agent-plugin/src/agent/createModel.ts | 9 +++ .../src/agent/drawingVerifier.ts | 57 ++++++++++++++++--- packages/cad-agent-plugin/src/i18n/en.ts | 1 + packages/cad-agent-plugin/src/i18n/tr.ts | 1 + packages/cad-agent-plugin/src/i18n/zh.ts | 1 + .../src/storage/LlmSettingsStore.ts | 5 ++ .../src/storage/modelCatalog.ts | 32 +++++++++++ .../src/ui/AgentChatPanel.vue | 1 + pnpm-lock.yaml | 15 +++++ 11 files changed, 150 insertions(+), 9 deletions(-) diff --git a/packages/cad-agent-plugin/package.json b/packages/cad-agent-plugin/package.json index b51c4845..c8ff7e3a 100644 --- a/packages/cad-agent-plugin/package.json +++ b/packages/cad-agent-plugin/package.json @@ -49,6 +49,7 @@ }, "dependencies": { "@ai-sdk/anthropic": "^2.0.0", + "@ai-sdk/google": "^2.0.0", "@ai-sdk/openai": "^2.0.0", "@ai-sdk/vue": "^2.0.0", "ai": "^5.0.0", diff --git a/packages/cad-agent-plugin/src/agent/createCadAgent.ts b/packages/cad-agent-plugin/src/agent/createCadAgent.ts index 7165cf32..780cb410 100644 --- a/packages/cad-agent-plugin/src/agent/createCadAgent.ts +++ b/packages/cad-agent-plugin/src/agent/createCadAgent.ts @@ -54,7 +54,7 @@ export function createCadAgent(settings: LlmSettings) { model: createModelFromSettings(settings), system: CAD_AGENT_SYSTEM_PROMPT, tools: createCadTools(), - stopWhen: stepCountIs(10) + stopWhen: stepCountIs(20) }) } @@ -136,6 +136,13 @@ export function createAgentChatTransport( const { userRequest, referenceImages } = extractConversationContext(validatedMessages) let verificationAttempts = 0 + // Verifier memory: feedback from every prior round and the screenshot + // taken before the latest edits, so review is progress-aware. + const feedbackHistory: string[] = [] + let previousPreviewDataUrl: string | undefined + // Consecutive rounds whose feedback is unchanged from the round + // before — a stalled loop we stop early instead of burning attempts. + let noProgressRounds = 0 while (!abortSignal?.aborted) { workingMessages = await streamAgentRound({ @@ -181,6 +188,10 @@ export function createAgentChatTransport( userRequest, referenceImages, preview.dataUrl, + { + previousFeedback: feedbackHistory, + previousDrawingImage: previousPreviewDataUrl + }, abortSignal ) } catch (error) { @@ -201,6 +212,21 @@ export function createAgentChatTransport( break } + // Detect a stalled loop: feedback unchanged from the prior round. + const normalize = (text: string) => + text.trim().toLowerCase().replace(/\s+/g, ' ') + const lastFeedback = feedbackHistory[feedbackHistory.length - 1] + if ( + lastFeedback && + normalize(verification.feedback) === normalize(lastFeedback) + ) { + noProgressRounds += 1 + } else { + noProgressRounds = 0 + } + feedbackHistory.push(verification.feedback) + previousPreviewDataUrl = preview.dataUrl + if (verificationAttempts >= MAX_VERIFICATION_ATTEMPTS) { appendAssistantText( write, @@ -209,6 +235,14 @@ export function createAgentChatTransport( break } + if (noProgressRounds >= 2) { + appendAssistantText( + write, + `\n${agentT('verificationNoProgress')}\n\n${verification.feedback.trim()}` + ) + break + } + appendAssistantText( write, `\n${agentT('verificationFailed')}\n${verification.feedback.trim()}\n\n${agentT('verificationContinuing')}` diff --git a/packages/cad-agent-plugin/src/agent/createModel.ts b/packages/cad-agent-plugin/src/agent/createModel.ts index a378750a..0acf2c0a 100644 --- a/packages/cad-agent-plugin/src/agent/createModel.ts +++ b/packages/cad-agent-plugin/src/agent/createModel.ts @@ -1,4 +1,5 @@ import { createAnthropic } from '@ai-sdk/anthropic' +import { createGoogleGenerativeAI } from '@ai-sdk/google' import { createOpenAI } from '@ai-sdk/openai' import type { LanguageModel } from 'ai' @@ -28,6 +29,14 @@ export function createModelFromSettings(settings: LlmSettings): LanguageModel { return anthropic(settings.model) } + if (settings.provider === 'google') { + const google = createGoogleGenerativeAI({ + apiKey: settings.apiKey, + baseURL: settings.baseUrl || undefined + }) + return google(settings.model) + } + if (settings.provider === 'openai') { const openai = createOpenAI({ apiKey: settings.apiKey, diff --git a/packages/cad-agent-plugin/src/agent/drawingVerifier.ts b/packages/cad-agent-plugin/src/agent/drawingVerifier.ts index 0c3dbcb5..9131ae9e 100644 --- a/packages/cad-agent-plugin/src/agent/drawingVerifier.ts +++ b/packages/cad-agent-plugin/src/agent/drawingVerifier.ts @@ -5,7 +5,7 @@ import type { LlmSettings } from '../storage/LlmSettingsStore' import { createModelFromSettings } from './createModel' /** Maximum number of screenshot verification rounds per user message. */ -export const MAX_VERIFICATION_ATTEMPTS = 5 +export const MAX_VERIFICATION_ATTEMPTS = 8 const verificationSchema = z.object({ passed: z @@ -25,14 +25,34 @@ const VERIFICATION_SYSTEM_PROMPT = `You verify CAD drawings produced by an autom Compare the current drawing screenshot against the user's request and any reference images. Focus on overall geometry, topology, proportions, and layout — not exact pixel alignment. -Rules: -- Ignore dimensions, dimension lines, and dimension text in reference images. The drawing tools cannot create dimensions, so do not require them in the output drawing. -- Ignore missing annotations or labels unless the user explicitly asked for text entities. -- Pass when the drawing reasonably captures the requested shape and layout. -- Fail only when important geometry is missing, wrong, or clearly misaligned.` +Tool limitations — NEVER fail a drawing for any of these, they are impossible with the available tools: +- The tools can only draw solid, continuous lines. They cannot produce dashed, dotted, or "hidden" line styles. Treat every edge as solid; do not ask for hidden or dashed lines. +- The tools cannot create dimensions, dimension lines, extension lines, arrows, or tolerance/GD&T symbols. Ignore these entirely. +- Ignore missing annotations, labels, hatching styles, or line weights unless the user explicitly asked for text entities. + +Memory and progress: +- You may be shown the issues you reported in previous rounds and the previous screenshot (before the latest edits). Use them to judge PROGRESS. +- Confirm whether each previously reported issue is now resolved. Do NOT re-raise an issue that has already been fixed. +- Do NOT introduce new nitpicks each round to keep failing the drawing. Only report issues that genuinely remain and are fixable with the tools above. +- If the only remaining differences are unfixable (line style, dimensions, cosmetic) or the drawing already captures the requested geometry and layout, set passed = true. + +Verdict: +- Pass when the drawing reasonably captures the requested shape, features, and layout. +- Fail only when important geometry is missing, wrong, or clearly misaligned AND it can be fixed with solid-line drawing tools. +- In feedback, list only the concrete remaining issues and the specific fix for each. Keep it short.` export type DrawingVerificationResult = z.infer +/** + * Prior-round context that gives the verifier memory across verification rounds. + */ +export interface VerificationHistory { + /** Feedback strings reported in previous rounds, oldest first. */ + previousFeedback: string[] + /** Screenshot data URL from before the latest round of edits, if available. */ + previousDrawingImage?: string +} + /** * Sends the current drawing screenshot to the LLM for visual verification. * @@ -40,12 +60,14 @@ export type DrawingVerificationResult = z.infer * @param userRequest - Original user drawing request text. * @param referenceImages - User-attached reference image data URLs, if any. * @param drawingImage - PNG data URL of the current drawing. + * @param history - Previous rounds' feedback and screenshot for progress-aware review. */ export async function verifyDrawing( settings: LlmSettings, userRequest: string, referenceImages: string[], drawingImage: string, + history?: VerificationHistory, abortSignal?: AbortSignal ): Promise { const model = createModelFromSettings(settings) @@ -68,6 +90,25 @@ export async function verifyDrawing( } } + if (history && history.previousFeedback.length > 0) { + userContent.push({ + type: 'text', + text: + 'Issues you reported in previous rounds (oldest first). Check whether each is now resolved and do not re-raise fixed or unfixable ones:\n' + + history.previousFeedback + .map((f, i) => `Round ${i + 1}: ${f.trim()}`) + .join('\n') + }) + } + + if (history?.previousDrawingImage) { + userContent.push({ + type: 'text', + text: 'Previous drawing screenshot (before the latest edits), for comparing progress:' + }) + userContent.push({ type: 'image', image: history.previousDrawingImage }) + } + userContent.push({ type: 'text', text: 'Current drawing screenshot to verify:' @@ -97,9 +138,9 @@ export function buildVerificationFeedbackMessage( feedback: string ): string { return [ - `Drawing verification failed (attempt ${attempt}/${maxAttempts}).`, + `Drawing verification (attempt ${attempt}/${maxAttempts}) found remaining issues:`, feedback.trim(), - 'Please update the drawing to address these issues.' + 'Fix ONLY these issues. Do not delete or redraw geometry that is already correct — modify or add just what is needed, using the entityIds you recorded. The tools draw solid continuous lines only, so do not attempt hidden, dashed, or dimension lines.' ] .filter(Boolean) .join('\n\n') diff --git a/packages/cad-agent-plugin/src/i18n/en.ts b/packages/cad-agent-plugin/src/i18n/en.ts index dab70abc..13381c38 100644 --- a/packages/cad-agent-plugin/src/i18n/en.ts +++ b/packages/cad-agent-plugin/src/i18n/en.ts @@ -17,6 +17,7 @@ export const agentEn = { 'Uses an OpenAI-compatible vision endpoint (default: SiliconFlow). For self-hosted vLLM, set Base URL to your server, e.g. http://127.0.0.1:8000/v1.', providerOpenai: 'OpenAI', providerAnthropic: 'Anthropic', + providerGoogle: 'Google Gemini', providerOpenaiCompatible: 'OpenAI Compatible', baseUrl: 'Base URL', model: 'Model', diff --git a/packages/cad-agent-plugin/src/i18n/tr.ts b/packages/cad-agent-plugin/src/i18n/tr.ts index 6a1e1d7c..e88d2251 100644 --- a/packages/cad-agent-plugin/src/i18n/tr.ts +++ b/packages/cad-agent-plugin/src/i18n/tr.ts @@ -17,6 +17,7 @@ export const agentTr = { 'OpenAI uyumlu bir görsel uç nokta kullanır (varsayılan: SiliconFlow). Kendi barındırdığınız vLLM için Taban URL alanını sunucunuza ayarlayın, ör. http://127.0.0.1:8000/v1.', providerOpenai: 'OpenAI', providerAnthropic: 'Anthropic', + providerGoogle: 'Google Gemini', providerOpenaiCompatible: 'OpenAI Uyumlu', baseUrl: 'Taban URL', model: 'Model', diff --git a/packages/cad-agent-plugin/src/i18n/zh.ts b/packages/cad-agent-plugin/src/i18n/zh.ts index be83a6ce..ba6915fe 100644 --- a/packages/cad-agent-plugin/src/i18n/zh.ts +++ b/packages/cad-agent-plugin/src/i18n/zh.ts @@ -17,6 +17,7 @@ export const agentZh = { '使用 OpenAI 兼容的视觉接口（默认：SiliconFlow）。自托管 vLLM 时请将接口地址改为本地服务，例如 http://127.0.0.1:8000/v1。', providerOpenai: 'OpenAI', providerAnthropic: 'Anthropic', + providerGoogle: 'Google Gemini', providerOpenaiCompatible: 'OpenAI 兼容', baseUrl: '接口地址', model: '模型', diff --git a/packages/cad-agent-plugin/src/storage/LlmSettingsStore.ts b/packages/cad-agent-plugin/src/storage/LlmSettingsStore.ts index 7eb12035..19902a92 100644 --- a/packages/cad-agent-plugin/src/storage/LlmSettingsStore.ts +++ b/packages/cad-agent-plugin/src/storage/LlmSettingsStore.ts @@ -4,6 +4,7 @@ import { decryptApiKey, encryptApiKey } from './apiKeyCrypto' export type LlmProviderId = | 'openai' | 'anthropic' + | 'google' | 'openai-compatible' | 'deepseek' | 'deepseek-vl' @@ -51,6 +52,10 @@ export const PROVIDER_DEFAULTS: Record< baseUrl: 'https://api.anthropic.com/v1', model: 'claude-3-5-haiku-latest' }, + google: { + baseUrl: 'https://generativelanguage.googleapis.com/v1beta', + model: 'gemini-2.5-flash' + }, 'openai-compatible': { baseUrl: 'https://api.openai.com/v1', model: 'gpt-4o-mini' diff --git a/packages/cad-agent-plugin/src/storage/modelCatalog.ts b/packages/cad-agent-plugin/src/storage/modelCatalog.ts index ec90e095..c7064deb 100644 --- a/packages/cad-agent-plugin/src/storage/modelCatalog.ts +++ b/packages/cad-agent-plugin/src/storage/modelCatalog.ts @@ -70,6 +70,38 @@ export const PROVIDER_MODEL_OPTIONS: Record< supportsVision: true } ], + google: [ + { + value: 'gemini-2.5-pro', + labelEn: 'Gemini 2.5 Pro', + labelZh: 'Gemini 2.5 Pro', + supportsVision: true + }, + { + value: 'gemini-2.5-flash', + labelEn: 'Gemini 2.5 Flash', + labelZh: 'Gemini 2.5 Flash', + supportsVision: true + }, + { + value: 'gemini-2.0-flash', + labelEn: 'Gemini 2.0 Flash', + labelZh: 'Gemini 2.0 Flash', + supportsVision: true + }, + { + value: 'gemini-1.5-pro', + labelEn: 'Gemini 1.5 Pro', + labelZh: 'Gemini 1.5 Pro', + supportsVision: true + }, + { + value: 'gemini-1.5-flash', + labelEn: 'Gemini 1.5 Flash', + labelZh: 'Gemini 1.5 Flash', + supportsVision: true + } + ], deepseek: [ { value: 'deepseek-v4-flash', diff --git a/packages/cad-agent-plugin/src/ui/AgentChatPanel.vue b/packages/cad-agent-plugin/src/ui/AgentChatPanel.vue index cdef0562..6d61624a 100644 --- a/packages/cad-agent-plugin/src/ui/AgentChatPanel.vue +++ b/packages/cad-agent-plugin/src/ui/AgentChatPanel.vue @@ -461,6 +461,7 @@ function isVerificationMessage(message: UIMessage): boolean { + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cdaf6d3d..06c0db3f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -115,6 +115,9 @@ importers: '@ai-sdk/anthropic': specifier: ^2.0.0 version: 2.0.84(zod@3.25.76) + '@ai-sdk/google': + specifier: ^2.0.0 + version: 2.0.78(zod@3.25.76) '@ai-sdk/openai': specifier: ^2.0.0 version: 2.0.110(zod@3.25.76) @@ -528,6 +531,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/google@2.0.78': + resolution: {integrity: sha512-21AeBlmeXcFVzujaW6I6bWSENYAVpaEYZ767Pv4LJeuSt7F6TR3/SutbLhi6n9XBiT+F5kWOtJnt/VPLHHb7XA==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/openai@2.0.110': resolution: {integrity: sha512-PVbq0vXo6LVBCKeIOiDsOfrUIL2IOkMUipWP53SJZppDo594kbu5BFs/NdwnJMAX4q6EM6sGPVv751RN9PrZ9A==} engines: {node: '>=18'} @@ -5515,6 +5524,12 @@ snapshots: '@vercel/oidc': 3.1.0 zod: 3.25.76 + '@ai-sdk/google@2.0.78(zod@3.25.76)': + dependencies: + '@ai-sdk/provider': 2.0.3 + '@ai-sdk/provider-utils': 3.0.28(zod@3.25.76) + zod: 3.25.76 + '@ai-sdk/openai@2.0.110(zod@3.25.76)': dependencies: '@ai-sdk/provider': 2.0.3