Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/cad-agent-plugin/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
},
"dependencies": {
"@ai-sdk/anthropic": "^2.0.0",
"@ai-sdk/google": "^2.0.0",
"@ai-sdk/openai": "^2.0.0",
"@ai-sdk/vue": "^2.0.0",
"ai": "^5.0.0",
Expand Down
36 changes: 35 additions & 1 deletion packages/cad-agent-plugin/src/agent/createCadAgent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export function createCadAgent(settings: LlmSettings) {
model: createModelFromSettings(settings),
system: CAD_AGENT_SYSTEM_PROMPT,
tools: createCadTools(),
stopWhen: stepCountIs(10)
stopWhen: stepCountIs(20)
})
}

Expand Down Expand Up @@ -136,6 +136,13 @@ export function createAgentChatTransport(
const { userRequest, referenceImages } =
extractConversationContext(validatedMessages)
let verificationAttempts = 0
// Verifier memory: feedback from every prior round and the screenshot
// taken before the latest edits, so review is progress-aware.
const feedbackHistory: string[] = []
let previousPreviewDataUrl: string | undefined
// Consecutive rounds whose feedback is unchanged from the round
// before — a stalled loop we stop early instead of burning attempts.
let noProgressRounds = 0

while (!abortSignal?.aborted) {
workingMessages = await streamAgentRound({
Expand Down Expand Up @@ -181,6 +188,10 @@ export function createAgentChatTransport(
userRequest,
referenceImages,
preview.dataUrl,
{
previousFeedback: feedbackHistory,
previousDrawingImage: previousPreviewDataUrl
},
abortSignal
)
} catch (error) {
Expand All @@ -201,6 +212,21 @@ export function createAgentChatTransport(
break
}

// Detect a stalled loop: feedback unchanged from the prior round.
const normalize = (text: string) =>
text.trim().toLowerCase().replace(/\s+/g, ' ')
const lastFeedback = feedbackHistory[feedbackHistory.length - 1]
if (
lastFeedback &&
normalize(verification.feedback) === normalize(lastFeedback)
) {
noProgressRounds += 1
} else {
noProgressRounds = 0
}
feedbackHistory.push(verification.feedback)
previousPreviewDataUrl = preview.dataUrl

if (verificationAttempts >= MAX_VERIFICATION_ATTEMPTS) {
appendAssistantText(
write,
Expand All @@ -209,6 +235,14 @@ export function createAgentChatTransport(
break
}

if (noProgressRounds >= 2) {
appendAssistantText(
write,
`\n${agentT('verificationNoProgress')}\n\n${verification.feedback.trim()}`
)
break
}

appendAssistantText(
write,
`\n${agentT('verificationFailed')}\n${verification.feedback.trim()}\n\n${agentT('verificationContinuing')}`
Expand Down
9 changes: 9 additions & 0 deletions packages/cad-agent-plugin/src/agent/createModel.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { createAnthropic } from '@ai-sdk/anthropic'
import { createGoogleGenerativeAI } from '@ai-sdk/google'
import { createOpenAI } from '@ai-sdk/openai'
import type { LanguageModel } from 'ai'

Expand Down Expand Up @@ -28,6 +29,14 @@ export function createModelFromSettings(settings: LlmSettings): LanguageModel {
return anthropic(settings.model)
}

if (settings.provider === 'google') {
const google = createGoogleGenerativeAI({
apiKey: settings.apiKey,
baseURL: settings.baseUrl || undefined
})
return google(settings.model)
}

if (settings.provider === 'openai') {
const openai = createOpenAI({
apiKey: settings.apiKey,
Expand Down
57 changes: 49 additions & 8 deletions packages/cad-agent-plugin/src/agent/drawingVerifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type { LlmSettings } from '../storage/LlmSettingsStore'
import { createModelFromSettings } from './createModel'

/** Maximum number of screenshot verification rounds per user message. */
export const MAX_VERIFICATION_ATTEMPTS = 5
export const MAX_VERIFICATION_ATTEMPTS = 8

const verificationSchema = z.object({
passed: z
Expand All @@ -25,27 +25,49 @@ const VERIFICATION_SYSTEM_PROMPT = `You verify CAD drawings produced by an autom
Compare the current drawing screenshot against the user's request and any reference images.
Focus on overall geometry, topology, proportions, and layout — not exact pixel alignment.

Rules:
- Ignore dimensions, dimension lines, and dimension text in reference images. The drawing tools cannot create dimensions, so do not require them in the output drawing.
- Ignore missing annotations or labels unless the user explicitly asked for text entities.
- Pass when the drawing reasonably captures the requested shape and layout.
- Fail only when important geometry is missing, wrong, or clearly misaligned.`
Tool limitations — NEVER fail a drawing for any of these, they are impossible with the available tools:
- The tools can only draw solid, continuous lines. They cannot produce dashed, dotted, or "hidden" line styles. Treat every edge as solid; do not ask for hidden or dashed lines.
- The tools cannot create dimensions, dimension lines, extension lines, arrows, or tolerance/GD&T symbols. Ignore these entirely.
- Ignore missing annotations, labels, hatching styles, or line weights unless the user explicitly asked for text entities.

Memory and progress:
- You may be shown the issues you reported in previous rounds and the previous screenshot (before the latest edits). Use them to judge PROGRESS.
- Confirm whether each previously reported issue is now resolved. Do NOT re-raise an issue that has already been fixed.
- Do NOT introduce new nitpicks each round to keep failing the drawing. Only report issues that genuinely remain and are fixable with the tools above.
- If the only remaining differences are unfixable (line style, dimensions, cosmetic) or the drawing already captures the requested geometry and layout, set passed = true.

Verdict:
- Pass when the drawing reasonably captures the requested shape, features, and layout.
- Fail only when important geometry is missing, wrong, or clearly misaligned AND it can be fixed with solid-line drawing tools.
- In feedback, list only the concrete remaining issues and the specific fix for each. Keep it short.`

export type DrawingVerificationResult = z.infer<typeof verificationSchema>

/**
* Prior-round context that gives the verifier memory across verification rounds.
*/
export interface VerificationHistory {
/** Feedback strings reported in previous rounds, oldest first. */
previousFeedback: string[]
/** Screenshot data URL from before the latest round of edits, if available. */
previousDrawingImage?: string
}

/**
* Sends the current drawing screenshot to the LLM for visual verification.
*
* @param settings - LLM provider configuration (must support vision).
* @param userRequest - Original user drawing request text.
* @param referenceImages - User-attached reference image data URLs, if any.
* @param drawingImage - PNG data URL of the current drawing.
* @param history - Previous rounds' feedback and screenshot for progress-aware review.
*/
export async function verifyDrawing(
settings: LlmSettings,
userRequest: string,
referenceImages: string[],
drawingImage: string,
history?: VerificationHistory,
abortSignal?: AbortSignal
): Promise<DrawingVerificationResult> {
const model = createModelFromSettings(settings)
Expand All @@ -68,6 +90,25 @@ export async function verifyDrawing(
}
}

if (history && history.previousFeedback.length > 0) {
userContent.push({
type: 'text',
text:
'Issues you reported in previous rounds (oldest first). Check whether each is now resolved and do not re-raise fixed or unfixable ones:\n' +
history.previousFeedback
.map((f, i) => `Round ${i + 1}: ${f.trim()}`)
.join('\n')
})
}

if (history?.previousDrawingImage) {
userContent.push({
type: 'text',
text: 'Previous drawing screenshot (before the latest edits), for comparing progress:'
})
userContent.push({ type: 'image', image: history.previousDrawingImage })
}

userContent.push({
type: 'text',
text: 'Current drawing screenshot to verify:'
Expand Down Expand Up @@ -97,9 +138,9 @@ export function buildVerificationFeedbackMessage(
feedback: string
): string {
return [
`Drawing verification failed (attempt ${attempt}/${maxAttempts}).`,
`Drawing verification (attempt ${attempt}/${maxAttempts}) found remaining issues:`,
feedback.trim(),
'Please update the drawing to address these issues.'
'Fix ONLY these issues. Do not delete or redraw geometry that is already correct — modify or add just what is needed, using the entityIds you recorded. The tools draw solid continuous lines only, so do not attempt hidden, dashed, or dimension lines.'
]
.filter(Boolean)
.join('\n\n')
Expand Down
1 change: 1 addition & 0 deletions packages/cad-agent-plugin/src/i18n/en.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export const agentEn = {
'Uses an OpenAI-compatible vision endpoint (default: SiliconFlow). For self-hosted vLLM, set Base URL to your server, e.g. http://127.0.0.1:8000/v1.',
providerOpenai: 'OpenAI',
providerAnthropic: 'Anthropic',
providerGoogle: 'Google Gemini',
providerOpenaiCompatible: 'OpenAI Compatible',
baseUrl: 'Base URL',
model: 'Model',
Expand Down
1 change: 1 addition & 0 deletions packages/cad-agent-plugin/src/i18n/tr.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export const agentTr = {
'OpenAI uyumlu bir görsel uç nokta kullanır (varsayılan: SiliconFlow). Kendi barındırdığınız vLLM için Taban URL alanını sunucunuza ayarlayın, ör. http://127.0.0.1:8000/v1.',
providerOpenai: 'OpenAI',
providerAnthropic: 'Anthropic',
providerGoogle: 'Google Gemini',
providerOpenaiCompatible: 'OpenAI Uyumlu',
baseUrl: 'Taban URL',
model: 'Model',
Expand Down
1 change: 1 addition & 0 deletions packages/cad-agent-plugin/src/i18n/zh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export const agentZh = {
'使用 OpenAI 兼容的视觉接口(默认:SiliconFlow)。自托管 vLLM 时请将接口地址改为本地服务,例如 http://127.0.0.1:8000/v1。',
providerOpenai: 'OpenAI',
providerAnthropic: 'Anthropic',
providerGoogle: 'Google Gemini',
providerOpenaiCompatible: 'OpenAI 兼容',
baseUrl: '接口地址',
model: '模型',
Expand Down
5 changes: 5 additions & 0 deletions packages/cad-agent-plugin/src/storage/LlmSettingsStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { decryptApiKey, encryptApiKey } from './apiKeyCrypto'
export type LlmProviderId =
| 'openai'
| 'anthropic'
| 'google'
| 'openai-compatible'
| 'deepseek'
| 'deepseek-vl'
Expand Down Expand Up @@ -51,6 +52,10 @@ export const PROVIDER_DEFAULTS: Record<
baseUrl: 'https://api.anthropic.com/v1',
model: 'claude-3-5-haiku-latest'
},
google: {
baseUrl: 'https://generativelanguage.googleapis.com/v1beta',
model: 'gemini-2.5-flash'
},
'openai-compatible': {
baseUrl: 'https://api.openai.com/v1',
model: 'gpt-4o-mini'
Expand Down
32 changes: 32 additions & 0 deletions packages/cad-agent-plugin/src/storage/modelCatalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,38 @@ export const PROVIDER_MODEL_OPTIONS: Record<
supportsVision: true
}
],
google: [
{
value: 'gemini-2.5-pro',
labelEn: 'Gemini 2.5 Pro',
labelZh: 'Gemini 2.5 Pro',
supportsVision: true
},
{
value: 'gemini-2.5-flash',
labelEn: 'Gemini 2.5 Flash',
labelZh: 'Gemini 2.5 Flash',
supportsVision: true
},
{
value: 'gemini-2.0-flash',
labelEn: 'Gemini 2.0 Flash',
labelZh: 'Gemini 2.0 Flash',
supportsVision: true
},
{
value: 'gemini-1.5-pro',
labelEn: 'Gemini 1.5 Pro',
labelZh: 'Gemini 1.5 Pro',
supportsVision: true
},
{
value: 'gemini-1.5-flash',
labelEn: 'Gemini 1.5 Flash',
labelZh: 'Gemini 1.5 Flash',
supportsVision: true
}
],
deepseek: [
{
value: 'deepseek-v4-flash',
Expand Down
1 change: 1 addition & 0 deletions packages/cad-agent-plugin/src/ui/AgentChatPanel.vue
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,7 @@ function isVerificationMessage(message: UIMessage): boolean {
<option value="deepseek-vl">{{ labels.providerDeepseekVl }}</option>
<option value="openai">{{ labels.providerOpenai }}</option>
<option value="anthropic">{{ labels.providerAnthropic }}</option>
<option value="google">{{ labels.providerGoogle }}</option>
<option value="openai-compatible">
{{ labels.providerOpenaiCompatible }}
</option>
Expand Down
15 changes: 15 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.