From 194af7b9d6dca6b127efd23832b39602db725da2 Mon Sep 17 00:00:00 2001 From: Navid Shad Date: Wed, 15 Apr 2026 16:46:58 +0300 Subject: [PATCH 1/3] feat: add auto-use images toggle to chat interface and update submission handlers across node components --- .../src/components/chat/BaseMessageInput.vue | 102 +++++++++++------- .../src/components/chat/ChatInput.vue | 4 +- .../src/components/graph/ChatInputNode.vue | 4 +- .../src/components/graph/ConversationNode.vue | 4 +- .../components/graph/ImageCollectionNode.vue | 4 +- .../src/components/graph/MediaNode.vue | 4 +- .../src/components/graph/SummaryNode.vue | 4 +- .../src/components/graph/ThumbnailNode.vue | 4 +- .../src/components/graph/VideoNode.vue | 4 +- 9 files changed, 80 insertions(+), 54 deletions(-) diff --git a/src/renderer/src/components/chat/BaseMessageInput.vue b/src/renderer/src/components/chat/BaseMessageInput.vue index ffc6bdc..330554c 100644 --- a/src/renderer/src/components/chat/BaseMessageInput.vue +++ b/src/renderer/src/components/chat/BaseMessageInput.vue @@ -21,17 +21,12 @@ -
- - - - - +
- -
- Count - -
+ +
+
+ + + + + + + + + +
- - - - - - - - +
+ + +
+ Samples + +
+ + + + + + + + + + +
+
@@ -98,6 +123,7 @@ const emit = defineEmits(['update:modelValue', 'update:attachedImages', 'send']) const internalText = ref(props.modelValue) const resultsCount = ref(1) const isThinkingMode = ref(false) +const autoUseImages = ref(false) const textareaRef = ref(null) const showAttachmentModal = ref(false) @@ -142,7 +168,7 @@ const removeAttachment = (index: number) => { const handleSend = () => { if (!internalText.value.trim() && props.attachedImages.length === 0) return - emit('send', internalText.value, [...props.attachedImages], resultsCount.value, isThinkingMode.value) + emit('send', internalText.value, [...props.attachedImages], resultsCount.value, isThinkingMode.value, autoUseImages.value) // Clear local state if parent doesn't reset via props (standard pattern) internalText.value = '' diff --git a/src/renderer/src/components/chat/ChatInput.vue b/src/renderer/src/components/chat/ChatInput.vue index 81026af..48ab2a1 100644 --- a/src/renderer/src/components/chat/ChatInput.vue +++ b/src/renderer/src/components/chat/ChatInput.vue @@ -53,8 +53,8 @@ const branchedVersion = computed(() => { return `v.${version}${type}` }) -const handleSend = (text: string, images: string[], count: number, isThinkingMode: boolean) => { - emit('send', text, images, count, isThinkingMode) +const handleSend = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { + emit('send', text, images, count, isThinkingMode, autoUseImages) userPrompt.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/graph/ChatInputNode.vue b/src/renderer/src/components/graph/ChatInputNode.vue index f99aedb..75ba7b7 100644 --- a/src/renderer/src/components/graph/ChatInputNode.vue +++ b/src/renderer/src/components/graph/ChatInputNode.vue @@ -22,9 +22,9 @@ const props = defineProps<{ data: any }>() const input = ref('') const attachedImages = ref([]) -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/graph/ConversationNode.vue b/src/renderer/src/components/graph/ConversationNode.vue index 29749e0..0754af0 100644 --- a/src/renderer/src/components/graph/ConversationNode.vue +++ b/src/renderer/src/components/graph/ConversationNode.vue @@ -434,9 +434,9 @@ const copyMessage = (id: string, content: string) => { }) } -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] showInput.value = false diff --git a/src/renderer/src/components/graph/ImageCollectionNode.vue b/src/renderer/src/components/graph/ImageCollectionNode.vue index 93a7db8..51e037a 100644 --- a/src/renderer/src/components/graph/ImageCollectionNode.vue +++ b/src/renderer/src/components/graph/ImageCollectionNode.vue @@ -110,9 +110,9 @@ const mediaUrl = (url: string) => { return url.startsWith('media://') ? url : `media://${url}` } -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/graph/MediaNode.vue b/src/renderer/src/components/graph/MediaNode.vue index 7a2143c..c4db5e3 100644 --- a/src/renderer/src/components/graph/MediaNode.vue +++ b/src/renderer/src/components/graph/MediaNode.vue @@ -173,9 +173,9 @@ const mediaUrl = (url: string) => { return url.startsWith('media://') ? url : `media://${url}` } -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/graph/SummaryNode.vue b/src/renderer/src/components/graph/SummaryNode.vue index 17db3ff..59963eb 100644 --- a/src/renderer/src/components/graph/SummaryNode.vue +++ b/src/renderer/src/components/graph/SummaryNode.vue @@ -45,9 +45,9 @@ const props = defineProps<{ data: any }>() const input = ref('') const attachedImages = ref([]) -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/graph/ThumbnailNode.vue b/src/renderer/src/components/graph/ThumbnailNode.vue index 269f4ff..1b11433 100644 --- a/src/renderer/src/components/graph/ThumbnailNode.vue +++ b/src/renderer/src/components/graph/ThumbnailNode.vue @@ -384,9 +384,9 @@ const handleUpscale = async (factor: string) => { } } -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/graph/VideoNode.vue b/src/renderer/src/components/graph/VideoNode.vue index 094002d..7c8547b 100644 --- a/src/renderer/src/components/graph/VideoNode.vue +++ b/src/renderer/src/components/graph/VideoNode.vue @@ -242,9 +242,9 @@ const handleSave = async () => { } } -const submit = (text: string, images: string[], count: number, isThinkingMode: boolean) => { +const submit = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { if ((text.trim() || images.length > 0) && props.data.onSubmit) { - props.data.onSubmit(text, images, count, isThinkingMode) + props.data.onSubmit(text, images, count, isThinkingMode, autoUseImages) input.value = '' attachedImages.value = [] } From 54b01f2ed63a40625a6bb65bb47b93de82361e7d Mon Sep 17 00:00:00 2001 From: Navid Shad Date: Wed, 15 Apr 2026 18:10:12 +0300 Subject: [PATCH 2/3] feat: integrate auto-reference image toggle into chat input, retry modal, and pipeline execution flow --- src/main/index.ts | 6 +-- src/main/pipeline/index.ts | 6 ++- src/main/pipeline/phases/image-intent.ts | 5 +++ src/main/pipeline/phases/intent.ts | 7 ++- src/main/pipeline/phases/supply.ts | 11 ++++- .../src/components/chat/BaseMessageInput.vue | 22 ++++++---- .../src/components/chat/ChatMessage.vue | 4 +- .../src/components/chat/RetryModal.vue | 43 +++++++++++++++++-- .../src/components/graph/ChatInputNode.vue | 2 + .../src/components/graph/ConversationNode.vue | 14 +++++- .../src/components/graph/MediaNode.vue | 2 + .../src/components/graph/ThumbnailNode.vue | 11 +++++ .../src/components/graph/VideoNode.vue | 11 +++++ .../src/composables/useGraphLayout.ts | 18 ++++---- src/renderer/src/stores/videoStore.ts | 24 +++++++---- src/shared/types.ts | 1 + 16 files changed, 149 insertions(+), 38 deletions(-) diff --git a/src/main/index.ts b/src/main/index.ts index d5333b5..9659da5 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -181,8 +181,8 @@ app.whenReady().then(() => { console.log('[FRONTEND LOG]', ...args) }) - ipcMain.handle('start-pipeline', async (event, { threadId, newAiMessageId, isThinkingMode }) => { - console.log(`\n===============\n[DEBUG IPC] start-pipeline called: threadId=${threadId}, newAiMessageId=${newAiMessageId}, isThinkingMode=${isThinkingMode}`) + ipcMain.handle('start-pipeline', async (event, { threadId, newAiMessageId, isThinkingMode, autoUseImages }) => { + console.log(`\n===============\n[DEBUG IPC] start-pipeline called: threadId=${threadId}, newAiMessageId=${newAiMessageId}, isThinkingMode=${isThinkingMode}, autoUseImages=${autoUseImages}`) const window = BrowserWindow.fromWebContents(event.sender) if (!window) { console.log(`[DEBUG IPC] FAILED: window not found`) @@ -206,7 +206,7 @@ app.whenReady().then(() => { // Prepare the base timeline const baseTimeline = userMsgId ? thread.messages.find(m => m.id === userMsgId)?.timeline : undefined; - const pipeline = new Pipeline(window, newAiMessageId, threadId, context, baseTimeline, userMsgId, attachedImages, isThinkingMode) + const pipeline = new Pipeline(window, newAiMessageId, threadId, context, baseTimeline, userMsgId, attachedImages, isThinkingMode, autoUseImages) // Ensure background processing is running (will resume/retry if tasks are missing/failed) if (thread.type === 'image') { diff --git a/src/main/pipeline/index.ts b/src/main/pipeline/index.ts index 62bc9d9..167c62e 100644 --- a/src/main/pipeline/index.ts +++ b/src/main/pipeline/index.ts @@ -25,6 +25,7 @@ export interface PipelineContext { intentResult?: import('../../shared/types').IntentResult; attachedImages?: string[]; isThinkingMode?: boolean; + autoUseImages?: boolean; } import { backgroundTaskManager } from '../tasks' @@ -41,10 +42,11 @@ export class Pipeline { private intentResult?: import('../../shared/types').IntentResult private attachedImages?: string[] private isThinkingMode: boolean = false + private autoUseImages: boolean = false private isFinished: boolean = false private abortController: AbortController = new AbortController() - constructor(browserWindow: BrowserWindow, messageId: string, threadId: string, context: string, baseTimeline?: EnrichedTimelineSegment[], editRefId?: string, attachedImages?: string[], isThinkingMode: boolean = false) { + constructor(browserWindow: BrowserWindow, messageId: string, threadId: string, context: string, baseTimeline?: EnrichedTimelineSegment[], editRefId?: string, attachedImages?: string[], isThinkingMode: boolean = false, autoUseImages: boolean = false) { this.browserWindow = browserWindow this.messageId = messageId this.threadId = threadId @@ -53,6 +55,7 @@ export class Pipeline { this.editRefId = editRefId this.attachedImages = attachedImages this.isThinkingMode = isThinkingMode + this.autoUseImages = autoUseImages } register(fn: PipelineFunction, options?: { skipIf?: (context: PipelineContext) => boolean }): this { @@ -150,6 +153,7 @@ export class Pipeline { baseTimeline: this.baseTimeline, attachedImages: this.attachedImages, isThinkingMode: this.isThinkingMode, + autoUseImages: this.autoUseImages, get intentResult() { return self.intentResult }, set intentResult(val) { self.intentResult = val }, signal: this.abortController.signal, diff --git a/src/main/pipeline/phases/image-intent.ts b/src/main/pipeline/phases/image-intent.ts index 95be278..dcbcc54 100644 --- a/src/main/pipeline/phases/image-intent.ts +++ b/src/main/pipeline/phases/image-intent.ts @@ -32,6 +32,7 @@ Rules (STRICT ENFORCEMENT): b) Create a DETAILED technical prompt for a creative image generator (like Gemini Image 3). c) This prompt should specify style, composition, lighting, and how to merge the elements from the selected images AND any provided ATTACHED IMAGES. d) PERSON NAMES: DO NOT mention specific real-world names (e.g., 'Olga Loiek') in the 'content' field. Instead, refer to them using generic descriptors based on the images, such as 'the speaker', 'the subject', 'the person in the video', or 'the main figure'. This is to avoid triggering safety/privacy filters. You can refer to 'Image X' or 'Image index Y' to point to specific people. + e) IMAGE ACCESS RESTRICTION: If the system note indicates "IMAGE ACCESS DISABLED", you CANNOT return 'selectedIndices' or trigger visual generation actions that require specific project images. In this case, use 'type': 'text' and ask the user to enable 'Smart Auto-References' (the blade icon). Respond ONLY with a JSON object following this schema: { @@ -82,6 +83,10 @@ ${context.context} User Prompt: ${lastUserPrompt} +${context.autoUseImages || (context.attachedImages && context.attachedImages.length > 0) + ? "[SYSTEM NOTE]: IMAGE ACCESS IS ENABLED. You can select project images as references." + : "[SYSTEM NOTE]: IMAGE ACCESS IS DISABLED. You can see visual descriptions but cannot access actual image files. To perform visual tasks, ask the user to enable 'Smart Auto-References'."} + (Note: If the user provided any attached images, they are passed as visual context to you.) ` console.log(`[IMAGE-INTENT] Sending prompt to AI. User prompt detected as: "${lastUserPrompt}"`) diff --git a/src/main/pipeline/phases/intent.ts b/src/main/pipeline/phases/intent.ts index 28b52ce..1a8cdad 100644 --- a/src/main/pipeline/phases/intent.ts +++ b/src/main/pipeline/phases/intent.ts @@ -59,7 +59,8 @@ Specific rules for 'content' field: - If type is 'generate-timeline': This is a COMPREHENSIVE and DETAILED technical description for the timeline builder agent. - If type is 'generate-thumbnail': This is a COMPREHENSIVE and DETAILED technical description for the thumbnail generator. Include visual elements, frames to extract (YOU MUST list specific timestamps in [HH:MM:SS] format for at least 2-3 relevant scenes), and any overlay text. - CRITICAL: When editing an existing timeline or thumbnail, specify EXACTLY which parts to keep, remove, or replace. The goal is maximum consistency with the REFERENCE TIMELINE except for the requested changes. It will NOT be shown to the user. 'content' for 'generate-thumbnail' MUST be technical and precise. -- PERSON NAMES: DO NOT mention specific real-world names (e.g., 'Olga Loiek') in the 'content' field for 'generate-thumbnail'. Instead, refer to them using generic descriptors based on the reference frames, such as 'the speaker', 'the subject', 'the person in the video', or 'the main figure'. This is to avoid triggering safety/privacy filters. You can refer to 'Scene X' or 'Image Y' to point to specific people.` +- PERSON NAMES: DO NOT mention specific real-world names (e.g., 'Olga Loiek') in the 'content' field for 'generate-thumbnail'. Instead, refer to them using generic descriptors based on the reference frames, such as 'the speaker', 'the subject', 'the person in the video', or 'the main figure'. This is to avoid triggering safety/privacy filters. You can refer to 'Scene X' or 'Image Y' to point to specific people. +- IMAGE ACCESS RESTRICTION: If the system note indicates "IMAGE ACCESS DISABLED", you CANNOT return 'selectedIndices' or trigger visual generation actions that require specific project frames. In this case, use 'type': 'text' and ask the user to enable 'Smart Auto-References' (the blade icon).` const INTENT_SCHEMA = { type: 'object', @@ -116,6 +117,10 @@ ${baseTimelineContext} Conversation History: ${context.context} + +${context.autoUseImages || (context.attachedImages && context.attachedImages.length > 0) + ? "[SYSTEM NOTE]: IMAGE ACCESS IS ENABLED. You can use project images as references and return 'selectedIndices'." + : "[SYSTEM NOTE]: IMAGE ACCESS IS DISABLED. You can see visual descriptions but cannot access actual image files. To perform visual tasks, ask the user to enable 'Smart Auto-References'."} ` // 5. Limit and Deduplicate Images (Base64 is expensive, limit to last 8) diff --git a/src/main/pipeline/phases/supply.ts b/src/main/pipeline/phases/supply.ts index dc432b9..ce499dc 100644 --- a/src/main/pipeline/phases/supply.ts +++ b/src/main/pipeline/phases/supply.ts @@ -4,8 +4,10 @@ import fs from 'fs' export const supplyController: PipelineFunction = async (data, context) => { context.updateStatus('Managing reference images...') + // Determine if we can use project images + const isImageAccessEnabled = context.autoUseImages || (context.attachedImages && context.attachedImages.length > 0) + // Case 1: User provided attachments - // Rule: If user provide frames, then none for auto select. if (context.attachedImages && context.attachedImages.length > 0) { console.log(`[SUPPLY] User provided ${context.attachedImages.length} attachments. Skipping auto-selection from video.`) const cleanedPaths = context.attachedImages.map(url => url.replace(/^media:\/+/i, '/')) @@ -13,6 +15,13 @@ export const supplyController: PipelineFunction = async (data, context) => { return } + // Case 2: No attachments, handle auto-selection ONLY if image access is enabled + if (!isImageAccessEnabled) { + console.log('[SUPPLY] Image access disabled and no attachments. Providing empty image list.') + context.next({ ...data, selectedReferenceImages: [] }) + return + } + // Case 2: No attachments, handle auto-selection from video frames or source images const referenceFrames = context.preprocessing?.['reference-frames'] || [] const sourceImages = context.preprocessing?.['sourceImages'] || [] diff --git a/src/renderer/src/components/chat/BaseMessageInput.vue b/src/renderer/src/components/chat/BaseMessageInput.vue index 330554c..fda1516 100644 --- a/src/renderer/src/components/chat/BaseMessageInput.vue +++ b/src/renderer/src/components/chat/BaseMessageInput.vue @@ -44,17 +44,18 @@ - + @@ -111,19 +112,20 @@ const props = withDefaults(defineProps<{ attachedImages?: string[] compact?: boolean submitIcon?: string + autoUseImages?: boolean }>(), { placeholder: 'Type a message...', attachedImages: () => [], compact: false, - submitIcon: 'IconSend' + submitIcon: 'IconSend', + autoUseImages: false }) -const emit = defineEmits(['update:modelValue', 'update:attachedImages', 'send']) +const emit = defineEmits(['update:modelValue', 'update:attachedImages', 'send', 'update:autoUseImages']) const internalText = ref(props.modelValue) const resultsCount = ref(1) const isThinkingMode = ref(false) -const autoUseImages = ref(false) const textareaRef = ref(null) const showAttachmentModal = ref(false) @@ -141,6 +143,10 @@ const normalizeUrl = (url: string) => { return url.startsWith('media://') ? url : `media://${url}` } +const toggleAutoUseImages = () => { + emit('update:autoUseImages', !props.autoUseImages) +} + const adjustTextarea = () => { if (textareaRef.value) { textareaRef.value.style.height = 'auto' @@ -168,7 +174,7 @@ const removeAttachment = (index: number) => { const handleSend = () => { if (!internalText.value.trim() && props.attachedImages.length === 0) return - emit('send', internalText.value, [...props.attachedImages], resultsCount.value, isThinkingMode.value, autoUseImages.value) + emit('send', internalText.value, [...props.attachedImages], resultsCount.value, isThinkingMode.value, props.autoUseImages) // Clear local state if parent doesn't reset via props (standard pattern) internalText.value = '' diff --git a/src/renderer/src/components/chat/ChatMessage.vue b/src/renderer/src/components/chat/ChatMessage.vue index cfe67dd..19251ad 100644 --- a/src/renderer/src/components/chat/ChatMessage.vue +++ b/src/renderer/src/components/chat/ChatMessage.vue @@ -198,8 +198,8 @@ const handleRetry = async () => { isRetryModalOpen.value = true } -const handleRetryConfirmed = async (messageId: string, shouldRemoveBranch: boolean, count: number, isThinkingMode: boolean) => { - await videoStore.retryMessage(messageId, shouldRemoveBranch, count, isThinkingMode) +const handleRetryConfirmed = async (messageId: string, shouldRemoveBranch: boolean, count: number, isThinkingMode: boolean, autoUseImages: boolean) => { + await videoStore.retryMessage(messageId, shouldRemoveBranch, count, isThinkingMode, autoUseImages) emit('retry', messageId) } diff --git a/src/renderer/src/components/chat/RetryModal.vue b/src/renderer/src/components/chat/RetryModal.vue index 5f85da7..a11e6a0 100644 --- a/src/renderer/src/components/chat/RetryModal.vue +++ b/src/renderer/src/components/chat/RetryModal.vue @@ -49,7 +49,8 @@
- Experimental + Experimental Thinking Mode Activate Chain of Thought (CoT)
@@ -60,6 +61,29 @@
+ + + + +
+
+
+ Smart Auto-References + + + +
+ Automatically use project images as references +
+ +
@@ -80,9 +104,10 @@