diff --git a/src/main/index.ts b/src/main/index.ts index d5333b5..9659da5 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -181,8 +181,8 @@ app.whenReady().then(() => { console.log('[FRONTEND LOG]', ...args) }) - ipcMain.handle('start-pipeline', async (event, { threadId, newAiMessageId, isThinkingMode }) => { - console.log(`\n===============\n[DEBUG IPC] start-pipeline called: threadId=${threadId}, newAiMessageId=${newAiMessageId}, isThinkingMode=${isThinkingMode}`) + ipcMain.handle('start-pipeline', async (event, { threadId, newAiMessageId, isThinkingMode, autoUseImages }) => { + console.log(`\n===============\n[DEBUG IPC] start-pipeline called: threadId=${threadId}, newAiMessageId=${newAiMessageId}, isThinkingMode=${isThinkingMode}, autoUseImages=${autoUseImages}`) const window = BrowserWindow.fromWebContents(event.sender) if (!window) { console.log(`[DEBUG IPC] FAILED: window not found`) @@ -206,7 +206,7 @@ app.whenReady().then(() => { // Prepare the base timeline const baseTimeline = userMsgId ? thread.messages.find(m => m.id === userMsgId)?.timeline : undefined; - const pipeline = new Pipeline(window, newAiMessageId, threadId, context, baseTimeline, userMsgId, attachedImages, isThinkingMode) + const pipeline = new Pipeline(window, newAiMessageId, threadId, context, baseTimeline, userMsgId, attachedImages, isThinkingMode, autoUseImages) // Ensure background processing is running (will resume/retry if tasks are missing/failed) if (thread.type === 'image') { diff --git a/src/main/pipeline/index.ts b/src/main/pipeline/index.ts index 62bc9d9..167c62e 100644 --- a/src/main/pipeline/index.ts +++ b/src/main/pipeline/index.ts @@ -25,6 +25,7 @@ export interface PipelineContext { intentResult?: import('../../shared/types').IntentResult; attachedImages?: string[]; isThinkingMode?: boolean; + autoUseImages?: boolean; } import { backgroundTaskManager } from '../tasks' @@ -41,10 +42,11 @@ export class Pipeline { private intentResult?: import('../../shared/types').IntentResult private attachedImages?: string[] private isThinkingMode: boolean = false + private autoUseImages: boolean = false private isFinished: boolean = false private abortController: AbortController = new AbortController() - constructor(browserWindow: BrowserWindow, messageId: string, threadId: string, context: string, baseTimeline?: EnrichedTimelineSegment[], editRefId?: string, attachedImages?: string[], isThinkingMode: boolean = false) { + constructor(browserWindow: BrowserWindow, messageId: string, threadId: string, context: string, baseTimeline?: EnrichedTimelineSegment[], editRefId?: string, attachedImages?: string[], isThinkingMode: boolean = false, autoUseImages: boolean = false) { this.browserWindow = browserWindow this.messageId = messageId this.threadId = threadId @@ -53,6 +55,7 @@ export class Pipeline { this.editRefId = editRefId this.attachedImages = attachedImages this.isThinkingMode = isThinkingMode + this.autoUseImages = autoUseImages } register(fn: PipelineFunction, options?: { skipIf?: (context: PipelineContext) => boolean }): this { @@ -150,6 +153,7 @@ export class Pipeline { baseTimeline: this.baseTimeline, attachedImages: this.attachedImages, isThinkingMode: this.isThinkingMode, + autoUseImages: this.autoUseImages, get intentResult() { return self.intentResult }, set intentResult(val) { self.intentResult = val }, signal: this.abortController.signal, diff --git a/src/main/pipeline/phases/image-intent.ts b/src/main/pipeline/phases/image-intent.ts index 95be278..dcbcc54 100644 --- a/src/main/pipeline/phases/image-intent.ts +++ b/src/main/pipeline/phases/image-intent.ts @@ -32,6 +32,7 @@ Rules (STRICT ENFORCEMENT): b) Create a DETAILED technical prompt for a creative image generator (like Gemini Image 3). c) This prompt should specify style, composition, lighting, and how to merge the elements from the selected images AND any provided ATTACHED IMAGES. d) PERSON NAMES: DO NOT mention specific real-world names (e.g., 'Olga Loiek') in the 'content' field. Instead, refer to them using generic descriptors based on the images, such as 'the speaker', 'the subject', 'the person in the video', or 'the main figure'. This is to avoid triggering safety/privacy filters. You can refer to 'Image X' or 'Image index Y' to point to specific people. + e) IMAGE ACCESS RESTRICTION: If the system note indicates "IMAGE ACCESS DISABLED", you CANNOT return 'selectedIndices' or trigger visual generation actions that require specific project images. In this case, use 'type': 'text' and ask the user to enable 'Smart Auto-References' (the blade icon). Respond ONLY with a JSON object following this schema: { @@ -82,6 +83,10 @@ ${context.context} User Prompt: ${lastUserPrompt} +${context.autoUseImages || (context.attachedImages && context.attachedImages.length > 0) + ? "[SYSTEM NOTE]: IMAGE ACCESS IS ENABLED. You can select project images as references." + : "[SYSTEM NOTE]: IMAGE ACCESS IS DISABLED. You can see visual descriptions but cannot access actual image files. To perform visual tasks, ask the user to enable 'Smart Auto-References'."} + (Note: If the user provided any attached images, they are passed as visual context to you.) ` console.log(`[IMAGE-INTENT] Sending prompt to AI. User prompt detected as: "${lastUserPrompt}"`) diff --git a/src/main/pipeline/phases/intent.ts b/src/main/pipeline/phases/intent.ts index 28b52ce..1a8cdad 100644 --- a/src/main/pipeline/phases/intent.ts +++ b/src/main/pipeline/phases/intent.ts @@ -59,7 +59,8 @@ Specific rules for 'content' field: - If type is 'generate-timeline': This is a COMPREHENSIVE and DETAILED technical description for the timeline builder agent. - If type is 'generate-thumbnail': This is a COMPREHENSIVE and DETAILED technical description for the thumbnail generator. Include visual elements, frames to extract (YOU MUST list specific timestamps in [HH:MM:SS] format for at least 2-3 relevant scenes), and any overlay text. - CRITICAL: When editing an existing timeline or thumbnail, specify EXACTLY which parts to keep, remove, or replace. The goal is maximum consistency with the REFERENCE TIMELINE except for the requested changes. It will NOT be shown to the user. 'content' for 'generate-thumbnail' MUST be technical and precise. -- PERSON NAMES: DO NOT mention specific real-world names (e.g., 'Olga Loiek') in the 'content' field for 'generate-thumbnail'. Instead, refer to them using generic descriptors based on the reference frames, such as 'the speaker', 'the subject', 'the person in the video', or 'the main figure'. This is to avoid triggering safety/privacy filters. You can refer to 'Scene X' or 'Image Y' to point to specific people.` +- PERSON NAMES: DO NOT mention specific real-world names (e.g., 'Olga Loiek') in the 'content' field for 'generate-thumbnail'. Instead, refer to them using generic descriptors based on the reference frames, such as 'the speaker', 'the subject', 'the person in the video', or 'the main figure'. This is to avoid triggering safety/privacy filters. You can refer to 'Scene X' or 'Image Y' to point to specific people. +- IMAGE ACCESS RESTRICTION: If the system note indicates "IMAGE ACCESS DISABLED", you CANNOT return 'selectedIndices' or trigger visual generation actions that require specific project frames. In this case, use 'type': 'text' and ask the user to enable 'Smart Auto-References' (the blade icon).` const INTENT_SCHEMA = { type: 'object', @@ -116,6 +117,10 @@ ${baseTimelineContext} Conversation History: ${context.context} + +${context.autoUseImages || (context.attachedImages && context.attachedImages.length > 0) + ? "[SYSTEM NOTE]: IMAGE ACCESS IS ENABLED. You can use project images as references and return 'selectedIndices'." + : "[SYSTEM NOTE]: IMAGE ACCESS IS DISABLED. You can see visual descriptions but cannot access actual image files. To perform visual tasks, ask the user to enable 'Smart Auto-References'."} ` // 5. Limit and Deduplicate Images (Base64 is expensive, limit to last 8) diff --git a/src/main/pipeline/phases/supply.ts b/src/main/pipeline/phases/supply.ts index dc432b9..ce499dc 100644 --- a/src/main/pipeline/phases/supply.ts +++ b/src/main/pipeline/phases/supply.ts @@ -4,8 +4,10 @@ import fs from 'fs' export const supplyController: PipelineFunction = async (data, context) => { context.updateStatus('Managing reference images...') + // Determine if we can use project images + const isImageAccessEnabled = context.autoUseImages || (context.attachedImages && context.attachedImages.length > 0) + // Case 1: User provided attachments - // Rule: If user provide frames, then none for auto select. if (context.attachedImages && context.attachedImages.length > 0) { console.log(`[SUPPLY] User provided ${context.attachedImages.length} attachments. Skipping auto-selection from video.`) const cleanedPaths = context.attachedImages.map(url => url.replace(/^media:\/+/i, '/')) @@ -13,6 +15,13 @@ export const supplyController: PipelineFunction = async (data, context) => { return } + // Case 2: No attachments, handle auto-selection ONLY if image access is enabled + if (!isImageAccessEnabled) { + console.log('[SUPPLY] Image access disabled and no attachments. Providing empty image list.') + context.next({ ...data, selectedReferenceImages: [] }) + return + } + // Case 2: No attachments, handle auto-selection from video frames or source images const referenceFrames = context.preprocessing?.['reference-frames'] || [] const sourceImages = context.preprocessing?.['sourceImages'] || [] diff --git a/src/renderer/src/components/chat/BaseMessageInput.vue b/src/renderer/src/components/chat/BaseMessageInput.vue index ffc6bdc..fda1516 100644 --- a/src/renderer/src/components/chat/BaseMessageInput.vue +++ b/src/renderer/src/components/chat/BaseMessageInput.vue @@ -21,17 +21,12 @@ -
- - - - - +
- -
- Count - -
+ +
+
+ + + + + + + + + +
- - - - - - - - +
+ + +
+ Samples + +
+ + + + + + + + + + +
+
@@ -86,14 +112,16 @@ const props = withDefaults(defineProps<{ attachedImages?: string[] compact?: boolean submitIcon?: string + autoUseImages?: boolean }>(), { placeholder: 'Type a message...', attachedImages: () => [], compact: false, - submitIcon: 'IconSend' + submitIcon: 'IconSend', + autoUseImages: false }) -const emit = defineEmits(['update:modelValue', 'update:attachedImages', 'send']) +const emit = defineEmits(['update:modelValue', 'update:attachedImages', 'send', 'update:autoUseImages']) const internalText = ref(props.modelValue) const resultsCount = ref(1) @@ -115,6 +143,10 @@ const normalizeUrl = (url: string) => { return url.startsWith('media://') ? url : `media://${url}` } +const toggleAutoUseImages = () => { + emit('update:autoUseImages', !props.autoUseImages) +} + const adjustTextarea = () => { if (textareaRef.value) { textareaRef.value.style.height = 'auto' @@ -142,7 +174,7 @@ const removeAttachment = (index: number) => { const handleSend = () => { if (!internalText.value.trim() && props.attachedImages.length === 0) return - emit('send', internalText.value, [...props.attachedImages], resultsCount.value, isThinkingMode.value) + emit('send', internalText.value, [...props.attachedImages], resultsCount.value, isThinkingMode.value, props.autoUseImages) // Clear local state if parent doesn't reset via props (standard pattern) internalText.value = '' diff --git a/src/renderer/src/components/chat/ChatInput.vue b/src/renderer/src/components/chat/ChatInput.vue index 81026af..48ab2a1 100644 --- a/src/renderer/src/components/chat/ChatInput.vue +++ b/src/renderer/src/components/chat/ChatInput.vue @@ -53,8 +53,8 @@ const branchedVersion = computed(() => { return `v.${version}${type}` }) -const handleSend = (text: string, images: string[], count: number, isThinkingMode: boolean) => { - emit('send', text, images, count, isThinkingMode) +const handleSend = (text: string, images: string[], count: number, isThinkingMode: boolean, autoUseImages: boolean) => { + emit('send', text, images, count, isThinkingMode, autoUseImages) userPrompt.value = '' attachedImages.value = [] } diff --git a/src/renderer/src/components/chat/ChatMessage.vue b/src/renderer/src/components/chat/ChatMessage.vue index cfe67dd..19251ad 100644 --- a/src/renderer/src/components/chat/ChatMessage.vue +++ b/src/renderer/src/components/chat/ChatMessage.vue @@ -198,8 +198,8 @@ const handleRetry = async () => { isRetryModalOpen.value = true } -const handleRetryConfirmed = async (messageId: string, shouldRemoveBranch: boolean, count: number, isThinkingMode: boolean) => { - await videoStore.retryMessage(messageId, shouldRemoveBranch, count, isThinkingMode) +const handleRetryConfirmed = async (messageId: string, shouldRemoveBranch: boolean, count: number, isThinkingMode: boolean, autoUseImages: boolean) => { + await videoStore.retryMessage(messageId, shouldRemoveBranch, count, isThinkingMode, autoUseImages) emit('retry', messageId) } diff --git a/src/renderer/src/components/chat/RetryModal.vue b/src/renderer/src/components/chat/RetryModal.vue index 5f85da7..a11e6a0 100644 --- a/src/renderer/src/components/chat/RetryModal.vue +++ b/src/renderer/src/components/chat/RetryModal.vue @@ -49,7 +49,8 @@
- Experimental + Experimental Thinking Mode Activate Chain of Thought (CoT)
@@ -60,6 +61,29 @@
+ + + + +
+
+
+ Smart Auto-References + + + +
+ Automatically use project images as references +
+ +
@@ -80,9 +104,10 @@