From b9fa93f797a8acd16e89b2edcd05b0f16060fb4a Mon Sep 17 00:00:00 2001 From: Navid Shad Date: Sat, 18 Jul 2026 16:42:23 +0300 Subject: [PATCH 01/41] feat: add timeline video editor (M0 scaffolding + M1 media pipeline) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the first two milestones of the timeline video editor PRD (video-editor-prd.md): M0 — Scaffolding - Thread.type 'editor' + EditorDocument data model in shared/types.ts (MediaAsset, Clip, Track, TimelineItem incl. speed/retime, persona & TimelineDiff stubs) - ThreadManager.createEditorThread (no chat preprocessing auto-start) and updateThreadWith queued mutator for race-free concurrent doc writes - /editor/:id route, Home "Video Editor" card, type-based thread routing - Full-bleed 4-zone editor shell (media / preview / inspector / timeline) with editorStore (ownership-split autosave vs thread-updated echoes) M1 — Media import + per-asset preprocessing + selectable pieces - src/main/editor: per-asset orchestrator (proxy -> scenes -> clips -> progressive thumbnails), K=3 concurrency cap, per-asset abort, task ids namespaced `${assetId}:step`, opt-in Gemini scene descriptions reusing extraction.generateSceneDescription via an asset-scoped context - IPC: create-editor-project, save-editor-doc, add-media-asset, import-media-url (per-asset progress), remove-media-asset, preprocess-media - Media panel with live per-step progress, error isolation + retry, interrupted-run resume; clip tray with selectable scene pieces; preview monitor with clip in/out playback; inspector - Editor-aware repairThreadPaths; scenedetect threshold param Co-Authored-By: Claude Opus 4.8 --- src/main/constants/paths.ts | 15 +- src/main/editor/assets.ts | 157 ++++ src/main/editor/preprocess.ts | 428 +++++++++ src/main/index.ts | 133 +++ src/main/scenedetect/index.ts | 10 +- src/main/threads/index.ts | 93 +- src/preload/index.ts | 17 + src/renderer/src/editor/VideoEditorPage.vue | 75 ++ .../src/editor/components/AssetRow.vue | 152 +++ .../src/editor/components/ClipTile.vue | 64 ++ .../src/editor/components/ClipTray.vue | 47 + .../editor/components/ImportMediaModal.vue | 136 +++ .../src/editor/components/InspectorPanel.vue | 120 +++ .../src/editor/components/MediaPanel.vue | 99 ++ .../src/editor/components/PreviewMonitor.vue | 121 +++ .../src/editor/components/PromptBar.vue | 20 + .../src/editor/components/TimelinePanel.vue | 56 ++ src/renderer/src/pages/HomePage.vue | 28 +- src/renderer/src/router.ts | 6 + src/renderer/src/stores/editorStore.ts | 373 ++++++++ src/shared/types.ts | 153 ++- tailwind.config.js | 5 + video-editor-prd.md | 888 ++++++++++++++++++ 23 files changed, 3183 insertions(+), 13 deletions(-) create mode 100644 src/main/editor/assets.ts create mode 100644 src/main/editor/preprocess.ts create mode 100644 src/renderer/src/editor/VideoEditorPage.vue create mode 100644 src/renderer/src/editor/components/AssetRow.vue create mode 100644 src/renderer/src/editor/components/ClipTile.vue create mode 100644 src/renderer/src/editor/components/ClipTray.vue create mode 100644 src/renderer/src/editor/components/ImportMediaModal.vue create mode 100644 src/renderer/src/editor/components/InspectorPanel.vue create mode 100644 src/renderer/src/editor/components/MediaPanel.vue create mode 100644 src/renderer/src/editor/components/PreviewMonitor.vue create mode 100644 src/renderer/src/editor/components/PromptBar.vue create mode 100644 src/renderer/src/editor/components/TimelinePanel.vue create mode 100644 src/renderer/src/stores/editorStore.ts create mode 100644 video-editor-prd.md diff --git a/src/main/constants/paths.ts b/src/main/constants/paths.ts index 0122318..80114cd 100644 --- a/src/main/constants/paths.ts +++ b/src/main/constants/paths.ts @@ -6,5 +6,18 @@ export const THREAD_DIRS = { GENERATED_VIDEOS: 'generated-videos', AUDIO: 'audio', VIDEO: 'video', - TRANSCRIPTS: 'transcripts' + TRANSCRIPTS: 'transcripts', + // Timeline editor: per-asset artifact root (tempDir/media//...) + MEDIA: 'media' +} as const + +// Subdirectories inside tempDir/media// for the timeline editor. +// FRAMES/ANALYSIS/AUDIO intentionally reuse THREAD_DIRS names so pipeline +// phases run per-asset when given an asset-scoped tempDir. +export const ASSET_DIRS = { + SOURCE: 'source', + PROXY: 'proxy', + ANALYSIS: THREAD_DIRS.ANALYSIS, + FRAMES: THREAD_DIRS.FRAMES, + AUDIO: THREAD_DIRS.AUDIO } as const diff --git a/src/main/editor/assets.ts b/src/main/editor/assets.ts new file mode 100644 index 0000000..ce06c05 --- /dev/null +++ b/src/main/editor/assets.ts @@ -0,0 +1,157 @@ +import fs from 'fs' +import path from 'path' +import { v4 as uuidv4 } from 'uuid' +import type { MediaAsset, Thread } from '@shared/types' +import { threadManager } from '../threads' +import { getVideoMetadata, sanitizeFilename } from '../ffmpeg' +import { ASSET_DIRS, THREAD_DIRS } from '../constants/paths' +import { abortAssetPreprocessing } from './preprocess' + +/** + * Media-asset CRUD for the timeline editor. + * Every artifact of an asset lives under tempDir/media// so + * concurrent imports can never collide and removal is one rm -rf. + * All document writes go through threadManager.updateThreadWith (queued + * mutators) so parallel per-asset updates cannot clobber each other. + */ + +export function getAssetDir(thread: Thread, assetId: string): string { + return path.join(thread.tempDir, THREAD_DIRS.MEDIA, assetId) +} + +function ensureDir(dir: string) { + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) +} + +/** + * Imports a source file as a MediaAsset: copies (or moves) it into the + * asset's source/ dir, probes metadata, and persists the asset record. + * A failed probe persists an error-state asset rather than throwing — + * the UI shows it with a retry/remove affordance. + */ +export async function createMediaAsset( + threadId: string, + options: { sourcePath: string; name?: string; move?: boolean; assetId?: string } +): Promise { + const thread = threadManager.getThread(threadId) + if (!thread || thread.type !== 'editor' || !thread.editor) return null + + if (!fs.existsSync(options.sourcePath) || fs.statSync(options.sourcePath).isDirectory()) { + throw new Error(`Invalid media source: "${options.sourcePath}" is not a file.`) + } + + const assetId = options.assetId || uuidv4() + const assetDir = getAssetDir(thread, assetId) + const sourceDir = path.join(assetDir, ASSET_DIRS.SOURCE) + ensureDir(sourceDir) + + const rawName = options.name || path.basename(options.sourcePath) + const fileName = sanitizeFilename(rawName) + const targetPath = path.join(sourceDir, fileName) + + // Only copy/move if the file isn't already inside the asset's source dir + // (URL imports download straight into it). + if (path.resolve(options.sourcePath) !== path.resolve(targetPath)) { + if (options.move) { + fs.renameSync(options.sourcePath, targetPath) + } else { + fs.copyFileSync(options.sourcePath, targetPath) + } + } + + const asset: MediaAsset = { + id: assetId, + kind: 'video', + name: rawName, + originalPath: targetPath, + preprocessing: {}, + preprocessState: 'pending', + clips: [], + createdAt: Date.now() + } + + try { + asset.metadata = await getVideoMetadata(targetPath) + } catch (error) { + console.error(`[editor] Failed to probe media ${targetPath}:`, error) + asset.preprocessState = 'error' + asset.preprocessError = 'Could not read video metadata (corrupt or unsupported file).' + } + + await threadManager.updateThreadWith(threadId, (t) => { + if (!t.editor) return null + return { editor: { ...t.editor, media: [...t.editor.media, asset] } } + }) + + return asset +} + +/** Merge a partial patch into one asset record (queued, safe under concurrency). */ +export function patchAsset( + threadId: string, + assetId: string, + patch: Partial | ((asset: MediaAsset) => Partial) +): Promise { + return threadManager.updateThreadWith(threadId, (thread) => { + if (!thread.editor) return null + const index = thread.editor.media.findIndex((a) => a.id === assetId) + if (index === -1) return null + + const current = thread.editor.media[index] + const resolved = typeof patch === 'function' ? patch(current) : patch + const media = [...thread.editor.media] + media[index] = { ...current, ...resolved } + return { editor: { ...thread.editor, media } } + }) +} + +/** Merge a preprocessing patch into one asset (mirrors PipelineContext.savePreprocessing). */ +export function patchAssetPreprocessing( + threadId: string, + assetId: string, + patch: Partial +): Promise { + return patchAsset(threadId, assetId, (asset) => ({ + preprocessing: { ...(asset.preprocessing || {}), ...patch } + })) +} + +/** + * Removes an asset: aborts any live preprocessing, deletes its artifact dir, + * and drops the asset plus its clips, timeline items, and namespaced tasks. + */ +export async function removeAsset(threadId: string, assetId: string): Promise { + abortAssetPreprocessing(threadId, assetId) + + const thread = threadManager.getThread(threadId) + if (!thread || !thread.editor) return false + + const assetDir = getAssetDir(thread, assetId) + if (fs.existsSync(assetDir)) { + try { + fs.rmSync(assetDir, { recursive: true, force: true }) + } catch (error) { + console.error(`[editor] Failed to delete asset dir ${assetDir}:`, error) + } + } + + const updated = await threadManager.updateThreadWith(threadId, (t) => { + if (!t.editor) return null + + const backgroundTasks = { ...(t.backgroundTasks || {}) } + for (const taskId of Object.keys(backgroundTasks)) { + if (taskId.startsWith(`${assetId}:`)) delete backgroundTasks[taskId] + } + + return { + editor: { + ...t.editor, + media: t.editor.media.filter((a) => a.id !== assetId), + timeline: t.editor.timeline.filter((item) => item.sourceAssetId !== assetId) + }, + backgroundTasks + } + }) + + return updated !== null +} diff --git a/src/main/editor/preprocess.ts b/src/main/editor/preprocess.ts new file mode 100644 index 0000000..992cdb7 --- /dev/null +++ b/src/main/editor/preprocess.ts @@ -0,0 +1,428 @@ +import fs from 'fs' +import path from 'path' +import { setMaxListeners } from 'events' +import { v4 as uuidv4 } from 'uuid' +import type { Clip, MediaAsset } from '@shared/types' +import type { PipelineContext } from '../pipeline' +import { threadManager } from '../threads' +import { backgroundTaskManager } from '../tasks' +import * as ffmpegAdapter from '../ffmpeg' +import { SceneDetector, checkScenedetectAvailability, type Scene } from '../scenedetect' +import * as extraction from '../pipeline/phases/extraction' +import { ASSET_DIRS } from '../constants/paths' +import { getAssetDir, patchAsset, patchAssetPreprocessing } from './assets' + +/** + * Per-asset preprocessing orchestrator for the timeline editor. + * + * Design (video-editor-prd.md §5.2/§7): a lean, editor-shaped step chain + * (proxy -> scenes -> thumbnails -> clips) calling the ffmpeg/scenedetect + * helpers directly, plus an OPT-IN Gemini descriptions step that reuses + * extraction.generateSceneDescription VERBATIM through an asset-scoped + * PipelineContext (tempDir = tempDir/media/), leaving the chat + * preprocessing chains untouched. + * + * Task ids are namespaced `${assetId}:` inside the existing + * thread.backgroundTasks + background-task-update broadcast. + */ + +export type PreprocessStep = 'proxy' | 'scenes' | 'thumbnails' | 'descriptions' + +const DEFAULT_STEPS: PreprocessStep[] = ['proxy', 'scenes', 'thumbnails'] + +export const SCENEDETECT_MISSING = 'scenedetect-missing' + +// ===== Concurrency cap (K=3) for ffmpeg/scenedetect-heavy work ===== +const MAX_CONCURRENT_HEAVY = 3 +let heavyRunning = 0 +const heavyQueue: Array<() => void> = [] + +async function withHeavySlot(fn: () => Promise): Promise { + if (heavyRunning >= MAX_CONCURRENT_HEAVY) { + await new Promise((resolve) => heavyQueue.push(resolve)) + } + heavyRunning++ + try { + return await fn() + } finally { + heavyRunning-- + const next = heavyQueue.shift() + if (next) next() + } +} + +// ===== Abort registry ===== +const abortControllers = new Map() + +const abortKey = (threadId: string, assetId: string) => `${threadId}:${assetId}` + +export function abortAssetPreprocessing(threadId: string, assetId: string): void { + const key = abortKey(threadId, assetId) + const controller = abortControllers.get(key) + if (controller) { + controller.abort() + abortControllers.delete(key) + } +} + +export function isAssetPreprocessing(threadId: string, assetId: string): boolean { + return abortControllers.has(abortKey(threadId, assetId)) +} + +// ===== Helpers ===== + +function getAsset(threadId: string, assetId: string): MediaAsset | null { + const thread = threadManager.getThread(threadId) + return thread?.editor?.media.find((a) => a.id === assetId) || null +} + +function exists(p?: string): boolean { + return !!p && fs.existsSync(p) +} + +const taskId = (assetId: string, step: PreprocessStep) => `${assetId}:${step}` + +async function setTask( + threadId: string, + assetId: string, + step: PreprocessStep, + updates: Parameters[2] +) { + await backgroundTaskManager.updateTask(threadId, taskId(assetId, step), { + name: step, + ...updates + }) +} + +/** + * Asset-scoped PipelineContext bridge so existing pipeline phases + * (generateSceneDescription today; transcript phases later) run per-asset. + * Modeled on backgroundTaskManager.createMockContext, but all reads/writes + * target the MediaAsset record and the asset's artifact dir. + */ +function createAssetContext( + threadId: string, + assetId: string, + step: PreprocessStep, + signal: AbortSignal +): PipelineContext { + const thread = threadManager.getThread(threadId)! + const asset = getAsset(threadId, assetId)! + const assetDir = getAssetDir(thread, assetId) + + return { + threadId, + videoPath: asset.proxyPath || asset.originalPath, + tempDir: assetDir, // THREAD_DIRS.FRAMES/ANALYSIS joins inside phases land under the asset dir + get preprocessing() { + return getAsset(threadId, assetId)?.preprocessing || {} + }, + messageId: `editor-${assetId}`, + context: '', + baseTimeline: undefined, + intentResult: undefined, + updateStatus: async (status: string) => { + await setTask(threadId, assetId, step, { state: 'running', status }) + }, + recordUsage: async (record) => { + await threadManager.updateThreadWith(threadId, (t) => ({ + usageHistory: [...(t.usageHistory || []), { ...record, timestamp: Date.now() }] + })) + }, + savePreprocessing: async (updates) => { + await patchAssetPreprocessing(threadId, assetId, updates) + }, + waitForTask: async () => { }, + next: () => { }, + finish: async () => { }, + fail: async (error: string) => { + await setTask(threadId, assetId, step, { state: 'error', error }) + }, + signal + } +} + +// ===== Steps ===== + +async function runProxyStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (exists(asset.proxyPath)) { + await setTask(threadId, assetId, 'proxy', { state: 'completed', progress: 100 }) + return + } + + await setTask(threadId, assetId, 'proxy', { state: 'running', status: 'Creating 480p proxy…', progress: 0 }) + + const thread = threadManager.getThread(threadId)! + const proxyDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.PROXY) + if (!fs.existsSync(proxyDir)) fs.mkdirSync(proxyDir, { recursive: true }) + + let proxyPath: string + if (await ffmpegAdapter.isVideoLowResolution(asset.originalPath)) { + // Already <=480p — reuse the original as the proxy + proxyPath = asset.originalPath + } else { + proxyPath = await withHeavySlot(() => + ffmpegAdapter.toLowResolution( + asset.originalPath, + proxyDir, + (percent) => setTask(threadId, assetId, 'proxy', { state: 'running', progress: percent }), + signal + ) + ) + } + + await patchAsset(threadId, assetId, { proxyPath }) + await patchAssetPreprocessing(threadId, assetId, { lowResVideoPath: proxyPath }) + await setTask(threadId, assetId, 'proxy', { state: 'completed', progress: 100 }) +} + +async function runScenesStep( + threadId: string, + assetId: string, + signal: AbortSignal, + threshold?: number +) { + const asset = getAsset(threadId, assetId)! + const forceRerun = threshold !== undefined + + if (!forceRerun && exists(asset.preprocessing.sceneTimesPath)) { + await setTask(threadId, assetId, 'scenes', { state: 'completed', progress: 100 }) + return + } + + await setTask(threadId, assetId, 'scenes', { state: 'running', status: 'Detecting scenes…' }) + + const available = await checkScenedetectAvailability() + if (!available) { + throw new Error(SCENEDETECT_MISSING) + } + + const thread = threadManager.getThread(threadId)! + const analysisDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.ANALYSIS) + if (!fs.existsSync(analysisDir)) fs.mkdirSync(analysisDir, { recursive: true }) + const sceneTimesPath = path.join(analysisDir, 'scenes.json') + + const detector = new SceneDetector() + const videoPath = asset.proxyPath || asset.originalPath + let scenes = await withHeavySlot(() => detector.detectScenes(videoPath, signal, threshold)) + + // A cut-less video (talking head, screen recording) can yield zero rows — + // fall back to a single whole-video scene so there is always one piece. + if (scenes.length === 0 && asset.metadata?.duration) { + scenes = [{ startTime: 0, endTime: asset.metadata.duration, duration: asset.metadata.duration }] + } + + fs.writeFileSync(sceneTimesPath, JSON.stringify(scenes, null, 2)) + await patchAssetPreprocessing(threadId, assetId, { sceneTimesPath }) + + // Derive clips IMMEDIATELY (thumbnails fill in progressively afterwards) so + // the pieces tray and counts populate as soon as scenes are known. + await deriveClips(threadId, assetId, scenes) + + await setTask(threadId, assetId, 'scenes', { + state: 'completed', + progress: 100, + status: `${scenes.length} scenes detected` + }) +} + +/** + * Map Scene[] -> Clip[] on the asset, preserving renderer-owned `selected` + * and prior `visual`/`thumbnailPath` by (in,out) epsilon-match on re-runs. + */ +async function deriveClips(threadId: string, assetId: string, scenes: Scene[]) { + const EPSILON = 0.05 + await patchAsset(threadId, assetId, (current) => { + const previous = current.clips || [] + const clips: Clip[] = scenes.map((scene, i) => { + const match = previous.find( + (c) => Math.abs(c.in - scene.startTime) <= EPSILON && Math.abs(c.out - scene.endTime) <= EPSILON + ) + return { + id: match?.id || uuidv4(), + sourceAssetId: assetId, + index: i + 1, + in: scene.startTime, + out: scene.endTime, + duration: scene.duration, + thumbnailPath: match?.thumbnailPath, + visual: match?.visual, + text: match?.text, + selected: match?.selected ?? false, + masterSegmentIndex: i + 1 + } + }) + return { clips } + }) +} + +async function runThumbnailsStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + const sceneTimesPath = asset.preprocessing.sceneTimesPath + if (!exists(sceneTimesPath)) { + await setTask(threadId, assetId, 'thumbnails', { state: 'completed', status: 'No scenes to thumbnail' }) + return + } + + const scenes: Scene[] = JSON.parse(fs.readFileSync(sceneTimesPath!, 'utf-8')) + const thread = threadManager.getThread(threadId)! + const framesDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.FRAMES) + if (!fs.existsSync(framesDir)) fs.mkdirSync(framesDir, { recursive: true }) + + await setTask(threadId, assetId, 'thumbnails', { state: 'running', status: 'Extracting thumbnails…', progress: 0 }) + + const videoPath = asset.proxyPath || asset.originalPath + const thumbnails: (string | undefined)[] = [] + const EPSILON = 0.05 + + // Patch finished thumbnails into the (already-derived) clips so tiles fill + // in progressively. Every updateTask/patchAsset persists the thread JSON, + // so both are THROTTLED to a batch cadence rather than per-frame. + const flushThumbnails = async () => { + await patchAsset(threadId, assetId, (current) => ({ + clips: (current.clips || []).map((clip) => { + const i = scenes.findIndex( + (s) => Math.abs(clip.in - s.startTime) <= EPSILON && Math.abs(clip.out - s.endTime) <= EPSILON + ) + return i !== -1 && thumbnails[i] + ? { ...clip, thumbnailPath: thumbnails[i] } + : clip + }) + })) + } + + const BATCH = Math.max(1, Math.min(8, Math.floor(scenes.length / 25) || 1)) + + await withHeavySlot(async () => { + for (let i = 0; i < scenes.length; i++) { + if (signal.aborted) throw new Error('Aborted') + const scene = scenes[i] + const midpoint = scene.startTime + scene.duration / 2 + try { + // extractFrame filenames are deterministic (…_frame_.jpg) — re-runs reuse them + const framePath = await ffmpegAdapter.extractFrame(videoPath, midpoint, framesDir, signal) + thumbnails.push(framePath) + } catch (error) { + if (signal.aborted) throw error + console.error(`[editor] Thumbnail failed for scene ${i} of asset ${assetId}:`, error) + thumbnails.push(undefined) // tolerate individual frame failures + } + + const done = i + 1 + if (done % BATCH === 0 || done === scenes.length) { + await flushThumbnails() + await setTask(threadId, assetId, 'thumbnails', { + state: 'running', + status: `${done}/${scenes.length} thumbnails`, + progress: Math.round((done / scenes.length) * 100) + }) + } + } + }) + + await setTask(threadId, assetId, 'thumbnails', { state: 'completed', progress: 100 }) +} + +async function runDescriptionsStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (!exists(asset.preprocessing.sceneTimesPath)) { + throw new Error('Scenes must be detected before describing them.') + } + + await setTask(threadId, assetId, 'descriptions', { state: 'running', status: 'Describing scenes…' }) + + // Reuse the existing pipeline phase VERBATIM through an asset-scoped context. + const context = createAssetContext(threadId, assetId, 'descriptions', signal) + await extraction.generateSceneDescription({}, context) + + if (signal.aborted) throw new Error('Aborted') + + // Merge generated descriptions back into the asset's clips (by scene index). + const descriptionsPath = getAsset(threadId, assetId)?.preprocessing.sceneDescriptionsPath + if (exists(descriptionsPath)) { + const descriptions: { index: number; description: string; framePath: string }[] = + JSON.parse(fs.readFileSync(descriptionsPath!, 'utf-8')) + const byIndex = new Map(descriptions.map((d) => [d.index, d])) + + await patchAsset(threadId, assetId, (current) => ({ + clips: (current.clips || []).map((clip) => { + const desc = byIndex.get(clip.index - 1) // descriptions are 0-based scene indices + return desc + ? { ...clip, visual: desc.description, thumbnailPath: clip.thumbnailPath || desc.framePath } + : clip + }) + })) + } + + await setTask(threadId, assetId, 'descriptions', { state: 'completed', progress: 100 }) +} + +// ===== Orchestrator ===== + +export async function preprocessMediaAsset( + threadId: string, + assetId: string, + options?: { steps?: PreprocessStep[]; threshold?: number } +): Promise { + const asset = getAsset(threadId, assetId) + if (!asset) return + if (isAssetPreprocessing(threadId, assetId)) return // already running + + const steps = options?.steps?.length ? options.steps : DEFAULT_STEPS + const controller = new AbortController() + // Every ffmpeg call (one per scene thumbnail) attaches an abort listener to + // this shared signal — lift Node's default cap of 10 to avoid leak warnings. + setMaxListeners(0, controller.signal) + abortControllers.set(abortKey(threadId, assetId), controller) + const { signal } = controller + + await patchAsset(threadId, assetId, { preprocessState: 'running', preprocessError: undefined }) + + // Pre-register every step as pending so the UI shows the full checklist + // immediately and each bar visibly transitions pending -> running -> done. + for (const step of steps) { + await setTask(threadId, assetId, step, { state: 'pending', progress: 0, status: undefined, error: undefined }) + } + + let currentStep: PreprocessStep | null = null + try { + // A threshold re-run must re-detect scenes even if outputs exist + if (options?.threshold !== undefined) { + await patchAssetPreprocessing(threadId, assetId, { sceneTimesPath: undefined }) + } + + for (const step of steps) { + if (signal.aborted) throw new Error('Aborted') + currentStep = step + switch (step) { + case 'proxy': await runProxyStep(threadId, assetId, signal); break + case 'scenes': await runScenesStep(threadId, assetId, signal, options?.threshold); break + case 'thumbnails': await runThumbnailsStep(threadId, assetId, signal); break + case 'descriptions': await runDescriptionsStep(threadId, assetId, signal); break + } + } + + await patchAsset(threadId, assetId, { preprocessState: 'completed' }) + } catch (error: any) { + if (signal.aborted) { + // Asset removed or run cancelled — leave whatever state the removal left + console.log(`[editor] Preprocessing aborted for asset ${assetId}`) + return + } + const message = error?.message || 'Preprocessing failed' + console.error(`[editor] Preprocessing failed for asset ${assetId} (step: ${currentStep}):`, error) + // Mark the failed step's task and the asset errored; other assets are unaffected. + if (currentStep) { + await setTask(threadId, assetId, currentStep, { state: 'error', error: message }) + } + await patchAsset(threadId, assetId, { + preprocessState: 'error', + preprocessError: message === SCENEDETECT_MISSING + ? `${SCENEDETECT_MISSING}: PySceneDetect is not installed — scene splitting is unavailable.` + : message + }) + } finally { + abortControllers.delete(abortKey(threadId, assetId)) + } +} diff --git a/src/main/index.ts b/src/main/index.ts index 9659da5..4ae7300 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -22,6 +22,9 @@ import { checkFFmpegAvailability, getVideoMetadata } from './ffmpeg' import { checkScenedetectAvailability } from './scenedetect' import { checkYtDlpAvailability, downloadVideo, getVideoFormats } from './ytdlp' import { dependencyManager } from './dependencies/manager' +import * as editorAssets from './editor/assets' +import * as editorPreprocess from './editor/preprocess' +import { v4 as uuidv4 } from 'uuid' import { THREAD_DIRS } from './constants/paths' import { GEMINI_MODEL_2_5_FLASH, MODEL_METADATA } from './constants/gemini' import { electronApp, optimizer, is } from '@electron-toolkit/utils' @@ -345,6 +348,129 @@ app.whenReady().then(() => { return true }) + // ===== Timeline Video Editor ===== + + // Creates an editor project thread. Deliberately does NOT auto-start + // preprocessing (media is imported per-asset inside the editor). + ipcMain.handle('create-editor-project', async (_event, { title }: { title?: string }) => { + return await threadManager.createEditorThread(title || 'Untitled Project') + }) + + // Autosave of RENDERER-OWNED editor fields only. Main owns preprocessing-derived + // asset state (preprocessing/proxyPath/metadata/preprocessState/clips content); + // the renderer owns selection, clip-selected flags, tracks, timeline, meta, personas. + // This ownership split is what prevents autosave/thread-updated echo clobbering. + ipcMain.handle('save-editor-doc', async (_event, { threadId, patch }: { + threadId: string + patch: { + tracks?: any[] + timeline?: any[] + timelineMeta?: any + selection?: any + activePersonaId?: string + customPersonas?: any[] + clipSelections?: Record + } + }) => { + return await threadManager.updateThreadWith(threadId, (thread) => { + if (thread.type !== 'editor' || !thread.editor) return null + + const editor = { ...thread.editor } + if (patch.tracks) editor.tracks = patch.tracks + if (patch.timeline) editor.timeline = patch.timeline + if (patch.timelineMeta) editor.timelineMeta = patch.timelineMeta + if (patch.selection) editor.selection = patch.selection + if (patch.activePersonaId !== undefined) editor.activePersonaId = patch.activePersonaId + if (patch.customPersonas) editor.customPersonas = patch.customPersonas + + // Fold clip-selected flags into the matching clips (the one renderer-owned + // field living inside main-owned MediaAsset records). + if (patch.clipSelections) { + editor.media = editor.media.map(asset => ({ + ...asset, + clips: asset.clips.map(clip => + patch.clipSelections![clip.id] !== undefined + ? { ...clip, selected: patch.clipSelections![clip.id] } + : clip + ) + })) + } + + return { editor } + }) + }) + + // Import a local file as a MediaAsset and start per-asset preprocessing. + ipcMain.handle('add-media-asset', async (_event, { threadId, filePath, name }: { + threadId: string, filePath: string, name?: string + }) => { + const asset = await editorAssets.createMediaAsset(threadId, { sourcePath: filePath, name }) + if (asset && asset.preprocessState !== 'error') { + // Fire-and-forget: progress streams via background-task-update + editorPreprocess.preprocessMediaAsset(threadId, asset.id).catch((error) => { + console.error(`[editor] preprocess failed for ${asset.id}:`, error) + }) + } + return asset + }) + + // Import from URL/YouTube: download straight into the asset's source dir, + // with per-asset progress (the legacy 'download-progress' event is keyless + // and breaks under concurrent imports). + ipcMain.handle('import-media-url', async (event, { threadId, url, resolution }: { + threadId: string, url: string, resolution?: string + }) => { + const thread = threadManager.getThread(threadId) + if (!thread || thread.type !== 'editor') throw new Error('Not an editor project') + + const assetId = uuidv4() + const sourceDir = join(thread.tempDir, 'media', assetId, 'source') + fs.mkdirSync(sourceDir, { recursive: true }) + + try { + const result = await downloadVideo(url, sourceDir, resolution, (percent) => { + event.sender.send('editor-import-progress', { threadId, assetId, url, percent }) + }) + + const asset = await editorAssets.createMediaAsset(threadId, { + sourcePath: result.path, + name: result.name, + assetId + }) + if (asset && asset.preprocessState !== 'error') { + editorPreprocess.preprocessMediaAsset(threadId, asset.id).catch((error) => { + console.error(`[editor] preprocess failed for ${asset.id}:`, error) + }) + } + return asset + } catch (error) { + // Failed import: remove the partial asset dir so nothing dangles + try { + fs.rmSync(join(thread.tempDir, 'media', assetId), { recursive: true, force: true }) + } catch { /* best effort */ } + throw error + } + }) + + ipcMain.handle('remove-media-asset', async (_event, { threadId, assetId }: { + threadId: string, assetId: string + }) => { + return await editorAssets.removeAsset(threadId, assetId) + }) + + // Retry / re-run / opt-in steps (e.g. ['descriptions']) for one asset. + ipcMain.handle('preprocess-media', async (_event, { threadId, assetId, steps, threshold }: { + threadId: string, assetId: string, steps?: string[], threshold?: number + }) => { + editorPreprocess.preprocessMediaAsset(threadId, assetId, { + steps: steps as any, + threshold + }).catch((error) => { + console.error(`[editor] preprocess retry failed for ${assetId}:`, error) + }) + return true + }) + ipcMain.handle('get-all-threads', () => { return threadManager.getAllThreads() }) @@ -354,6 +480,13 @@ app.whenReady().then(() => { }) ipcMain.handle('delete-thread', (_event, id) => { + // Editor threads: abort any live per-asset preprocessing before deletion + const thread = threadManager.getThread(id) + if (thread?.type === 'editor' && thread.editor) { + for (const asset of thread.editor.media) { + editorPreprocess.abortAssetPreprocessing(id, asset.id) + } + } return threadManager.deleteThread(id) }) diff --git a/src/main/scenedetect/index.ts b/src/main/scenedetect/index.ts index 107a15a..5772bf6 100644 --- a/src/main/scenedetect/index.ts +++ b/src/main/scenedetect/index.ts @@ -173,11 +173,11 @@ export class SceneDetector { * @returns Array of detected scenes sorted by start time. * @throws If the CLI exits non-zero, the CSV is missing, or values cannot be parsed. */ - async detectScenes(videoPath: string, signal?: AbortSignal): Promise { + async detectScenes(videoPath: string, signal?: AbortSignal, threshold: number = CONTENT_THRESHOLD): Promise { const tempDir = await fs.mkdtemp(join(tmpdir(), 'scenedetect-')) try { - await this.runScenedetect(videoPath, tempDir, signal) + await this.runScenedetect(videoPath, tempDir, signal, threshold) const csvPath = await this.locateCsvFile(tempDir, videoPath) const csvContent = await fs.readFile(csvPath, 'utf-8') return this.parseCsv(csvContent) @@ -194,14 +194,14 @@ export class SceneDetector { /** * Execute the scenedetect CLI process. */ - private async runScenedetect(videoPath: string, outputDir: string, signal?: AbortSignal): Promise { + private async runScenedetect(videoPath: string, outputDir: string, signal?: AbortSignal, threshold: number = CONTENT_THRESHOLD): Promise { const pathOrRef = await resolveScenedetectPath() - + return new Promise((resolve, reject) => { const scenedetectArgs = [ '-i', videoPath, 'detect-content', - '-t', String(CONTENT_THRESHOLD), + '-t', String(threshold), 'list-scenes', '-o', outputDir, '-f', CSV_FILENAME, diff --git a/src/main/threads/index.ts b/src/main/threads/index.ts index 5889779..03f5129 100644 --- a/src/main/threads/index.ts +++ b/src/main/threads/index.ts @@ -3,7 +3,7 @@ import fs from 'fs' import path from 'path' import { v4 as uuidv4 } from 'uuid' import { MessageRole, FileType } from '@shared/types' -import type { Message, Thread, Usage, UsageRecord, VideoMetadata } from '@shared/types' +import type { EditorDocument, Message, Thread, Track, Usage, UsageRecord, VideoMetadata } from '@shared/types' import { settingsManager } from '../settings' import { getVideoMetadata } from '../ffmpeg' import { THREAD_DIRS } from '../constants/paths' @@ -137,6 +137,58 @@ class ThreadManager { return thread } + // Create a new timeline-editor project thread. + // Unlike createThread, no videoPath is required and NO preprocessing auto-starts — + // media is imported per-asset inside the editor (see src/main/editor/). + async createEditorThread(title: string): Promise { + const id = uuidv4() + const tempDir = settingsManager.getThreadTempDir(id) + + const seedTrack = (kind: Track['kind'], name: string, order: number): Track => ({ + id: uuidv4(), + kind, + name, + order, + muted: false, + locked: false, + hidden: false, + height: 64 + }) + + const editor: EditorDocument = { + schemaVersion: 1, + media: [], + tracks: [ + seedTrack('video', 'V1', 0), + seedTrack('audio', 'A1', 1), + seedTrack('overlay', 'OV', 2) + ], + timeline: [], + timelineMeta: { fps: 30, width: 1920, height: 1080, duration: 0 }, + activePersonaId: '', + turns: [], + historyRef: { currentStepId: '', stepCount: 0 }, + selection: {} + } + + const thread: Thread = { + id, + title, + type: 'editor', + preprocessing: {}, + tempDir, + messages: [], + versionCounter: 0, + usageHistory: [], + editor, + createdAt: Date.now(), + updatedAt: Date.now() + } + + this.saveThread(thread) + return thread + } + // Helper to normalize paths (handles symlinks like /var vs /private/var on macOS) private normalize(p: string | undefined): string | undefined { @@ -209,6 +261,30 @@ class ThreadManager { return msg }) + // Update editor document media paths (timeline editor threads) + if (thread.editor?.media) { + for (const asset of thread.editor.media) { + asset.originalPath = fixPath(asset.originalPath) || asset.originalPath + asset.proxyPath = fixPath(asset.proxyPath) + if (asset.preprocessing) { + for (const key of Object.keys(asset.preprocessing) as Array) { + const value = asset.preprocessing[key] + if (typeof value === 'string') { + (asset.preprocessing as any)[key] = fixPath(value) + } else if (Array.isArray(value)) { + (asset.preprocessing as any)[key] = value.map(fixPath) + } + } + } + for (const clip of asset.clips || []) { + clip.thumbnailPath = fixPath(clip.thumbnailPath) + } + for (const entry of asset.filmstrip || []) { + entry.thumbnailPath = fixPath(entry.thumbnailPath) || entry.thumbnailPath + } + } + } + // Save the repaired thread back to metadata and mirror it this.saveThread(thread) } @@ -320,14 +396,20 @@ class ThreadManager { } } - // Update a thread atomically - updateThread(id: string, updates: Partial): Promise { + // Update a thread atomically via a mutator that runs INSIDE the queued closure. + // The mutator receives the freshest thread state and returns the partial update + // (or null to skip). This makes concurrent read-modify-write patterns safe + // (e.g. multiple assets patching thread.editor.media in parallel). + updateThreadWith(id: string, mutator: (thread: Thread) => Partial | null): Promise { const existingQueue = this.updateQueues.get(id) || Promise.resolve() const nextUpdate = existingQueue.then(async () => { const thread = this.getThread(id) if (!thread) return null + const updates = mutator(thread) + if (!updates) return thread + const updatedThread = { ...thread, ...updates, @@ -359,6 +441,11 @@ class ThreadManager { return nextUpdate } + // Update a thread atomically with a precomputed partial. + updateThread(id: string, updates: Partial): Promise { + return this.updateThreadWith(id, () => updates) + } + private deleteFile(filePath: string) { if (!filePath) return const cleanPath = filePath.replace('file://', '') diff --git a/src/preload/index.ts b/src/preload/index.ts index f96f709..a216b46 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -48,6 +48,23 @@ const api = { getModelSettings: () => ipcRenderer.invoke('get-model-settings'), setModelSettings: (settings: any) => ipcRenderer.invoke('set-model-settings', settings), resetModelSettings: () => ipcRenderer.invoke('reset-model-settings'), + // Timeline Video Editor + createEditorProject: (title?: string) => ipcRenderer.invoke('create-editor-project', { title }), + saveEditorDoc: (data: { threadId: string, patch: any }) => + ipcRenderer.invoke('save-editor-doc', data), + addMediaAsset: (data: { threadId: string, filePath: string, name?: string }) => + ipcRenderer.invoke('add-media-asset', data), + importMediaUrl: (data: { threadId: string, url: string, resolution?: string }) => + ipcRenderer.invoke('import-media-url', data), + removeMediaAsset: (data: { threadId: string, assetId: string }) => + ipcRenderer.invoke('remove-media-asset', data), + preprocessMedia: (data: { threadId: string, assetId: string, steps?: string[], threshold?: number }) => + ipcRenderer.invoke('preprocess-media', data), + onEditorImportProgress: (callback: (data: { threadId: string, assetId: string, url?: string, percent: number }) => void) => { + const listener = (_event: any, data: any) => callback(data) + ipcRenderer.on('editor-import-progress', listener) + return () => ipcRenderer.removeListener('editor-import-progress', listener) + }, // Thread Management createThread: (data: { videoPath?: string, videoName: string, imagePaths?: string[] }) => ipcRenderer.invoke('create-thread', data), diff --git a/src/renderer/src/editor/VideoEditorPage.vue b/src/renderer/src/editor/VideoEditorPage.vue new file mode 100644 index 0000000..ebb8b03 --- /dev/null +++ b/src/renderer/src/editor/VideoEditorPage.vue @@ -0,0 +1,75 @@ + + + diff --git a/src/renderer/src/editor/components/AssetRow.vue b/src/renderer/src/editor/components/AssetRow.vue new file mode 100644 index 0000000..a682f53 --- /dev/null +++ b/src/renderer/src/editor/components/AssetRow.vue @@ -0,0 +1,152 @@ + + + diff --git a/src/renderer/src/editor/components/ClipTile.vue b/src/renderer/src/editor/components/ClipTile.vue new file mode 100644 index 0000000..fa68d9f --- /dev/null +++ b/src/renderer/src/editor/components/ClipTile.vue @@ -0,0 +1,64 @@ + + + diff --git a/src/renderer/src/editor/components/ClipTray.vue b/src/renderer/src/editor/components/ClipTray.vue new file mode 100644 index 0000000..78f3804 --- /dev/null +++ b/src/renderer/src/editor/components/ClipTray.vue @@ -0,0 +1,47 @@ + + + diff --git a/src/renderer/src/editor/components/ImportMediaModal.vue b/src/renderer/src/editor/components/ImportMediaModal.vue new file mode 100644 index 0000000..51d3fa1 --- /dev/null +++ b/src/renderer/src/editor/components/ImportMediaModal.vue @@ -0,0 +1,136 @@ + + + diff --git a/src/renderer/src/editor/components/InspectorPanel.vue b/src/renderer/src/editor/components/InspectorPanel.vue new file mode 100644 index 0000000..b0e6f8f --- /dev/null +++ b/src/renderer/src/editor/components/InspectorPanel.vue @@ -0,0 +1,120 @@ + + + diff --git a/src/renderer/src/editor/components/MediaPanel.vue b/src/renderer/src/editor/components/MediaPanel.vue new file mode 100644 index 0000000..d43c93a --- /dev/null +++ b/src/renderer/src/editor/components/MediaPanel.vue @@ -0,0 +1,99 @@ + + + diff --git a/src/renderer/src/editor/components/PreviewMonitor.vue b/src/renderer/src/editor/components/PreviewMonitor.vue new file mode 100644 index 0000000..4eca51d --- /dev/null +++ b/src/renderer/src/editor/components/PreviewMonitor.vue @@ -0,0 +1,121 @@ + + + diff --git a/src/renderer/src/editor/components/PromptBar.vue b/src/renderer/src/editor/components/PromptBar.vue new file mode 100644 index 0000000..a1f3a4c --- /dev/null +++ b/src/renderer/src/editor/components/PromptBar.vue @@ -0,0 +1,20 @@ + diff --git a/src/renderer/src/editor/components/TimelinePanel.vue b/src/renderer/src/editor/components/TimelinePanel.vue new file mode 100644 index 0000000..016bb09 --- /dev/null +++ b/src/renderer/src/editor/components/TimelinePanel.vue @@ -0,0 +1,56 @@ + + + diff --git a/src/renderer/src/pages/HomePage.vue b/src/renderer/src/pages/HomePage.vue index 4ed645d..2801c96 100644 --- a/src/renderer/src/pages/HomePage.vue +++ b/src/renderer/src/pages/HomePage.vue @@ -21,8 +21,8 @@ /> - @@ -33,6 +33,19 @@ + + + + + @@ -55,7 +68,16 @@ const videoStore = useVideoStore() const loading = ref(true) const openThread = (id: string) => { - router.push(`/chat/${id}`) + const thread = videoStore.threads.find((t) => t.id === id) + router.push(thread?.type === 'editor' ? `/editor/${id}` : `/chat/${id}`) +} + +const handleCreateEditorProject = async () => { + const thread = await (window as any).api.createEditorProject('Untitled Project') + if (thread) { + videoStore.threads.unshift(thread) + router.push(`/editor/${thread.id}`) + } } const handleCreateImageEdit = async () => { diff --git a/src/renderer/src/router.ts b/src/renderer/src/router.ts index de97675..f53b6ee 100644 --- a/src/renderer/src/router.ts +++ b/src/renderer/src/router.ts @@ -5,6 +5,7 @@ import SettingsPage from './pages/SettingsPage.vue' import HomePage from './pages/HomePage.vue' import GraphChatPage from './pages/GraphChatPage.vue' +import VideoEditorPage from './editor/VideoEditorPage.vue' const routes = [ { @@ -28,6 +29,11 @@ const routes = [ name: 'chat', component: GraphChatPage }, + { + path: '/editor/:id', + name: 'editor', + component: VideoEditorPage + }, { path: '/settings', diff --git a/src/renderer/src/stores/editorStore.ts b/src/renderer/src/stores/editorStore.ts new file mode 100644 index 0000000..9dad71a --- /dev/null +++ b/src/renderer/src/stores/editorStore.ts @@ -0,0 +1,373 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import type { BackgroundTask, Clip, EditorDocument, MediaAsset, Thread } from '@shared/types' + +/** + * Store for the timeline video editor (/editor/:id). + * + * Ownership split (prevents autosave <-> thread-updated echo clobbering): + * - MAIN owns preprocessing-derived asset fields: preprocessing, proxyPath, + * metadata, preprocessState, preprocessError, clips content. + * - RENDERER owns: selection, clip.selected flags, tracks, timeline, + * timelineMeta, activePersonaId, customPersonas. + * persistDoc() sends only renderer-owned fields; the thread-updated handler + * merges only main-owned fields. + */ +export const useEditorStore = defineStore('editor', () => { + const api = (window as any).api + + // ===== State ===== + const threadId = ref(null) + const thread = ref(null) + const doc = ref(null) + const backgroundTasks = ref>({}) + const selectedAssetId = ref(null) + const selectedClipId = ref(null) + const urlImports = ref>({}) + const interruptedAssets = ref>(new Set()) + const dirty = ref(false) + const saveInFlight = ref(false) + const deps = ref<{ ffmpeg: boolean; scenedetect: boolean; ytDlp: boolean }>({ + ffmpeg: true, + scenedetect: true, + ytDlp: true + }) + const loading = ref(false) + + let autosaveTimer: ReturnType | null = null + + // ===== Computeds ===== + const assets = computed(() => doc.value?.media ?? []) + const isEmpty = computed(() => assets.value.length === 0) + + const selectedAsset = computed(() => + assets.value.find((a) => a.id === selectedAssetId.value) || null + ) + + const selectedClip = computed(() => { + if (!selectedClipId.value) return null + for (const asset of assets.value) { + const clip = asset.clips.find((c) => c.id === selectedClipId.value) + if (clip) return clip + } + return null + }) + + const STEP_ORDER = ['proxy', 'scenes', 'thumbnails', 'descriptions'] + + const assetTasks = (assetId: string): BackgroundTask[] => { + const prefix = `${assetId}:` + return Object.entries(backgroundTasks.value) + .filter(([id]) => id.startsWith(prefix)) + .map(([, task]) => task) + .sort((a, b) => { + const stepOf = (t: BackgroundTask) => STEP_ORDER.indexOf(t.id.split(':')[1] || '') + return stepOf(a) - stepOf(b) + }) + } + + const assetProgress = (assetId: string): { step: string; percent: number } | null => { + const running = assetTasks(assetId).find((t) => t.state === 'running') + if (!running) return null + return { step: running.id.split(':')[1] || running.name, percent: running.progress ?? 0 } + } + + // ===== Persistence ===== + const markDirty = () => { + dirty.value = true + scheduleAutosave() + } + + const scheduleAutosave = () => { + if (autosaveTimer) clearTimeout(autosaveTimer) + autosaveTimer = setTimeout(() => persistDoc(), 800) + } + + const persistDoc = async () => { + if (!threadId.value || !doc.value || !dirty.value) return + saveInFlight.value = true + try { + const clipSelections: Record = {} + for (const asset of doc.value.media) { + for (const clip of asset.clips) { + clipSelections[clip.id] = clip.selected + } + } + const patch = JSON.parse(JSON.stringify({ + tracks: doc.value.tracks, + timeline: doc.value.timeline, + timelineMeta: doc.value.timelineMeta, + selection: doc.value.selection, + activePersonaId: doc.value.activePersonaId, + customPersonas: doc.value.customPersonas, + clipSelections + })) + await api.saveEditorDoc({ threadId: threadId.value, patch }) + dirty.value = false + } catch (error) { + console.error('[editorStore] Failed to persist editor doc:', error) + } finally { + saveInFlight.value = false + } + } + + // ===== Loading ===== + const loadProject = async (id: string): Promise => { + loading.value = true + try { + const loaded: Thread | null = await api.getThread(id) + if (!loaded || loaded.type !== 'editor' || !loaded.editor) { + return false + } + threadId.value = id + thread.value = loaded + doc.value = JSON.parse(JSON.stringify(loaded.editor)) + dirty.value = false + + backgroundTasks.value = (await api.getBackgroundTasks(id)) || {} + + // Detect assets interrupted by an app quit mid-preprocess: persisted as + // 'running' but with no live task process. Any incoming task update for + // the asset clears the flag. + interruptedAssets.value = new Set( + (doc.value?.media || []) + .filter((a) => a.preprocessState === 'running') + .map((a) => a.id) + ) + + // Fire-and-forget: the scenedetect/yt-dlp binary checks spawn processes + // and can take seconds — never block first paint on them. + api.checkSystemRequirements() + .then((reqs: any) => { + deps.value = { + ffmpeg: reqs?.ffmpegAvailable !== false, + scenedetect: reqs?.scenedetectAvailable !== false, + ytDlp: reqs?.ytDlpAvailable !== false + } + }) + .catch(() => { /* non-blocking */ }) + + if (!selectedAssetId.value && assets.value.length > 0) { + selectedAssetId.value = assets.value[0].id + } + return true + } finally { + loading.value = false + } + } + + // ===== Media actions (M1) ===== + const upsertAsset = (incoming: MediaAsset) => { + if (!doc.value) return + const index = doc.value.media.findIndex((a) => a.id === incoming.id) + if (index === -1) { + doc.value.media.push(incoming) + } else { + doc.value.media[index] = mergeMainOwnedAsset(doc.value.media[index], incoming) + } + } + + const addLocalMedia = async () => { + if (!threadId.value) return + const result = await api.selectVideo() + if (!result?.path) return + const asset: MediaAsset | null = await api.addMediaAsset({ + threadId: threadId.value, + filePath: result.path, + name: result.name + }) + if (asset) { + upsertAsset(asset) + selectedAssetId.value = asset.id + } + } + + const importUrl = async (url: string, resolution?: string) => { + if (!threadId.value) return + try { + const asset: MediaAsset | null = await api.importMediaUrl({ + threadId: threadId.value, + url, + resolution + }) + if (asset) { + delete urlImports.value[asset.id] + upsertAsset(asset) + selectedAssetId.value = asset.id + } + return asset + } catch (error) { + console.error('[editorStore] URL import failed:', error) + throw error + } + } + + const removeAsset = async (assetId: string) => { + if (!threadId.value || !doc.value) return + const confirmed = await api.showConfirmation({ + title: 'Remove media', + message: 'Remove this media and all its clips from the project?', + detail: 'The imported copy and generated thumbnails will be deleted. The original file on disk is not affected.', + buttons: ['Cancel', 'Remove'] + }) + if (!confirmed || confirmed.response !== 1) return + const success = await api.removeMediaAsset({ threadId: threadId.value, assetId }) + if (success) { + doc.value.media = doc.value.media.filter((a) => a.id !== assetId) + if (selectedAssetId.value === assetId) selectedAssetId.value = doc.value.media[0]?.id || null + if (selectedClip.value?.sourceAssetId === assetId) selectedClipId.value = null + } + } + + const retryAsset = async (assetId: string, steps?: string[]) => { + if (!threadId.value) return + interruptedAssets.value.delete(assetId) + await api.preprocessMedia({ threadId: threadId.value, assetId, steps }) + } + + const describeAsset = async (assetId: string) => { + if (!threadId.value) return + await api.preprocessMedia({ threadId: threadId.value, assetId, steps: ['descriptions'] }) + } + + // ===== Selection ===== + const selectAsset = (assetId: string | null) => { + selectedAssetId.value = assetId + selectedClipId.value = null + } + + const selectClip = (clipId: string | null) => { + selectedClipId.value = clipId + } + + const toggleClipSelected = (clipId: string) => { + if (!doc.value) return + for (const asset of doc.value.media) { + const clip = asset.clips.find((c) => c.id === clipId) + if (clip) { + clip.selected = !clip.selected + markDirty() + return + } + } + } + + // ===== Ownership-split merge for thread-updated echoes ===== + const mergeMainOwnedAsset = (local: MediaAsset, remote: MediaAsset): MediaAsset => { + // Take main-owned fields from remote; preserve renderer-owned clip.selected + // while an autosave is pending (remote may be a pre-save echo). + const preserveSelected = dirty.value || saveInFlight.value + const localSelected = new Map(local.clips.map((c) => [c.id, c.selected])) + return { + ...remote, + clips: remote.clips.map((clip) => ({ + ...clip, + selected: preserveSelected && localSelected.has(clip.id) + ? (localSelected.get(clip.id) as boolean) + : clip.selected + })) + } + } + + const onThreadUpdatedMerge = (updated: Thread) => { + if (!threadId.value || updated.id !== threadId.value) return + thread.value = updated + if (!updated.editor || !doc.value) return + + const remoteMedia = updated.editor.media || [] + const remoteIds = new Set(remoteMedia.map((a) => a.id)) + + // Merge main-owned media state + const merged: MediaAsset[] = remoteMedia.map((remote) => { + const local = doc.value!.media.find((a) => a.id === remote.id) + if (remote.preprocessState !== 'running') interruptedAssets.value.delete(remote.id) + return local ? mergeMainOwnedAsset(local, remote) : remote + }) + // Keep local-only assets (mid-creation, not yet broadcast) + for (const local of doc.value.media) { + if (!remoteIds.has(local.id)) { + const stillImporting = urlImports.value[local.id] !== undefined + if (stillImporting) merged.push(local) + } + } + doc.value.media = merged + + // Renderer-owned doc fields: only accept remote when no local edit is pending + if (!dirty.value && !saveInFlight.value) { + doc.value.tracks = updated.editor.tracks + doc.value.timeline = updated.editor.timeline + doc.value.timelineMeta = updated.editor.timelineMeta + doc.value.selection = updated.editor.selection + doc.value.activePersonaId = updated.editor.activePersonaId + doc.value.customPersonas = updated.editor.customPersonas + } + + if (selectedAssetId.value && !doc.value.media.some((a) => a.id === selectedAssetId.value)) { + selectedAssetId.value = doc.value.media[0]?.id || null + } + } + + // ===== Singleton IPC listeners (registered once, like videoStore) ===== + if (typeof window !== 'undefined' && api) { + api.onBackgroundTaskUpdate((data: { threadId: string; taskId: string; task: BackgroundTask }) => { + if (data.threadId !== threadId.value) return + backgroundTasks.value = { ...backgroundTasks.value, [data.taskId]: data.task } + const assetId = data.taskId.split(':')[0] + if (assetId) interruptedAssets.value.delete(assetId) + }) + + api.onThreadUpdated((updated: Thread) => { + try { + onThreadUpdatedMerge(updated) + } catch (error) { + console.error('[editorStore] thread-updated merge failed:', error) + } + }) + + if (api.onEditorImportProgress) { + api.onEditorImportProgress((data: { threadId: string; assetId: string; url?: string; percent: number }) => { + if (data.threadId !== threadId.value) return + urlImports.value = { + ...urlImports.value, + [data.assetId]: { url: data.url || '', percent: data.percent } + } + }) + } + } + + return { + // state + threadId, + thread, + doc, + backgroundTasks, + selectedAssetId, + selectedClipId, + urlImports, + interruptedAssets, + dirty, + saveInFlight, + deps, + loading, + // computeds + assets, + isEmpty, + selectedAsset, + selectedClip, + // helpers + assetTasks, + assetProgress, + // actions + loadProject, + addLocalMedia, + importUrl, + removeAsset, + retryAsset, + describeAsset, + selectAsset, + selectClip, + toggleClipSelected, + markDirty, + persistDoc + } +}) diff --git a/src/shared/types.ts b/src/shared/types.ts index bd441b3..d56daa8 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -70,10 +70,12 @@ export interface Message { createdAt: number; } +export type ThreadType = 'video' | 'image' | 'editor' + export interface Thread { id: string title: string - type?: 'video' | 'image' // New + type?: ThreadType videoPath?: string // Now optional preprocessing: { /** @@ -158,10 +160,159 @@ export interface Thread { videoMetadata?: VideoMetadata usageHistory?: UsageRecord[] missing?: boolean + /** Present only when type === 'editor' — the timeline editor document. */ + editor?: EditorDocument createdAt: number updatedAt: number } +// ============================================================ +// Timeline Video Editor (see video-editor-prd.md §6) +// Canonical time unit is SECONDS. SRT strings only appear at the +// boundary with legacy generateTimeline/assembleVideo. +// ============================================================ + +export interface EditorDocument { + schemaVersion: 1 + media: MediaAsset[] // imported sources + tracks: Track[] // lane definitions + timeline: TimelineItem[] // placements across tracks — the live EDL + timelineMeta: TimelineMeta + activePersonaId: string + customPersonas?: EditorPersona[] // project-local personas + turns: PromptTurn[] // prompt-request log (no snapshots) + historyRef: EditorHistoryRef // pointer into the sidecar history file + selection?: { clipIds?: string[]; itemIds?: string[] } +} + +export interface TimelineMeta { + fps: number + width: number + height: number + duration: number // seconds; derived = max(item.timelineStart + item.duration) + aspectRatio?: string // "16:9" | "9:16" +} + +export type MediaKind = 'video' | 'image' | 'audio' + +export type MediaPreprocessState = 'pending' | 'running' | 'completed' | 'error' + +export interface MediaAsset { + id: string + kind: MediaKind + name: string + originalPath: string // absolute; served via media:// + proxyPath?: string // 480p proxy === lowResVideoPath + metadata?: VideoMetadata + /** Reuses the EXACT shape of Thread['preprocessing'] so existing phases run per-asset. */ + preprocessing: Thread['preprocessing'] + preprocessTasks?: Record + preprocessState?: MediaPreprocessState + preprocessError?: string + clips: Clip[] // derived from scene detection + filmstrip?: FilmstripEntry[] + createdAt: number +} + +export interface FilmstripEntry { time: number; thumbnailPath: string } + +export interface Clip { // a selectable scene piece + id: string + sourceAssetId: string + index: number // 1-based, mirrors Scene/EnrichedTimelineSegment ordering + in: number // seconds into source (Scene.startTime) + out: number // seconds into source (Scene.endTime) + duration: number // out - in + thumbnailPath?: string + visual?: string // === EnrichedTimelineSegment.visual — context-preview text + text?: string // transcript excerpt overlapping [in,out] + selected: boolean // media-panel multi-select + masterSegmentIndex?: number // back-reference into the enriched master timeline +} + +export type TrackKind = 'video' | 'audio' | 'overlay' | 'text' + +export interface Track { + id: string + kind: TrackKind // 'overlay' | 'text' = "other objects for later" + name: string + order: number // stacking order (0 = bottom video) + muted: boolean + locked: boolean + hidden: boolean + height: number // px in the timeline UI +} + +export interface TimelineItem { // an instance of a clip placed on a track + id: string + trackId: string + sourceAssetId: string + sourceClipId?: string // set when dragged from a detected Clip + masterSegmentIndex?: number // present only for a whole, un-split master scene + timelineStart: number // seconds on the sequence timeline + in: number // seconds into source (trim start) + out: number // seconds into source (trim end) + speed: number // constant playback rate (default 1.0); retime tool + preservePitch: boolean // default true — audio retimed with atempo keeps pitch + duration: number // ON-TIMELINE duration = (out - in) / speed + label?: string + gain?: number // audio gain multiplier (default 1.0) + muted?: boolean + // ---- Stubs (deferred: effects/transitions/transforms/text) ---- + transform?: TimelineItemTransform + effects?: EffectRef[] + transition?: { in?: TransitionRef; out?: TransitionRef } + text?: TextOverlaySpec +} + +export interface TimelineItemTransform { x?: number; y?: number; scale?: number; rotation?: number } +export interface EffectRef { id: string; kind: string; params?: Record } // stub +export interface TransitionRef { kind: string; duration: number } // stub +export interface TextOverlaySpec { content: string; style?: Record } // stub + +export interface EditorPersona { + id: string + name: string + icon: string // emoji or Tabler icon id + description: string + systemPrompt: string // the whole backing for v1 + builtin: boolean // seeded personas can't be deleted, only cloned + tone?: string + mode?: 'longform' | 'summarize' + defaults?: { targetDurationSec?: number | null; aspectRatio?: string; pacing?: 'tight' | 'balanced' | 'relaxed' } + featureSets?: FeatureSetRef[] // deferred; always [] in v1 +} + +export interface FeatureSetRef { id: string; name: string } // stub + +export interface EditorHistoryRef { + currentStepId: string + stepCount: number +} + +export interface PromptTurn { + id: string + personaId: string + prompt: string + baseStepId: string + resultStepId?: string + status: 'pending' | 'running' | 'completed' | 'error' + error?: string + diff?: TimelineDiff + usage?: Usage + cost?: number + createdAt: number +} + +export type TimelineDiff = { + schemaVersion: number + addItems?: TimelineItem[] + removeItemIds?: string[] + updateItems?: Array<{ id: string } & Partial> + addTracks?: Track[] + removeTrackIds?: string[] +} + export interface TimelineSegment { index: number start: string diff --git a/tailwind.config.js b/tailwind.config.js index 62b1d0f..8786628 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -39,6 +39,7 @@ export default { 'fade-in-up': 'fadeInUp 0.5s ease-out', 'fade-in': 'fadeIn 0.5s ease-out', 'pulse-soft': 'pulseSoft 2s infinite ease-in-out', + 'indeterminate': 'indeterminate 1.2s infinite ease-in-out', }, keyframes: { fadeInUp: { @@ -53,6 +54,10 @@ export default { '0%, 100%': { opacity: '1' }, '50%': { opacity: '0.8' }, }, + indeterminate: { + '0%': { transform: 'translateX(-100%)' }, + '100%': { transform: 'translateX(100%)' }, + }, }, }, }, diff --git a/video-editor-prd.md b/video-editor-prd.md new file mode 100644 index 0000000..9b171ca --- /dev/null +++ b/video-editor-prd.md @@ -0,0 +1,888 @@ +# FrameFlow — Timeline Video Editor PRD + +FrameFlow today edits video through a single surface: an AI-driven node-graph "chat editor" where the user talks to Gemini, the model writes a timeline blueprint, and FFmpeg assembles a single-source cut. This PRD specifies a **second, coexisting surface** — a traditional CapCut/Premiere/DaVinci-style **timeline video editor** with a media browser, a program monitor, a multi-track timeline, direct manual editing, and a prompt box driven by swappable AI **editor personas**. It reuses FrameFlow's existing scene-detection, per-scene visual-description enrichment, background-task, `media://` playback, and thread-persistence machinery, while introducing a per-media preprocessing model, a timeline document, a persona library, and — the single largest new engineering lift — a multi-source/multi-track FFmpeg render engine that the current single-source `assembleVideo` cannot provide. + +| | | +|---|---| +| **Status** | Draft | +| **Owner** | navidshad72@gmail.com | +| **Date** | 2026-07-18 | +| **App** | FrameFlow (Electron + Vue 3 + Vite, TypeScript) | +| **Feature** | Traditional timeline video editor (new surface, alongside the AI graph editor) | + +--- + +## 1. Overview & Motivation + +**What this is.** A new, full-bleed editor page (route `/editor/:id`) reachable from a "Video Editor" tile on the Home grid. It presents the layout professionals already know — a media/browser panel on the left, a large preview/program monitor in the center, an inspector on the right, and a multi-track timeline across the bottom — plus a docked prompt bar carrying a selectable AI persona. The user imports one or more videos, FrameFlow breaks each into selectable scene "pieces," the user drags pieces onto tracks or asks a persona to build/refine the edit by prompt, and finally exports a rendered file. + +**It is length-agnostic.** This editor is **not** a summarizer. It targets full-length editorial work as much as short cuts: a 3-minute highlight *and* a 90-minute podcast, a vlog, a webinar, or a lecture kept at (or near) full length. A summary is just one possible output; the more common long-form goals are *cleaning* (remove filler/silence/dead air), *retiming* (speed a section up or slow it down), *reordering*, *chaptering*, and *trimming* — all without necessarily shortening the whole piece. Everything below (scene splitting, personas, budgets, AI context) is designed so a multi-hour timeline works, not only a short one (see the scalability treatment in §8). + +**Why add it alongside the graph editor.** The existing graph editor (`src/renderer/src/pages/GraphChatPage.vue`) is outcome-oriented: you describe what you want and Gemini decides the whole cut. It is excellent for "summarize this lecture to 3 minutes," but it offers **no direct manipulation** — no dragging a clip, trimming a frame, muting a track, or placing two sources side by side. Real editing work needs a spatial, manual surface where time runs left→right and parallel content stacks as tracks. Rather than bolt manual controls onto the node graph (Vue Flow is the wrong interaction model for a 1-D timeline — see §7), we add a purpose-built editor that treats the timeline as a shared document both the human and the AI mutate. + +**Who it is for.** +- **Long-form editors** — podcasters, vloggers, course/webinar creators, streamers cutting VODs — who keep most of the runtime and want to *tighten* it: remove filler and silence, retime slow stretches, reorder segments, add chapters. This is the primary audience, and the personas and budgets below are shaped for it. +- **Summarization users** who like FrameFlow's AI cuts but want to hand-tweak the result (nudge a boundary, drop one more scene, reorder). +- **Prosumer editors** who expect a familiar NLE layout and manual tools, but want an AI co-editor on tap. +- **Semantic editors** — FrameFlow's differentiator: every selectable piece already carries a one-line **visual description** (`EnrichedTimelineSegment.visual`), so users can see and reason about clips by meaning, not just thumbnails — which matters most on a long timeline where thumbnails alone are hard to scan. + +The two paradigms stay cleanly separated by a single `Thread.type` discriminator and separate routes; they share persistence, preprocessing, background tasks, and the Gemini pipeline. + +--- + +## 2. Goals & Non-Goals + +### Goals (v1) +- Ship a CapCut/Premiere-style four-zone layout (browser, monitor, timeline, inspector) that reads as a real editor **(feature 1)**. +- **Serve full-length editorial work, not only summaries** — a multi-hour podcast/vlog/webinar can be imported, cleaned, retimed, reordered, and chaptered while keeping most of its runtime. +- Media panel to add local files and URL/YouTube imports; each imported video runs the existing preprocessing steps and is auto-split into selectable scene pieces **(feature 2)**. +- Preview/program monitor and a multi-track timeline; drag pieces from the browser onto tracks **(feature 3)**. +- Timeline view toggle: **frames-preview** (thumbnail filmstrip) ⇄ **context-preview** (per-scene visual/text descriptions) **(feature 4)**. +- Track types: **video**, **audio**, and **overlay/"other objects"** lanes present in the model and UI (overlay items inert in v1) **(feature 5)**. +- A prompt box for AI-driven edits **and** first-class manual editing, sharing one timeline document **(feature 6)**. Manual tools include **section timing control** — trim, ripple-trim, **cut (split + delete)**, and **retime/speed** to reduce *or extend* a section's on-timeline duration, plus move, snap, and zoom. +- **Constant per-clip speed (retime)** so a section can be sped up (compress a slow stretch) or slowed down (stretch it out), with a numeric target-duration entry per item. (Keyframed *speed ramps* remain deferred — see non-goals.) +- Editor **personas** covering **both long-form editorial and summarization**: predefined (read-only, cloneable) + user-defined; each persona is a system-prompt/text profile for v1 **(feature 7)**. +- Persona schema forward-compatible with future **feature-sets**, present as a typed stub **(feature 8)**. +- Full keyboard-driven editing plus baseline accessibility for the timeline canvas (§5.11). +- Export a rendered file (any length), reusing the existing single-source fast path where the timeline is degenerate and introducing a multi-source render path incrementally. +- **Scale to long content:** virtualized timeline/clip tray, lazy filmstrips, opt-in enrichment, and **windowed AI context** so a multi-hour project stays responsive and stays within the model's context window (§8). + +### Non-Goals (v1 — explicitly deferred) +- **Full NLE multi-track compositing.** v1 renders a single primary video track (+ a mixed audio output) robustly; PiP overlay stacking, per-clip transforms, and blend/opacity are P2+ (§9 M5). +- **Effects & transitions engine.** No LUTs, filters, color grading, `xfade`/`acrossfade` transitions, or keyframing. `TimelineItem.effects/transition/transform` ship as typed stubs only. **Exception:** a single **constant per-clip speed** (`TimelineItem.speed`, applied via `setpts`/`atempo` at export) *is* in v1 — it's the retime control (§5.6); keyframed/animated **speed ramps** are the part that stays deferred. +- **Silence/filler auto-removal as a fully automatic pass.** v1 offers silence detection as an *assistive* auto-cut the user reviews (§5.6), not a one-click destructive "clean the whole podcast" batch; transcript-word-level filler removal ("um"/"uh") is deferred to v2 with the effects engine. +- **Persona feature-sets.** Personas are text (system prompt + a few typed defaults) in v1. `featureSets` is a stubbed `[]` — no effect-group binding **(feature 8 is explicitly deferred beyond the schema stub)**. +- **Pro trim modes.** No roll/slip/slide, no separate source monitor with full 3-point editing, no JKL/dynamic trimming, no nested sequences. (Split, drag-trim, ripple-delete, snap, zoom are in.) +- **Collaboration / multi-user / cloud sync.** Local desktop, single user, single project at a time. +- **Real-time composited scrubbing preview.** The monitor previews via EDL playback of proxies (single `