diff --git a/.gitignore b/.gitignore index d816e79..f658e18 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,19 @@ release .DS_Store .npmrc +.playwright-mcp/page-2026-07-18T20-27-09-682Z.png +.playwright-mcp/page-2026-07-18T20-27-30-442Z.yml +.playwright-mcp/page-2026-07-18T20-28-21-488Z.png +.playwright-mcp/page-2026-07-18T20-28-37-637Z.yml +.playwright-mcp/page-2026-07-18T20-29-00-648Z.yml +.playwright-mcp/page-2026-07-18T20-29-06-561Z.png +.playwright-mcp/page-2026-07-18T20-29-30-963Z.png +.playwright-mcp/page-2026-07-18T20-30-00-042Z.yml +.playwright-mcp/page-2026-07-18T20-30-13-245Z.png +.playwright-mcp/page-2026-07-18T20-33-32-651Z.png +.playwright-mcp/page-2026-07-18T20-35-28-430Z.png +.playwright-mcp/page-2026-07-18T21-05-49-608Z.yml +.playwright-mcp/page-2026-07-18T21-06-19-346Z.yml +.playwright-mcp/page-2026-07-18T21-06-51-089Z.yml +.playwright-mcp/page-2026-07-18T21-07-01-786Z.yml +.playwright-mcp/page-2026-07-18T21-07-06-202Z.yml diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..fcbfcf3 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "playwright": { + "command": "npx", + "args": [ + "-y", + "@playwright/mcp@latest", + "--cdp-endpoint", + "http://127.0.0.1:9222" + ] + } + } +} diff --git a/package-lock.json b/package-lock.json index c9a8bda..d57e268 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "ffprobe-static": "^3.1.0", "fluent-ffmpeg": "^2.1.3", "markdown-it": "^14.1.1", - "pilotui": "^1.28.1", + "pilotui": "^1.29.1", "pinia": "^3.0.4", "uuid": "^13.0.0", "vue": "^3.4.21", @@ -1026,12 +1026,12 @@ } }, "node_modules/@storybook/builder-vite": { - "version": "8.6.15", - "resolved": "https://registry.npmjs.org/@storybook/builder-vite/-/builder-vite-8.6.15.tgz", - "integrity": "sha512-9Y05/ndZE6/eI7ZIUCD/QtH2htRIUs9j1gxE6oW0zRo9TJO1iqxfLNwgzd59KEkId7gdZxPei0l+LGTUGXYKRg==", + "version": "8.6.18", + "resolved": "https://registry.npmjs.org/@storybook/builder-vite/-/builder-vite-8.6.18.tgz", + "integrity": "sha512-XLqnOv4C36jlTd4uC8xpWBxv+7GV4/05zWJ0wAcU4qflorropUTirt4UQPGkwIzi+BVAhs9pJj+m4k0IWJtpHg==", "license": "MIT", "dependencies": { - "@storybook/csf-plugin": "8.6.15", + "@storybook/csf-plugin": "8.6.18", "browser-assert": "^1.2.1", "ts-dedent": "^2.0.0" }, @@ -1040,7 +1040,7 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "storybook": "^8.6.15", + "storybook": "^8.6.18", "vite": "^4.0.0 || ^5.0.0 || ^6.0.0" } }, @@ -1077,9 +1077,9 @@ } }, "node_modules/@storybook/core/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "license": "ISC", "peer": true, "bin": { @@ -1090,9 +1090,9 @@ } }, "node_modules/@storybook/csf-plugin": { - "version": "8.6.15", - "resolved": "https://registry.npmjs.org/@storybook/csf-plugin/-/csf-plugin-8.6.15.tgz", - "integrity": "sha512-ZLz/mtOoE1Jj2lE4pK3U7MmYrv5+lot3mGtwxGb832tcABMc97j9O+reCVxZYc7DeFbBuuEdMT9rBL/O3kXYmw==", + "version": "8.6.18", + "resolved": "https://registry.npmjs.org/@storybook/csf-plugin/-/csf-plugin-8.6.18.tgz", + "integrity": "sha512-x1ioz/L0CwaelCkHci3P31YtvwayN3FBftvwQOPbvRh9qeb4Cpz5IdVDmyvSxxYwXN66uAORNoqgjTi7B4/y5Q==", "license": "MIT", "dependencies": { "unplugin": "^1.3.1" @@ -1102,7 +1102,7 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "storybook": "^8.6.15" + "storybook": "^8.6.18" } }, "node_modules/@storybook/theming": { @@ -1158,9 +1158,9 @@ } }, "node_modules/@tanstack/virtual-core": { - "version": "3.13.18", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.13.18.tgz", - "integrity": "sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg==", + "version": "3.17.4", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.4.tgz", + "integrity": "sha512-nGm5KteqxasUdThLc2izl6dHUqLv0LQj7Nuyo5gYalTPf/U8a9ermvsl7reT+6ioBW1l8WfpP/mcU338nLXpqw==", "license": "MIT", "funding": { "type": "github", @@ -1168,12 +1168,12 @@ } }, "node_modules/@tanstack/vue-virtual": { - "version": "3.13.18", - "resolved": "https://registry.npmjs.org/@tanstack/vue-virtual/-/vue-virtual-3.13.18.tgz", - "integrity": "sha512-6pT8HdHtTU5Z+t906cGdCroUNA5wHjFXsNss9gwk7QAr1VNZtz9IQCs2Nhx0gABK48c+OocHl2As+TMg8+Hy4A==", + "version": "3.13.32", + "resolved": "https://registry.npmjs.org/@tanstack/vue-virtual/-/vue-virtual-3.13.32.tgz", + "integrity": "sha512-E8OCutx7QnwZdvpJijz0Q2PHsYDWBWjnGr3TvgWiqxTU35jB1kVhtkd93scRV7tTFuId2tg3x2iFiw+IE4evjQ==", "license": "MIT", "dependencies": { - "@tanstack/virtual-core": "3.13.18" + "@tanstack/virtual-core": "3.17.4" }, "funding": { "type": "github", @@ -1193,15 +1193,6 @@ "node": ">= 10" } }, - "node_modules/@trysound/sax": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", - "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", - "license": "ISC", - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -1753,9 +1744,9 @@ "license": "MIT" }, "node_modules/acorn": { - "version": "8.15.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", - "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", + "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", "license": "MIT", "bin": { "acorn": "bin/acorn" @@ -2491,15 +2482,15 @@ } }, "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", + "integrity": "sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ==", "license": "MIT", "peer": true, "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "get-intrinsic": "^1.3.0", "set-function-length": "^1.2.2" }, "engines": { @@ -5256,9 +5247,9 @@ "license": "MIT" }, "node_modules/lodash-es": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.23.tgz", - "integrity": "sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", + "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", "license": "MIT" }, "node_modules/lodash.clonedeep": { @@ -5295,6 +5286,7 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", "license": "MIT" }, "node_modules/lodash.isplainobject": { @@ -5888,9 +5880,10 @@ } }, "node_modules/pilotui": { - "version": "1.28.1", - "resolved": "https://registry.npmjs.org/pilotui/-/pilotui-1.28.1.tgz", - "integrity": "sha512-S9MJt00HEBUUBk775bOSTWkkkVjjce7lFsvOC+q0ovqfAo3/x0vGHkSUyzNrj2nelAyoZQQuCYYsA1Pn20Ylow==", + "version": "1.29.1", + "resolved": "https://registry.npmjs.org/pilotui/-/pilotui-1.29.1.tgz", + "integrity": "sha512-DwAv34kB1K+17OR/bsJly+yE/I19mHkbQstbaThbmiu4vA/o2wwbj6dPQFbpgwkoJEFFIY7ojzCt9cy4evcfpw==", + "license": "MIT", "dependencies": { "@headlessui/vue": "^1.7.23", "@storybook/builder-vite": "^8.4.5", @@ -6788,9 +6781,9 @@ } }, "node_modules/recast": { - "version": "0.23.11", - "resolved": "https://registry.npmjs.org/recast/-/recast-0.23.11.tgz", - "integrity": "sha512-YTUo+Flmw4ZXiWfQKGcwwc11KnoRAYgzAE2E7mXKCjSviTKShtxBsN6YUUBB2gtaBzKzeKunxhUwNHQuRryhWA==", + "version": "0.23.12", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.23.12.tgz", + "integrity": "sha512-dEWRjcINDu/F4l2dYx57ugBtD7HV9KXESyxhzw/MqWLeglJrsjJKqACPyUPg+6AF8mIgm+Zi0dZ3ACoIg+QtpA==", "license": "MIT", "peer": true, "dependencies": { @@ -7037,10 +7030,9 @@ } }, "node_modules/sax": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.4.tgz", - "integrity": "sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==", - "dev": true, + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", "license": "BlueOak-1.0.0", "engines": { "node": ">=11.0.0" @@ -7236,6 +7228,7 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==", + "deprecated": "Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility", "license": "MIT" }, "node_modules/stat-mode": { @@ -7427,17 +7420,17 @@ } }, "node_modules/svgo": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.0.tgz", - "integrity": "sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.3.tgz", + "integrity": "sha512-5EZD0pafXX6PphdwOGCiVLDSaV1xyuQao2blHajHLsPxr07q4mmEjdtXEWgG07ae2mIz8Ex2CDXNCTiXhy3Khw==", "license": "MIT", "dependencies": { - "@trysound/sax": "0.2.0", "commander": "^7.2.0", "css-select": "^4.1.3", "css-tree": "^1.1.3", "csso": "^4.2.0", "picocolors": "^1.0.0", + "sax": "^1.5.0", "stable": "^0.1.8" }, "bin": { @@ -7457,9 +7450,9 @@ } }, "node_modules/sweetalert2": { - "version": "11.26.18", - "resolved": "https://registry.npmjs.org/sweetalert2/-/sweetalert2-11.26.18.tgz", - "integrity": "sha512-3O5feBqV+hTIOwCRKGuZGHosjiuBAKP/vpBl6vKFZeVYfCUGdXqXuuidn6YXHan3f6e62UdmnjwJBt8UtDVBhg==", + "version": "11.26.25", + "resolved": "https://registry.npmjs.org/sweetalert2/-/sweetalert2-11.26.25.tgz", + "integrity": "sha512-+hunCOJdJ6FLj04T9YSLvvZXRjsvIkTeTKP2e4VF8CaBias961BTnWiSFAy7F/CM5eq3QK2Rraoc5Gzftslvkg==", "license": "MIT", "funding": { "type": "individual", @@ -7688,9 +7681,9 @@ } }, "node_modules/ts-dedent": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", - "integrity": "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", + "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", "license": "MIT", "engines": { "node": ">=6.10" @@ -8124,14 +8117,14 @@ } }, "node_modules/which-typed-array": { - "version": "1.1.20", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.20.tgz", - "integrity": "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==", + "version": "1.1.22", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.22.tgz", + "integrity": "sha512-fvO4ExWMFsqyhG3AiPAObMuY1lxaqgYcxbc49CNdWDDECOJNgQyvsOWVwbZc+qf3rzRtxojBK+CMEv0Ld5CYpw==", "license": "MIT", "peer": true, "dependencies": { "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.8", + "call-bind": "^1.0.9", "call-bound": "^1.0.4", "for-each": "^0.3.5", "get-proto": "^1.0.1", diff --git a/package.json b/package.json index 7ddd94f..9f5468d 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "typecheck:web": "vue-tsc -p tsconfig.web.json --noEmit", "typecheck": "npm run typecheck:node && npm run typecheck:web || true", "start": "electron-vite preview", + "start:debug": "electron-vite build && electron out/main/index.js --remote-debugging-port=9222", "dev": "electron-vite dev", "build": "electron-vite build", "postinstall": "electron-builder install-app-deps", @@ -33,7 +34,7 @@ "ffprobe-static": "^3.1.0", "fluent-ffmpeg": "^2.1.3", "markdown-it": "^14.1.1", - "pilotui": "^1.28.1", + "pilotui": "^1.29.1", "pinia": "^3.0.4", "uuid": "^13.0.0", "vue": "^3.4.21", diff --git a/src/main/constants/gemini.ts b/src/main/constants/gemini.ts index 97f33af..b2754e1 100644 --- a/src/main/constants/gemini.ts +++ b/src/main/constants/gemini.ts @@ -96,6 +96,7 @@ export const DEFAULT_MODEL_SETTINGS: ModelSettings = { 'image-extraction': GEMINI_MODEL_2_5_FLASH_LITE, 'image-intent': GEMINI_MODEL_2_5_FLASH, 'image-generation': GEMINI_MODEL_3_1_FLASH_IMAGE_PREVIEW, - 'image-upscale': GEMINI_MODEL_3_1_FLASH_IMAGE_PREVIEW + 'image-upscale': GEMINI_MODEL_3_1_FLASH_IMAGE_PREVIEW, + 'editor-edit': GEMINI_MODEL_3_FLASH_PREVIEW } } diff --git a/src/main/constants/paths.ts b/src/main/constants/paths.ts index 0122318..d95b32c 100644 --- a/src/main/constants/paths.ts +++ b/src/main/constants/paths.ts @@ -6,5 +6,20 @@ export const THREAD_DIRS = { GENERATED_VIDEOS: 'generated-videos', AUDIO: 'audio', VIDEO: 'video', - TRANSCRIPTS: 'transcripts' + TRANSCRIPTS: 'transcripts', + // Timeline editor: per-asset artifact root (tempDir/media//...) + MEDIA: 'media', + // Timeline editor: export workdirs + final renders (tempDir/exports/...) + EXPORTS: 'exports' +} as const + +// Subdirectories inside tempDir/media// for the timeline editor. +// FRAMES/ANALYSIS/AUDIO intentionally reuse THREAD_DIRS names so pipeline +// phases run per-asset when given an asset-scoped tempDir. +export const ASSET_DIRS = { + SOURCE: 'source', + PROXY: 'proxy', + ANALYSIS: THREAD_DIRS.ANALYSIS, + FRAMES: THREAD_DIRS.FRAMES, + AUDIO: THREAD_DIRS.AUDIO } as const diff --git a/src/main/constants/personas.ts b/src/main/constants/personas.ts new file mode 100644 index 0000000..76fcdb6 --- /dev/null +++ b/src/main/constants/personas.ts @@ -0,0 +1,196 @@ +import type { EditorPersona } from '@shared/types' + +/** + * Built-in editor personas (video-editor-prd.md §5.8). + * NEVER persisted to settings.json — merged with the user's saved personas at + * read time, so prompt improvements ship with app updates. Long-form personas + * carry `targetDurationSec: null` = length-preserving editorial mode; the + * summarize group targets a shorter duration. + */ + +export const DEFAULT_PERSONA_ID = 'podcast-editor' + +export const BUILTIN_PERSONAS: EditorPersona[] = [ + // ===== Long-form (length-preserving editorial) ===== + { + id: 'podcast-editor', + name: 'Podcast Editor', + icon: '🎙️', + description: 'Keeps the full conversation; removes dead air and false starts; tightens rambling stretches.', + builtin: true, + mode: 'longform', + tone: 'neutral', + defaults: { targetDurationSec: null, pacing: 'relaxed' }, + featureSets: [], + systemPrompt: [ + 'You are a seasoned podcast editor. Your job is to keep the FULL conversation', + 'intact while making it tighter and more listenable.', + '- Preserve every distinct point, story, and speaker turn.', + '- Remove dead air, long silences, false starts, and duplicated takes.', + '- Where a stretch rambles but its content matters, prefer a gentle speed-up', + ' (retime via the speed field, up to about 1.3x) over deletion.', + '- Keep chronological order unless the user explicitly asks to reorder.', + '- Never cut for length alone: the goal is a cleaner version of the whole', + ' episode, not a shorter one.' + ].join('\n') + }, + { + id: 'longform-polish', + name: 'Long-Form Polisher', + icon: '🎬', + description: 'For vlogs, webinars, and streams: removes setup fumbles and dead time, smooths pacing.', + builtin: true, + mode: 'longform', + tone: 'warm', + defaults: { targetDurationSec: null, pacing: 'balanced' }, + featureSets: [], + systemPrompt: [ + 'You are polishing long-form footage (vlogs, webinars, streams, talks).', + '- Remove setup fumbles, technical interruptions, and stretches where', + ' nothing substantive happens.', + '- Keep the whole narrative and all substantive content.', + '- Smooth the pacing: trim slow intros, tighten transitions between topics.', + '- Reorder segments only when it clearly improves flow, and explain why in', + ' your rationale when you do.' + ].join('\n') + }, + { + id: 'silence-cleaner', + name: 'Silence & Filler Cleaner', + icon: '🧹', + description: 'Aggressively removes silence, dead air, and empty gaps; keeps all substantive content.', + builtin: true, + mode: 'longform', + tone: 'neutral', + defaults: { targetDurationSec: null, pacing: 'tight' }, + featureSets: [], + systemPrompt: [ + 'You specialize in cleaning silence and dead air out of recordings.', + '- Propose removing scenes whose descriptions indicate silence, empty', + ' pauses, dead air, filler, or no visible/audible activity, and scenes', + ' that are extremely short with no described content.', + '- Leave every scene with spoken or substantive content fully intact.', + '- Do not shorten meaningful segments; your only lever is removing the', + ' empty material between them.' + ].join('\n') + }, + { + id: 'chapter-organizer', + name: 'Chapter Organizer', + icon: '🗂️', + description: 'Keeps full length; segments the piece into labeled chapters at topic boundaries.', + builtin: true, + mode: 'longform', + tone: 'authoritative', + defaults: { targetDurationSec: null, pacing: 'relaxed' }, + featureSets: [], + systemPrompt: [ + 'You organize long recordings into chapters.', + '- Keep the full runtime; do not remove or retime content unless asked.', + '- Identify logical topic boundaries from the scene descriptions and', + ' propose markers (addMarkers) at each boundary with a short, clear label.', + '- Prefer 5-12 chapters for a typical long recording; merge minor topic', + ' shifts into their parent chapter.', + '- Reorder segments only if the user explicitly asks.' + ].join('\n') + }, + { + id: 'study-notes', + name: 'Lecture Study-Notes', + icon: '🎓', + description: 'Retains every scene with distinct instructional content; drops only silence and repetition.', + builtin: true, + mode: 'longform', + tone: 'authoritative', + defaults: { targetDurationSec: null, pacing: 'relaxed' }, + featureSets: [], + systemPrompt: [ + 'You are preparing lecture footage for studying.', + '- Retain every scene with distinct instructional content: explanations,', + ' examples, derivations, demonstrations, and summaries.', + '- Drop only silence, repetition, administrative asides, and off-topic', + ' tangents.', + '- Prioritize completeness over brevity and always preserve chronological', + ' order.' + ].join('\n') + }, + + // ===== Summarize (target a shorter duration) ===== + { + id: 'concise-summarizer', + name: 'Concise Summarizer', + icon: '✂️', + description: 'Cuts to the essential information; drops redundancy and dead air.', + builtin: true, + mode: 'summarize', + tone: 'neutral', + defaults: { targetDurationSec: 60, pacing: 'tight' }, + featureSets: [], + systemPrompt: [ + 'You produce concise summaries of longer footage.', + '- Keep only the scenes that advance the core message.', + '- Drop redundancy, dead air, and anything tangential.', + '- Prefer the shortest coherent edit that still tells the whole story.', + '- The result must stand alone: someone who never saw the original should', + ' understand it.' + ].join('\n') + }, + { + id: 'highlight-reel', + name: 'Highlight Reel', + icon: '⚡', + description: 'Selects the most visually striking, high-energy scenes with strong opening and closing beats.', + builtin: true, + mode: 'summarize', + tone: 'energetic', + defaults: { targetDurationSec: 30, pacing: 'tight' }, + featureSets: [], + systemPrompt: [ + 'You cut high-energy highlight reels.', + '- Select the most visually striking scenes: peaks, reactions, motion,', + ' and moments of impact, using the scene descriptions as your guide.', + '- Favor fast pacing with short clips.', + '- Open on a strong hook and end on a memorable closing beat.' + ].join('\n') + }, + { + id: 'storyteller', + name: 'Storyteller', + icon: '📖', + description: 'Assembles a narrative arc — setup, development, payoff — preserving emotional throughline.', + builtin: true, + mode: 'summarize', + tone: 'warm', + defaults: { targetDurationSec: 180, pacing: 'balanced' }, + featureSets: [], + systemPrompt: [ + 'You are a narrative editor building a story from footage.', + '- Assemble a clear arc: setup, development, payoff.', + '- Preserve context and the emotional throughline; keep transitions', + ' between selected scenes logical.', + '- Prefer scenes that carry narrative weight over merely pretty ones.' + ].join('\n') + }, + { + id: 'social-shorts', + name: 'Social / Vertical Shorts', + icon: '📱', + description: 'Vertical-friendly short: hook in the first seconds, one clear idea, rapid cuts.', + builtin: true, + mode: 'summarize', + tone: 'playful', + defaults: { targetDurationSec: 45, aspectRatio: '9:16', pacing: 'tight' }, + featureSets: [], + systemPrompt: [ + 'You cut short-form vertical video for social feeds.', + '- Hook the viewer within the first 3 seconds.', + '- One clear idea per short; drop everything else.', + '- Rapid cuts, no dead time, and end on a strong beat that invites a', + ' rewatch.' + ].join('\n') + } +] + +export function findBuiltinPersona(id: string): EditorPersona | undefined { + return BUILTIN_PERSONAS.find((p) => p.id === id) +} diff --git a/src/main/editor/assets.ts b/src/main/editor/assets.ts new file mode 100644 index 0000000..50db4e2 --- /dev/null +++ b/src/main/editor/assets.ts @@ -0,0 +1,407 @@ +import fs from 'fs' +import path from 'path' +import { v4 as uuidv4 } from 'uuid' +import type { Clip, MediaAsset, Thread } from '@shared/types' +import { threadManager } from '../threads' +import { getVideoMetadata, getAudioMetadata, sanitizeFilename } from '../ffmpeg' +import { ASSET_DIRS, THREAD_DIRS } from '../constants/paths' +import { abortAssetPreprocessing } from './preprocess' + +const AUDIO_EXTENSIONS = new Set([ + '.mp3', '.wav', '.m4a', '.aac', '.flac', '.ogg', '.oga', '.opus', '.wma' +]) + +/** Classify an imported source as audio or video by file extension. */ +export function detectMediaKind(filePath: string): 'audio' | 'video' { + return AUDIO_EXTENSIONS.has(path.extname(filePath).toLowerCase()) ? 'audio' : 'video' +} + +/** + * Probes a source and returns its real kind + metadata based on which streams + * exist. `hint` (from the extension) only decides which probe runs first so the + * common case costs one ffprobe; both are tried before giving up. + */ +async function probeMediaKind( + filePath: string, + hint: 'audio' | 'video' +): Promise<{ kind: 'audio' | 'video'; metadata: MediaAsset['metadata'] } | null> { + const tryVideo = async () => ({ kind: 'video' as const, metadata: await getVideoMetadata(filePath) }) + const tryAudio = async () => ({ kind: 'audio' as const, metadata: await getAudioMetadata(filePath) }) + const [first, second] = hint === 'audio' ? [tryAudio, tryVideo] : [tryVideo, tryAudio] + try { + return await first() + } catch { + try { + return await second() + } catch (error) { + console.error(`[editor] Failed to probe media ${filePath}:`, error) + return null + } + } +} + +/** + * Media-asset CRUD for the timeline editor. + * Every artifact of an asset lives under tempDir/media// so + * concurrent imports can never collide and removal is one rm -rf. + * All document writes go through threadManager.updateThreadWith (queued + * mutators) so parallel per-asset updates cannot clobber each other. + */ + +export function getAssetDir(thread: Thread, assetId: string): string { + return path.join(thread.tempDir, THREAD_DIRS.MEDIA, assetId) +} + +function ensureDir(dir: string) { + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) +} + +/** + * Imports a source file as a MediaAsset: copies (or moves) it into the + * asset's source/ dir, probes metadata, and persists the asset record. + * A failed probe persists an error-state asset rather than throwing — + * the UI shows it with a retry/remove affordance. + */ +export async function createMediaAsset( + threadId: string, + options: { sourcePath: string; name?: string; move?: boolean; assetId?: string; referenceInPlace?: boolean } +): Promise { + const thread = threadManager.getThread(threadId) + if (!thread || thread.type !== 'editor' || !thread.editor) return null + + if (!fs.existsSync(options.sourcePath) || fs.statSync(options.sourcePath).isDirectory()) { + throw new Error(`Invalid media source: "${options.sourcePath}" is not a file.`) + } + + const assetId = options.assetId || uuidv4() + const assetDir = getAssetDir(thread, assetId) + + const rawName = options.name || path.basename(options.sourcePath) + const fileName = sanitizeFilename(rawName) + + // Reference the user's file where it lives instead of copying it in. + // Copying a large local source (10–15 GB) with fs.copyFileSync blocks the + // main process and freezes the whole UI (including the file dialog). The + // editor only needs the path: proxy/thumbnails/export read originalPath + // directly, media:// serves any absolute path, removeAsset only clears the + // asset dir, and path-repair leaves out-of-tempDir paths untouched. URL + // imports still live under the asset dir (they download straight into it). + let originalPath: string + if (options.referenceInPlace) { + originalPath = options.sourcePath + } else { + const sourceDir = path.join(assetDir, ASSET_DIRS.SOURCE) + ensureDir(sourceDir) + originalPath = path.join(sourceDir, fileName) + if (path.resolve(options.sourcePath) !== path.resolve(originalPath)) { + if (options.move) { + fs.renameSync(options.sourcePath, originalPath) + } else { + fs.copyFileSync(options.sourcePath, originalPath) + } + } + } + + const asset: MediaAsset = { + id: assetId, + kind: 'video', + name: rawName, + originalPath, + preprocessing: {}, + preprocessState: 'pending', + clips: [], + createdAt: Date.now() + } + + // Kind is decided by what streams actually exist, not the extension — + // container formats (.webm/.mkv/.m4a) can be audio-only. The extension only + // picks which probe to try first so the common case is a single ffprobe. + const probed = await probeMediaKind(originalPath, detectMediaKind(originalPath)) + if (probed) { + asset.kind = probed.kind + asset.metadata = probed.metadata + } else { + asset.preprocessState = 'error' + asset.preprocessError = 'Could not read media metadata (corrupt or unsupported file).' + } + + await threadManager.updateThreadWith(threadId, (t) => { + if (!t.editor) return null + const patch: Partial = { editor: { ...t.editor, media: [...t.editor.media, asset] } } + // Auto-name the project after the first imported media while the title + // is still the default — mirrors how chat threads name from the video. + if (!t.title || t.title === 'Untitled Project') { + const derived = deriveProjectTitle(rawName) + if (derived) patch.title = derived + } + return patch + }) + + return asset +} + +/** Filename → a clean project title: drop the extension, tidy separators. */ +function deriveProjectTitle(fileName: string): string { + const base = fileName.replace(/\.[^./\\]+$/, '') // strip a trailing extension + return base.replace(/[_]+/g, ' ').trim().slice(0, 120) +} + +/** Merge a partial patch into one asset record (queued, safe under concurrency). */ +export function patchAsset( + threadId: string, + assetId: string, + patch: Partial | ((asset: MediaAsset) => Partial) +): Promise { + return threadManager.updateThreadWith(threadId, (thread) => { + if (!thread.editor) return null + const index = thread.editor.media.findIndex((a) => a.id === assetId) + if (index === -1) return null + + const current = thread.editor.media[index] + const resolved = typeof patch === 'function' ? patch(current) : patch + const media = [...thread.editor.media] + media[index] = { ...current, ...resolved } + return { editor: { ...thread.editor, media } } + }) +} + +/** Merge a preprocessing patch into one asset (mirrors PipelineContext.savePreprocessing). */ +export function patchAssetPreprocessing( + threadId: string, + assetId: string, + patch: Partial +): Promise { + return patchAsset(threadId, assetId, (asset) => ({ + preprocessing: { ...(asset.preprocessing || {}), ...patch } + })) +} + +// ===== Scene-piece corrections (PRD §5.2 merge / split further) ===== +// Clips are main-owned; these run through patchAsset's queued mutator and +// return the updated asset so the renderer can patch locally without waiting +// for the thread-updated echo. + +const CLIP_ADJACENCY_EPSILON = 0.1 // seconds — detected scenes abut exactly +const MIN_SPLIT_PIECE_SEC = 0.2 + +function reindexClips(clips: Clip[]): Clip[] { + return clips.map((c, i) => ({ ...c, index: i + 1 })) +} + +function getUpdatedAsset(threadId: string, assetId: string): MediaAsset | null { + return threadManager.getThread(threadId)?.editor?.media.find((a) => a.id === assetId) || null +} + +/** Validate a merge selection; throws a user-facing error on rejection. */ +function validateMergeTargets(asset: MediaAsset, clipIds: string[]): Clip[] { + const targets = asset.clips + .filter((c) => clipIds.includes(c.id)) + .sort((a, b) => a.in - b.in) + if (targets.length < 2) throw new Error('Selected pieces no longer exist.') + + for (let i = 1; i < targets.length; i++) { + if (Math.abs(targets[i].in - targets[i - 1].out) > CLIP_ADJACENCY_EPSILON) { + throw new Error('Only adjacent pieces can be merged.') + } + } + return targets +} + +/** + * Merge a run of ADJACENT clips (by source time) into one piece spanning + * their union. Non-adjacent selections are rejected — merging across a hole + * would silently swallow unselected content. + * + * Validation runs BEFORE the queued mutator (updateThreadWith swallows mutator + * throws), so rejections propagate to the renderer; the mutator re-validates + * and skips silently if the asset changed mid-flight. + */ +export async function mergeClips( + threadId: string, + assetId: string, + clipIds: string[] +): Promise { + if (clipIds.length < 2) throw new Error('Select at least two pieces to merge.') + const current = getUpdatedAsset(threadId, assetId) + if (!current) throw new Error('Asset not found.') + validateMergeTargets(current, clipIds) + + await patchAsset(threadId, assetId, (asset) => { + let targets: Clip[] + try { + targets = validateMergeTargets(asset, clipIds) + } catch { + return {} // asset changed mid-flight — skip, don't corrupt + } + + const first = targets[0] + const last = targets[targets.length - 1] + const merged: Clip = { + ...first, + out: last.out, + duration: last.out - first.in, + visual: targets.map((t) => t.visual).filter(Boolean).join(' ') || undefined, + text: targets.map((t) => t.text).filter(Boolean).join(' ') || undefined, + selected: true, + masterSegmentIndex: undefined // spans multiple master scenes + } + + const remainingIds = new Set(clipIds.filter((id) => id !== first.id)) + const clips = asset.clips + .map((c) => (c.id === first.id ? merged : c)) + .filter((c) => !remainingIds.has(c.id)) + return { clips: reindexClips(clips) } + }) + + return getUpdatedAsset(threadId, assetId) +} + +/** + * Split one clip into two pieces at `atSec` (source seconds; defaults to the + * midpoint). Both halves share the source thumbnail; the left half keeps the + * original id so selection/inspector focus degrades gracefully. + */ +export async function splitClip( + threadId: string, + assetId: string, + clipId: string, + atSec?: number +): Promise { + // Pre-validate outside the queued mutator (see mergeClips). + const current = getUpdatedAsset(threadId, assetId) + const candidate = current?.clips.find((c) => c.id === clipId) + if (!candidate) throw new Error('Piece no longer exists.') + const cutAt = atSec ?? candidate.in + candidate.duration / 2 + if (cutAt - candidate.in < MIN_SPLIT_PIECE_SEC || candidate.out - cutAt < MIN_SPLIT_PIECE_SEC) { + throw new Error('Piece is too short to split.') + } + + await patchAsset(threadId, assetId, (asset) => { + const target = asset.clips.find((c) => c.id === clipId) + if (!target) return {} // asset changed mid-flight — skip + + const at = atSec ?? target.in + target.duration / 2 + if (at - target.in < MIN_SPLIT_PIECE_SEC || target.out - at < MIN_SPLIT_PIECE_SEC) { + return {} + } + + const left: Clip = { + ...target, + out: at, + duration: at - target.in, + selected: true, + masterSegmentIndex: undefined // sub-range of a master scene + } + const right: Clip = { + ...target, + id: uuidv4(), + in: at, + duration: target.out - at, + selected: true, + masterSegmentIndex: undefined + } + + const clips = asset.clips.flatMap((c) => (c.id === clipId ? [left, right] : [c])) + return { clips: reindexClips(clips) } + }) + + return getUpdatedAsset(threadId, assetId) +} + +/** + * Remove derived Gemini data (transcript or scene descriptions) from ONE + * asset: deletes the backing JSON, clears the preprocessing path fields and + * the derived clip field (text / visual), and drops the step's task so the + * Inspector offers to re-run. The pieces (segmentation) are kept. + */ +export async function clearAssetData( + threadId: string, + assetId: string, + kind: 'transcript' | 'descriptions' +): Promise { + const asset = getUpdatedAsset(threadId, assetId) + if (!asset) return null + + // Delete the backing files (best-effort). Audio is intentionally kept so a + // later re-transcribe can skip re-extraction. + const thread = threadManager.getThread(threadId)! + if (kind === 'transcript') { + // The transcripts dir holds only transcript artifacts (json + raw + // response txt) — remove it wholesale for a clean sweep. + const dir = path.join(getAssetDir(thread, assetId), THREAD_DIRS.TRANSCRIPTS) + if (fs.existsSync(dir)) { + try { fs.rmSync(dir, { recursive: true, force: true }) } catch (e) { console.error('[editor] clear transcript:', e) } + } + } else { + // The analysis dir also holds scenes.json (kept) — delete just the file. + const f = asset.preprocessing?.sceneDescriptionsPath + if (f && fs.existsSync(f)) { + try { fs.rmSync(f, { force: true }) } catch (e) { console.error('[editor] clear descriptions:', e) } + } + } + + await threadManager.updateThreadWith(threadId, (t) => { + if (!t.editor) return null + const media = t.editor.media.map((a) => { + if (a.id !== assetId) return a + const preprocessing = { ...a.preprocessing } + if (kind === 'transcript') { + delete preprocessing.transcriptPath + delete preprocessing.rawTranscriptPath + delete preprocessing.correctedTranscriptPath + } else { + delete preprocessing.sceneDescriptionsPath + } + const clips = a.clips.map((c) => + kind === 'transcript' ? { ...c, text: undefined } : { ...c, visual: undefined } + ) + return { ...a, preprocessing, clips } + }) + // Drop the step's namespaced task(s) so a re-run reads as fresh. + const backgroundTasks = { ...(t.backgroundTasks || {}) } + delete backgroundTasks[`${assetId}:${kind === 'transcript' ? 'transcript' : 'descriptions'}`] + return { editor: { ...t.editor, media }, backgroundTasks } + }) + + return getUpdatedAsset(threadId, assetId) +} + +/** + * Removes an asset: aborts any live preprocessing, deletes its artifact dir, + * and drops the asset plus its clips, timeline items, and namespaced tasks. + */ +export async function removeAsset(threadId: string, assetId: string): Promise { + abortAssetPreprocessing(threadId, assetId) + + const thread = threadManager.getThread(threadId) + if (!thread || !thread.editor) return false + + const assetDir = getAssetDir(thread, assetId) + if (fs.existsSync(assetDir)) { + try { + fs.rmSync(assetDir, { recursive: true, force: true }) + } catch (error) { + console.error(`[editor] Failed to delete asset dir ${assetDir}:`, error) + } + } + + const updated = await threadManager.updateThreadWith(threadId, (t) => { + if (!t.editor) return null + + const backgroundTasks = { ...(t.backgroundTasks || {}) } + for (const taskId of Object.keys(backgroundTasks)) { + if (taskId.startsWith(`${assetId}:`)) delete backgroundTasks[taskId] + } + + return { + editor: { + ...t.editor, + media: t.editor.media.filter((a) => a.id !== assetId), + timeline: t.editor.timeline.filter((item) => item.sourceAssetId !== assetId) + }, + backgroundTasks + } + }) + + return updated !== null +} diff --git a/src/main/editor/context.ts b/src/main/editor/context.ts new file mode 100644 index 0000000..53728af --- /dev/null +++ b/src/main/editor/context.ts @@ -0,0 +1,192 @@ +import type { EditorDocument, MediaAsset, TimelineItem } from '@shared/types' +import { computeScope, type AiScope } from '@shared/ai-scope' +import { itemDuration } from '@shared/timeline' + +/** + * Builds the prompt context text for the editor AI (PRD §5.7). + * Applies the degradation ladder so a multi-hour project always fits the + * budget — ending, if it must, in an EXPLICIT truncation flag (never silent). + */ + +export const MAX_SCENES_PER_ASSET = 60 // ladder: 60 -> 30 -> 12 +export const VISUAL_MAX_CHARS = 160 +export const CONTEXT_CHAR_BUDGET = 120_000 // ~30k tokens +export const OUTLINE_GIST_COUNT = 2 + +export interface PromptContextResult { + contextText: string + scope: AiScope + tokenEstimate: number + truncated: boolean + thinContext: boolean +} + +const fmt = (n: number) => Math.round(n * 100) / 100 + +export function buildPromptContext( + doc: EditorDocument, + prompt: string, + opts: { selectedItemIds: string[]; playheadSec: number; widen?: 'chapter' | 'full' } +): PromptContextResult { + const scope = computeScope({ + timeline: doc.timeline, + markers: doc.markers || [], + selectedItemIds: opts.selectedItemIds, + playheadSec: opts.playheadSec, + mediaIds: doc.media.map((a) => a.id), + widen: opts.widen + }) + + const inScopeIds = new Set(scope.itemIds) + const inScopeItems = doc.timeline + .filter((i) => inScopeIds.has(i.id)) + .sort((a, b) => a.timelineStart - b.timelineStart) + const scopeAssets = doc.media.filter((a) => scope.assetIds.includes(a.id)) + + const clipOf = (item: TimelineItem) => + doc.media.find((a) => a.id === item.sourceAssetId)?.clips.find((c) => c.id === item.sourceClipId) + + const thinContext = !scopeAssets.some((a) => a.clips.some((c) => !!c.visual)) + + // Degradation ladder state + let scenesCap = MAX_SCENES_PER_ASSET + let dropNeighborVisuals = false + let dropUnusedAssetScenes = false + let truncated = false + + const usedAssetIds = new Set(inScopeItems.map((i) => i.sourceAssetId)) + const selectedSet = new Set(opts.selectedItemIds) + + const build = (): string => { + const lines: string[] = [] + + // ---- PROJECT ---- + lines.push('PROJECT') + lines.push(`- Sequence: ${fmt(doc.timelineMeta.duration)}s, ${doc.timelineMeta.fps}fps, ${doc.timelineMeta.width}x${doc.timelineMeta.height}`) + for (const track of [...doc.tracks].sort((a, b) => a.order - b.order)) { + const flags = [track.locked && 'locked', track.muted && 'muted', track.hidden && 'hidden'].filter(Boolean).join(', ') + lines.push(`- Track ${track.id} "${track.name}" (${track.kind}${flags ? ', ' + flags : ''})`) + } + for (const asset of doc.media) { + const described = asset.clips.some((c) => !!c.visual) + lines.push(`- Asset ${asset.id} "${asset.name}" ${asset.kind} ${fmt(asset.metadata?.duration || 0)}s — ${asset.clips.length} scenes, descriptions: ${described ? 'yes' : 'no'}`) + } + lines.push('') + + // ---- SCOPE ---- + lines.push(`SCOPE: ${scope.label}`) + lines.push('') + + // ---- TIMELINE OUTLINE (only when scoped narrower than full) ---- + if (scope.kind !== 'full' && doc.timeline.length > 0) { + lines.push('TIMELINE OUTLINE') + const markers = [...(doc.markers || [])].sort((a, b) => a.time - b.time) + const totalEnd = doc.timelineMeta.duration || 0 + const bounds: { t0: number; t1: number; label: string }[] = [] + if (markers.length > 0) { + const cuts = [0, ...markers.map((m) => m.time), totalEnd] + for (let k = 0; k < cuts.length - 1; k++) { + if (cuts[k + 1] - cuts[k] < 0.5) continue + const label = k === 0 ? 'Intro' : markers[k - 1]?.label || `#${k}` + bounds.push({ t0: cuts[k], t1: cuts[k + 1], label }) + } + } else { + for (let t = 0; t < totalEnd; t += 600) { + bounds.push({ t0: t, t1: Math.min(t + 600, totalEnd), label: `#${bounds.length + 1}` }) + } + } + for (const b of bounds) { + const items = doc.timeline.filter((i) => i.timelineStart >= b.t0 && i.timelineStart < b.t1) + if (!items.length) continue + const dur = items.reduce((sum, i) => sum + itemDuration(i), 0) + const gists = items + .map((i) => clipOf(i)?.visual) + .filter(Boolean) + .slice(0, OUTLINE_GIST_COUNT) + .map((v) => `"${v!.slice(0, 120)}"`) + .join(' / ') + lines.push(`- Chapter "${b.label}" [${fmt(b.t0)}–${fmt(b.t1)}s]: ${items.length} items, ${fmt(dur)}s${gists ? `; gist: ${gists}` : ''}`) + } + lines.push('') + } + + // ---- ITEMS IN SCOPE ---- + const header = scope.kind === 'full' ? 'CURRENT TIMELINE' : `ITEMS IN SCOPE (${inScopeItems.length})` + lines.push(header) + if (inScopeItems.length === 0) { + lines.push('(empty — build a new cut from the available scenes)') + } + for (const item of inScopeItems) { + const track = doc.tracks.find((t) => t.id === item.trackId) + const clip = clipOf(item) + const showVisual = clip?.visual && (!dropNeighborVisuals || selectedSet.has(item.id) || scope.kind !== 'selection') + const visual = showVisual ? ` | ${clip!.visual!.slice(0, VISUAL_MAX_CHARS)}` : '' + lines.push( + `- ${item.id} | ${track?.name || '?'} | at ${fmt(item.timelineStart)}s | src ${fmt(item.in)}–${fmt(item.out)}s | ${item.speed}x | "${item.label || ''}"${visual}` + ) + } + lines.push('') + + // ---- AVAILABLE SCENES ---- + lines.push('AVAILABLE SCENES (add material from here via addClips)') + for (const asset of scopeAssets) { + if (dropUnusedAssetScenes && !usedAssetIds.has(asset.id)) { + lines.push(`Asset ${asset.id} "${asset.name}": ${asset.clips.length} scenes omitted for brevity`) + continue + } + lines.push(`Asset ${asset.id} "${asset.name}" (${fmt(asset.metadata?.duration || 0)}s):`) + const clips = asset.clips + if (clips.length <= scenesCap) { + for (const clip of clips) { + const visual = clip.visual ? ` "${clip.visual.slice(0, VISUAL_MAX_CHARS)}"` : '' + lines.push(`- #${clip.index} [${fmt(clip.in)}–${fmt(clip.out)}s, ${fmt(clip.duration)}s]${visual}`) + } + } else { + // Always include scenes already used by in-scope items, then sample evenly + const usedSceneIds = new Set(inScopeItems.filter((i) => i.sourceAssetId === asset.id).map((i) => i.sourceClipId)) + const used = clips.filter((c) => usedSceneIds.has(c.id)) + const rest = clips.filter((c) => !usedSceneIds.has(c.id)) + const budget = Math.max(scenesCap - used.length, 4) + const step = Math.max(1, Math.floor(rest.length / budget)) + const sampled = rest.filter((_, i) => i % step === 0).slice(0, budget) + const shown = [...used, ...sampled].sort((a, b) => a.index - b.index) + for (const clip of shown) { + const visual = clip.visual ? ` "${clip.visual.slice(0, VISUAL_MAX_CHARS)}"` : '' + lines.push(`- #${clip.index} [${fmt(clip.in)}–${fmt(clip.out)}s, ${fmt(clip.duration)}s]${visual}`) + } + lines.push(`- … +${clips.length - shown.length} more scenes between ${fmt(clips[0].in)}s and ${fmt(clips[clips.length - 1].out)}s (evenly sampled above)`) + } + } + lines.push('') + + // ---- USER REQUEST ---- + lines.push('USER REQUEST') + lines.push(prompt) + + return lines.join('\n') + } + + // Degradation ladder until we fit the budget + let text = build() + if (text.length > CONTEXT_CHAR_BUDGET) { scenesCap = 30; text = build() } + if (text.length > CONTEXT_CHAR_BUDGET) { scenesCap = 12; text = build() } + if (text.length > CONTEXT_CHAR_BUDGET) { dropNeighborVisuals = true; text = build() } + if (text.length > CONTEXT_CHAR_BUDGET) { dropUnusedAssetScenes = true; text = build() } + if (text.length > CONTEXT_CHAR_BUDGET) { + // Final resort: hard-truncate — but never the ITEMS IN SCOPE section. + // AVAILABLE SCENES comes after ITEMS, so cutting the tail (before USER + // REQUEST) only loses scene listings. + const requestBlock = `\nUSER REQUEST\n${prompt}` + text = text.slice(0, CONTEXT_CHAR_BUDGET - requestBlock.length - 64) + + '\n[... scene listings truncated — long project ...]\n' + requestBlock + truncated = true + } + + return { + contextText: text, + scope, + tokenEstimate: Math.round(text.length / 4), + truncated, + thinContext + } +} diff --git a/src/main/editor/history.ts b/src/main/editor/history.ts new file mode 100644 index 0000000..b1a26bd --- /dev/null +++ b/src/main/editor/history.ts @@ -0,0 +1,129 @@ +import { app } from 'electron' +import fs from 'fs' +import path from 'path' +import type { EditorHistoryFile, EditorHistoryStep, TimelineSnapshot } from '@shared/types' +import { threadManager } from '../threads' + +/** + * Undo/redo sidecar persistence for the timeline editor (PRD §6). + * History lives OUTSIDE threads/{id}.json so the debounced doc autosave is + * decoupled from history growth. The renderer is authoritative for applying + * diffs; this module only loads/appends/prunes and persists. + * + * The pointer of record is doc.historyRef.currentStepId (persisted with the + * doc in one atomic write); the sidecar's currentStepId is informational. + */ + +const MAX_STEPS = 50 + +function historyDir(): string { + return path.join(app.getPath('userData'), 'editor-history') +} + +function historyPath(threadId: string): string { + return path.join(historyDir(), `${threadId}.json`) +} + +function emptyHistory(threadId: string): EditorHistoryFile { + return { threadId, steps: [], keyframes: [], currentStepId: '' } +} + +function writeHistory(file: EditorHistoryFile) { + const dir = historyDir() + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) + // Write-temp-then-rename so a crash mid-write can't corrupt the file. + const target = historyPath(file.threadId) + const temp = `${target}.tmp` + fs.writeFileSync(temp, JSON.stringify(file)) + fs.renameSync(temp, target) +} + +export function loadHistory(threadId: string): EditorHistoryFile { + const filePath = historyPath(threadId) + if (!fs.existsSync(filePath)) return emptyHistory(threadId) + try { + const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as EditorHistoryFile + if (!Array.isArray(parsed.steps)) return emptyHistory(threadId) + return { ...emptyHistory(threadId), ...parsed, threadId } + } catch (error) { + console.error(`[editor-history] Failed to read history for ${threadId}:`, error) + return emptyHistory(threadId) + } +} + +/** + * Appends a step: truncates any redo branch after the current pointer, + * assigns a monotonic seq, enforces the ring cap, stores an optional + * keyframe, and moves the pointer to the new step. + * Returns the assigned seq + resulting step count (renderer mirrors the ring). + */ +export async function pushStep( + threadId: string, + step: EditorHistoryStep, + keyframe?: TimelineSnapshot +): Promise<{ seq: number; stepCount: number }> { + const file = loadHistory(threadId) + + // Kill the redo branch beyond the current pointer + const pointerIndex = file.steps.findIndex((s) => s.id === file.currentStepId) + if (pointerIndex !== -1 && pointerIndex < file.steps.length - 1) { + const removed = new Set(file.steps.slice(pointerIndex + 1).map((s) => s.id)) + file.steps = file.steps.slice(0, pointerIndex + 1) + file.keyframes = file.keyframes.filter((k) => !removed.has(k.stepId)) + } else if (pointerIndex === -1 && file.currentStepId === '' && file.steps.length > 0) { + // Pointer at pre-history root: everything is a redo branch — drop it + file.steps = [] + file.keyframes = [] + } + + const seq = await threadManager.getNextVersion(threadId) + const stored: EditorHistoryStep = { ...step, seq } + file.steps.push(stored) + + if (keyframe) { + file.keyframes.push({ ...keyframe, stepId: stored.id }) + } + + // Ring cap: evict oldest steps; keep the newest keyframe at-or-before the + // new oldest step as the replay baseline, drop older ones. + if (file.steps.length > MAX_STEPS) { + file.steps = file.steps.slice(file.steps.length - MAX_STEPS) + const oldestSeq = file.steps[0].seq + const validIds = new Set(file.steps.map((s) => s.id)) + const baseline = file.keyframes + .filter((k) => !validIds.has(k.stepId)) + .pop() // newest evicted keyframe becomes the baseline + file.keyframes = file.keyframes.filter((k) => validIds.has(k.stepId)) + if (baseline) file.keyframes.unshift(baseline) + void oldestSeq + } + + file.currentStepId = stored.id + writeHistory(file) + return { seq, stepCount: file.steps.length } +} + +export function setPointer(threadId: string, currentStepId: string): boolean { + const file = loadHistory(threadId) + file.currentStepId = currentStepId + writeHistory(file) + return true +} + +/** + * Wipes the fine-grained ring (used on revision switches: old steps' diffs + * were computed against pre-switch states — redoing them onto a switched + * snapshot would corrupt it, and a pointer-only reset would rehydrate them). + */ +export function clearHistory(threadId: string): void { + writeHistory(emptyHistory(threadId)) +} + +export function deleteHistory(threadId: string): void { + const filePath = historyPath(threadId) + try { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath) + } catch (error) { + console.error(`[editor-history] Failed to delete history for ${threadId}:`, error) + } +} diff --git a/src/main/editor/preprocess.ts b/src/main/editor/preprocess.ts new file mode 100644 index 0000000..e1f486f --- /dev/null +++ b/src/main/editor/preprocess.ts @@ -0,0 +1,830 @@ +import fs from 'fs' +import path from 'path' +import { setMaxListeners } from 'events' +import { v4 as uuidv4 } from 'uuid' +import type { Clip, MediaAsset, SilenceRegion } from '@shared/types' +import type { PipelineContext } from '../pipeline' +import { threadManager } from '../threads' +import { backgroundTaskManager } from '../tasks' +import * as ffmpegAdapter from '../ffmpeg' +import { SceneDetector, checkScenedetectAvailability, type Scene } from '../scenedetect' +import * as extraction from '../pipeline/phases/extraction' +import { ASSET_DIRS } from '../constants/paths' +import { getAssetDir, patchAsset, patchAssetPreprocessing } from './assets' + +/** + * Per-asset preprocessing orchestrator for the timeline editor. + * + * Design (video-editor-prd.md §5.2/§7): a lean, editor-shaped step chain + * (proxy -> scenes -> thumbnails -> clips) calling the ffmpeg/scenedetect + * helpers directly, plus an OPT-IN Gemini descriptions step that reuses + * extraction.generateSceneDescription VERBATIM through an asset-scoped + * PipelineContext (tempDir = tempDir/media/), leaving the chat + * preprocessing chains untouched. + * + * Task ids are namespaced `${assetId}:` inside the existing + * thread.backgroundTasks + background-task-update broadcast. + */ + +export type PreprocessStep = + 'proxy' | 'scenes' | 'thumbnails' | 'descriptions' | 'audio' | 'transcript' | 'filmstrip' | 'segment' + +// Transcript runs UP FRONT (before scenes) so pieces derive from real speech +// segments; scene detection stays as the fallback piece source for videos +// with no usable transcript and for explicit sensitivity re-runs. +const DEFAULT_STEPS: PreprocessStep[] = ['proxy', 'audio', 'transcript', 'scenes', 'thumbnails'] + +// Audio-only assets have no picture: skip proxy/scenes/thumbnails/filmstrip. +// Pieces come from a LOCAL, energy-based segmentation (silence split + interval +// fallback for music) — no Gemini needed. Transcription stays opt-in (the +// Transcribe button runs 'audio'+'transcript' on demand). +const AUDIO_STEPS: PreprocessStep[] = ['segment'] + +// Audio/transcript failures (missing Gemini key, quota) must not brick the +// import — the chain continues and pieces fall back to scene detection. +const SOFT_FAIL_STEPS: PreprocessStep[] = ['audio', 'transcript'] + +export const SCENEDETECT_MISSING = 'scenedetect-missing' + +// ===== Concurrency cap (K=3) for ffmpeg/scenedetect-heavy work ===== +const MAX_CONCURRENT_HEAVY = 3 +let heavyRunning = 0 +const heavyQueue: Array<() => void> = [] + +async function withHeavySlot(fn: () => Promise): Promise { + if (heavyRunning >= MAX_CONCURRENT_HEAVY) { + await new Promise((resolve) => heavyQueue.push(resolve)) + } + heavyRunning++ + try { + return await fn() + } finally { + heavyRunning-- + const next = heavyQueue.shift() + if (next) next() + } +} + +// ===== Abort registry ===== +const abortControllers = new Map() + +const abortKey = (threadId: string, assetId: string) => `${threadId}:${assetId}` + +export function abortAssetPreprocessing(threadId: string, assetId: string): void { + const key = abortKey(threadId, assetId) + const controller = abortControllers.get(key) + if (controller) { + controller.abort() + abortControllers.delete(key) + } +} + +export function isAssetPreprocessing(threadId: string, assetId: string): boolean { + return abortControllers.has(abortKey(threadId, assetId)) +} + +// ===== Helpers ===== + +function getAsset(threadId: string, assetId: string): MediaAsset | null { + const thread = threadManager.getThread(threadId) + return thread?.editor?.media.find((a) => a.id === assetId) || null +} + +function exists(p?: string): boolean { + return !!p && fs.existsSync(p) +} + +const taskId = (assetId: string, step: PreprocessStep) => `${assetId}:${step}` + +/** + * Throttled per-step progress reporter: every setTask persists the thread + * JSON and broadcasts, so noisy sources (ffmpeg/scenedetect ticks) are + * quantized to ≥5% jumps. + */ +function makeProgressReporter(threadId: string, assetId: string, step: PreprocessStep) { + let last = -1 + return (percent: number, status?: string) => { + const p = Math.min(100, Math.max(0, Math.round(percent))) + if (p - last < 5 && p !== 100) return + last = p + void setTask(threadId, assetId, step, { state: 'running', progress: p, ...(status ? { status } : {}) }) + } +} + +/** + * Best-effort numeric progress from a phase's human status text — reused + * phases only report strings (e.g. "Converting to audio... 45%", + * "Analyzing scenes 51 to 100 / 418..."). Returns null when no number. + */ +function progressFromStatus(status: string): number | null { + const pct = status.match(/(\d{1,3})\s*%/) + if (pct) return Math.min(100, Number(pct[1])) + const count = status.match(/(\d+)\s*(?:to\s+(\d+))?\s*\/\s*(\d+)/) + if (count) { + const done = Number(count[2] ?? count[1]) + const total = Number(count[3]) + if (total > 0 && done <= total) return Math.round((done / total) * 100) + } + return null +} + +async function setTask( + threadId: string, + assetId: string, + step: PreprocessStep, + updates: Parameters[2] +) { + await backgroundTaskManager.updateTask(threadId, taskId(assetId, step), { + name: step, + ...updates + }) +} + +/** + * Asset-scoped PipelineContext bridge so existing pipeline phases + * (generateSceneDescription today; transcript phases later) run per-asset. + * Modeled on backgroundTaskManager.createMockContext, but all reads/writes + * target the MediaAsset record and the asset's artifact dir. + */ +function createAssetContext( + threadId: string, + assetId: string, + step: PreprocessStep, + signal: AbortSignal +): PipelineContext { + const thread = threadManager.getThread(threadId)! + const asset = getAsset(threadId, assetId)! + const assetDir = getAssetDir(thread, assetId) + + return { + threadId, + videoPath: asset.proxyPath || asset.originalPath, + tempDir: assetDir, // THREAD_DIRS.FRAMES/ANALYSIS joins inside phases land under the asset dir + get preprocessing() { + return getAsset(threadId, assetId)?.preprocessing || {} + }, + messageId: `editor-${assetId}`, + context: '', + baseTimeline: undefined, + intentResult: undefined, + updateStatus: async (status: string) => { + const progress = progressFromStatus(status) + await setTask(threadId, assetId, step, { + state: 'running', + status, + ...(progress !== null ? { progress } : {}) + }) + }, + recordUsage: async (record) => { + await threadManager.updateThreadWith(threadId, (t) => ({ + usageHistory: [...(t.usageHistory || []), { ...record, timestamp: Date.now() }] + })) + }, + savePreprocessing: async (updates) => { + await patchAssetPreprocessing(threadId, assetId, updates) + }, + waitForTask: async () => { }, + next: () => { }, + finish: async () => { }, + fail: async (error: string) => { + await setTask(threadId, assetId, step, { state: 'error', error }) + }, + signal + } +} + +// ===== Steps ===== + +async function runProxyStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (exists(asset.proxyPath)) { + await setTask(threadId, assetId, 'proxy', { state: 'completed', progress: 100 }) + return + } + + await setTask(threadId, assetId, 'proxy', { state: 'running', status: 'Creating 480p proxy…', progress: 0 }) + + const thread = threadManager.getThread(threadId)! + const proxyDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.PROXY) + if (!fs.existsSync(proxyDir)) fs.mkdirSync(proxyDir, { recursive: true }) + + let proxyPath: string + if (await ffmpegAdapter.isVideoLowResolution(asset.originalPath)) { + // Already <=480p — reuse the original as the proxy + proxyPath = asset.originalPath + } else { + proxyPath = await withHeavySlot(() => + ffmpegAdapter.toLowResolution( + asset.originalPath, + proxyDir, + (percent) => setTask(threadId, assetId, 'proxy', { state: 'running', progress: percent }), + signal, + { sourceFps: asset.metadata?.fps } + ) + ) + } + + await patchAsset(threadId, assetId, { proxyPath }) + await patchAssetPreprocessing(threadId, assetId, { lowResVideoPath: proxyPath }) + await setTask(threadId, assetId, 'proxy', { state: 'completed', progress: 100 }) +} + +async function runScenesStep( + threadId: string, + assetId: string, + signal: AbortSignal, + threshold?: number +) { + const asset = getAsset(threadId, assetId)! + const forceRerun = threshold !== undefined + + if (!forceRerun && exists(asset.preprocessing.sceneTimesPath)) { + await setTask(threadId, assetId, 'scenes', { state: 'completed', progress: 100 }) + return + } + + await setTask(threadId, assetId, 'scenes', { state: 'running', status: 'Detecting scenes…', progress: 0 }) + const reportScenes = makeProgressReporter(threadId, assetId, 'scenes') + + const available = await checkScenedetectAvailability() + if (!available) { + throw new Error(SCENEDETECT_MISSING) + } + + const thread = threadManager.getThread(threadId)! + const analysisDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.ANALYSIS) + if (!fs.existsSync(analysisDir)) fs.mkdirSync(analysisDir, { recursive: true }) + const sceneTimesPath = path.join(analysisDir, 'scenes.json') + + const detector = new SceneDetector() + const videoPath = asset.proxyPath || asset.originalPath + let scenes = await withHeavySlot(() => + detector.detectScenes(videoPath, signal, threshold, (p) => reportScenes(p)) + ) + + // A cut-less video (talking head, screen recording) can yield zero rows — + // fall back to a single whole-video scene so there is always one piece. + if (scenes.length === 0 && asset.metadata?.duration) { + scenes = [{ startTime: 0, endTime: asset.metadata.duration, duration: asset.metadata.duration }] + } + + fs.writeFileSync(sceneTimesPath, JSON.stringify(scenes, null, 2)) + await patchAssetPreprocessing(threadId, assetId, { sceneTimesPath }) + + // Pieces come from the TRANSCRIPT when one was derived earlier in the + // chain; scenes only produce pieces as the fallback (no transcript) or on + // an explicit sensitivity re-run (the user asked for scene pieces). + const hasClips = (getAsset(threadId, assetId)?.clips || []).length > 0 + if (forceRerun || !hasClips) { + await deriveClips(threadId, assetId, scenes) + } + + await setTask(threadId, assetId, 'scenes', { + state: 'completed', + progress: 100, + status: `${scenes.length} scenes detected` + }) +} + +/** + * Map Scene[] -> Clip[] on the asset, preserving renderer-owned `selected` + * and prior `visual`/`thumbnailPath` by (in,out) epsilon-match on re-runs. + */ +async function deriveClips(threadId: string, assetId: string, scenes: Scene[]) { + const EPSILON = 0.05 + await patchAsset(threadId, assetId, (current) => { + const previous = current.clips || [] + const clips: Clip[] = scenes.map((scene, i) => { + const match = previous.find( + (c) => Math.abs(c.in - scene.startTime) <= EPSILON && Math.abs(c.out - scene.endTime) <= EPSILON + ) + return { + id: match?.id || uuidv4(), + sourceAssetId: assetId, + index: i + 1, + in: scene.startTime, + out: scene.endTime, + duration: scene.duration, + thumbnailPath: match?.thumbnailPath, + visual: match?.visual, + text: match?.text, + selected: match?.selected ?? false, + masterSegmentIndex: i + 1 + } + }) + return { clips } + }) +} + +// Audio segmentation tuning (§5.2 for audio). +const AUDIO_SILENCE_DB = -30 // below this is "silence" for splitting +const AUDIO_SILENCE_MIN = 0.6 // min silence gap (s) that forces a cut +const AUDIO_MIN_PIECE = 1.0 // drop/merge pieces shorter than this +const AUDIO_TARGET_PIECE = 20 // interval-fallback / max piece length (s) + +/** + * Divides an audio duration into CONTIGUOUS pieces (full coverage — silence is + * split, not removed; the Silence Finder handles removal). Cut points are the + * midpoints of silence gaps; long runs (music, long monologue) are subdivided + * into ~AUDIO_TARGET_PIECE chunks and slivers merge into their predecessor. + */ +export function segmentAudio(duration: number, silence: SilenceRegion[]): { in: number; out: number }[] { + const cuts = new Set([0, duration]) + for (const s of silence) { + const a = Math.max(0, Math.min(s.start, duration)) + const b = Math.max(0, Math.min(s.end, duration)) + const mid = (a + b) / 2 + if (mid > AUDIO_MIN_PIECE && mid < duration - AUDIO_MIN_PIECE) cuts.add(mid) + } + const points = [...cuts].sort((x, y) => x - y) + + // Contiguous pieces, then subdivide long ones (music → even chunks). + const pieces: { in: number; out: number }[] = [] + for (let i = 0; i < points.length - 1; i++) { + const seg = { in: points[i], out: points[i + 1] } + const len = seg.out - seg.in + if (len <= AUDIO_TARGET_PIECE * 1.5) { + pieces.push(seg) + } else { + const n = Math.ceil(len / AUDIO_TARGET_PIECE) + const step = len / n + for (let k = 0; k < n; k++) { + pieces.push({ in: seg.in + k * step, out: k === n - 1 ? seg.out : seg.in + (k + 1) * step }) + } + } + } + + // Merge slivers into the previous piece so nothing is unusably short. + const merged: { in: number; out: number }[] = [] + for (const p of pieces) { + if (merged.length && p.out - p.in < AUDIO_MIN_PIECE) { + merged[merged.length - 1].out = p.out + } else { + merged.push({ ...p }) + } + } + return merged.length ? merged : [{ in: 0, out: duration }] +} + +/** + * Local, energy-based audio segmentation (no Gemini): silence split + interval + * fallback, so speech splits at pauses and music splits into even chunks. Runs + * silencedetect on the source directly (audio-kind assets have no proxy). + */ +async function runSegmentStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + const duration = asset.metadata?.duration || 0 + if (duration <= 0) { + await setTask(threadId, assetId, 'segment', { state: 'completed', progress: 100, status: 'No audio duration' }) + return + } + + await setTask(threadId, assetId, 'segment', { state: 'running', status: 'Segmenting audio…', progress: 0 }) + const reportSeg = makeProgressReporter(threadId, assetId, 'segment') + + const src = exists(asset.preprocessing.audioPath) ? asset.preprocessing.audioPath! : asset.originalPath + let silence: SilenceRegion[] = [] + try { + silence = await withHeavySlot(() => + ffmpegAdapter.detectSilence(src, { noiseDb: AUDIO_SILENCE_DB, minDurationSec: AUDIO_SILENCE_MIN }, signal, (p) => reportSeg(p)) + ) + } catch (error) { + if (signal.aborted) throw new Error('Aborted') + console.warn(`[editor] silencedetect failed for audio ${assetId} — using interval chunks:`, error) + } + + if (signal.aborted) throw new Error('Aborted') + const ranges = segmentAudio(duration, silence) + await patchAsset(threadId, assetId, { + clips: ranges.map((r, i) => ({ + id: uuidv4(), + sourceAssetId: assetId, + index: i + 1, + in: r.in, + out: r.out, + duration: r.out - r.in, + selected: false + })) + }) + + await setTask(threadId, assetId, 'segment', { state: 'completed', progress: 100, status: `${ranges.length} pieces` }) +} + +/** + * Guarantees an audio asset has at least one placeable piece. When segmentation + * produced pieces this is a no-op; otherwise it creates one clip spanning the + * whole file (labelled by name) so the asset can always be dropped on the audio + * track. Non-audio assets are left to scene detection. + */ +async function ensureAudioHasClip(threadId: string, assetId: string): Promise { + const asset = getAsset(threadId, assetId) + if (!asset || asset.kind !== 'audio') return + if ((asset.clips || []).length > 0) return + const duration = asset.metadata?.duration || 0 + if (duration <= 0) return + await patchAsset(threadId, assetId, { + clips: [{ + id: uuidv4(), + sourceAssetId: assetId, + index: 1, + in: 0, + out: duration, + duration, + selected: false + }] + }) +} + +// Gap handling for transcript-derived pieces — mirrors timeline/enrichment.ts +// so silence between statements stays visible and selectable. +const MIN_GAP_SEC = 0.5 +const MAX_GAP_CHUNK_SEC = 15.0 + +/** + * Map the transcript -> Clip[]: one piece per spoken statement, plus + * "[Silence]" pieces over gaps, so the tray shows REAL editorial segments + * (what was said) instead of visual scene cuts. Prior clip props survive by + * (in,out) epsilon-match. + */ +async function deriveClipsFromTranscript(threadId: string, assetId: string): Promise { + const asset = getAsset(threadId, assetId) + const transcriptPath = asset?.preprocessing.transcriptPath + if (!asset || !exists(transcriptPath)) return 0 + + const items: { start: string; end: string; text: string }[] = + JSON.parse(fs.readFileSync(transcriptPath!, 'utf-8')) + const duration = asset.metadata?.duration || 0 + + type Seg = { in: number; out: number; text?: string } + const segments: Seg[] = [] + const pushGap = (from: number, to: number) => { + let cursor = from + while (to - cursor > MIN_GAP_SEC) { + const end = Math.min(cursor + MAX_GAP_CHUNK_SEC, to) + segments.push({ in: cursor, out: end, text: '[Silence]' }) + cursor = end + } + } + + let previousEnd = 0 + for (const item of [...items].sort((a, b) => srtToSeconds(a.start) - srtToSeconds(b.start))) { + const start = srtToSeconds(item.start) + const end = srtToSeconds(item.end) + if (!(end > start)) continue + if (start - previousEnd > MIN_GAP_SEC) pushGap(previousEnd, start) + segments.push({ in: start, out: end, text: item.text }) + previousEnd = Math.max(previousEnd, end) + } + if (duration - previousEnd > MIN_GAP_SEC) pushGap(previousEnd, duration) + + if (segments.length === 0) return 0 + + const EPSILON = 0.05 + await patchAsset(threadId, assetId, (current) => { + const previous = current.clips || [] + const clips: Clip[] = segments.map((seg, i) => { + const match = previous.find( + (c) => Math.abs(c.in - seg.in) <= EPSILON && Math.abs(c.out - seg.out) <= EPSILON + ) + return { + id: match?.id || uuidv4(), + sourceAssetId: assetId, + index: i + 1, + in: seg.in, + out: seg.out, + duration: seg.out - seg.in, + thumbnailPath: match?.thumbnailPath, + visual: match?.visual, + text: seg.text, + selected: match?.selected ?? false, + masterSegmentIndex: undefined // speech segments don't map to master scenes + } + }) + return { clips } + }) + return segments.length +} + +async function runThumbnailsStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + // Thumbnails follow the CLIPS (whatever derived them — transcript segments + // or scene detection), one midpoint frame per piece. + const targets = (asset.clips || []).map((c) => ({ id: c.id, midpoint: c.in + c.duration / 2 })) + if (targets.length === 0) { + await setTask(threadId, assetId, 'thumbnails', { state: 'completed', status: 'No pieces to thumbnail' }) + return + } + + const thread = threadManager.getThread(threadId)! + const framesDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.FRAMES) + if (!fs.existsSync(framesDir)) fs.mkdirSync(framesDir, { recursive: true }) + + await setTask(threadId, assetId, 'thumbnails', { state: 'running', status: 'Extracting thumbnails…', progress: 0 }) + + const videoPath = asset.proxyPath || asset.originalPath + const thumbnails = new Map() // clipId -> framePath + + // Patch finished thumbnails into the (already-derived) clips so tiles fill + // in progressively. Every updateTask/patchAsset persists the thread JSON, + // so both are THROTTLED to a batch cadence rather than per-frame. + const flushThumbnails = async () => { + await patchAsset(threadId, assetId, (current) => ({ + clips: (current.clips || []).map((clip) => + thumbnails.has(clip.id) ? { ...clip, thumbnailPath: thumbnails.get(clip.id) } : clip + ) + })) + } + + const BATCH = Math.max(1, Math.min(8, Math.floor(targets.length / 25) || 1)) + + await withHeavySlot(async () => { + for (let i = 0; i < targets.length; i++) { + if (signal.aborted) throw new Error('Aborted') + try { + // extractFrame filenames are deterministic (…_frame_.jpg) — re-runs reuse them + const framePath = await ffmpegAdapter.extractFrame(videoPath, targets[i].midpoint, framesDir, signal) + thumbnails.set(targets[i].id, framePath) + } catch (error) { + if (signal.aborted) throw error + console.error(`[editor] Thumbnail failed for piece ${i} of asset ${assetId}:`, error) + // tolerate individual frame failures + } + + const done = i + 1 + if (done % BATCH === 0 || done === targets.length) { + await flushThumbnails() + await setTask(threadId, assetId, 'thumbnails', { + state: 'running', + status: `${done}/${targets.length} thumbnails`, + progress: Math.round((done / targets.length) * 100) + }) + } + } + }) + + await setTask(threadId, assetId, 'thumbnails', { state: 'completed', progress: 100 }) +} + +async function runDescriptionsStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (!exists(asset.preprocessing.sceneTimesPath)) { + throw new Error('Scenes must be detected before describing them.') + } + + await setTask(threadId, assetId, 'descriptions', { state: 'running', status: 'Describing scenes…' }) + + // Reuse the existing pipeline phase VERBATIM through an asset-scoped context. + const context = createAssetContext(threadId, assetId, 'descriptions', signal) + await extraction.generateSceneDescription({}, context) + + if (signal.aborted) throw new Error('Aborted') + + // Merge generated descriptions back into the asset's clips (by scene index). + const descriptionsPath = getAsset(threadId, assetId)?.preprocessing.sceneDescriptionsPath + if (exists(descriptionsPath)) { + const descriptions: { index: number; description: string; framePath: string }[] = + JSON.parse(fs.readFileSync(descriptionsPath!, 'utf-8')) + const byIndex = new Map(descriptions.map((d) => [d.index, d])) + + await patchAsset(threadId, assetId, (current) => ({ + clips: (current.clips || []).map((clip) => { + const desc = byIndex.get(clip.index - 1) // descriptions are 0-based scene indices + return desc + ? { ...clip, visual: desc.description, thumbnailPath: clip.thumbnailPath || desc.framePath } + : clip + }) + })) + } + + await setTask(threadId, assetId, 'descriptions', { state: 'completed', progress: 100 }) +} + +async function runAudioStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (exists(asset.preprocessing.audioPath)) { + await setTask(threadId, assetId, 'audio', { state: 'completed', progress: 100 }) + return + } + + await setTask(threadId, assetId, 'audio', { state: 'running', status: 'Extracting audio…' }) + + // Reuse the existing extraction phase VERBATIM through an asset-scoped context. + const context = createAssetContext(threadId, assetId, 'audio', signal) + await withHeavySlot(async () => { await extraction.convertToAudio({}, context) }) + + if (signal.aborted) throw new Error('Aborted') + await setTask(threadId, assetId, 'audio', { state: 'completed', progress: 100 }) +} + +async function runTranscriptStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (!exists(asset.preprocessing.audioPath)) { + throw new Error('Audio must be extracted before transcription.') + } + + // A single Gemini call has no incremental signal — report honest stage + // milestones so the bar still moves: upload/transcribe → merge → done. + await setTask(threadId, assetId, 'transcript', { + state: 'running', status: 'Uploading audio & transcribing…', progress: 10 + }) + + // Reuse the raw-transcript phase VERBATIM (single Gemini pass — the editor + // keeps transcription lean; the chat flow's corrected pass is not run here). + const context = createAssetContext(threadId, assetId, 'transcript', signal) + await extraction.extractRawTranscript({}, context) + + if (signal.aborted) throw new Error('Aborted') + + await setTask(threadId, assetId, 'transcript', { + state: 'running', status: 'Deriving pieces from speech…', progress: 85 + }) + + // Fresh import (no pieces yet): pieces ARE the transcript segments. + // Existing scene pieces (sensitivity re-run, later re-transcribe): keep + // them and just fill their text by overlap. + const hasClips = (getAsset(threadId, assetId)?.clips || []).length > 0 + if (hasClips) { + await mergeTranscriptIntoClips(threadId, assetId) + } else { + const derived = await deriveClipsFromTranscript(threadId, assetId) + if (derived === 0) { + // Empty/unusable transcript — scenes step will derive fallback pieces + await setTask(threadId, assetId, 'transcript', { + state: 'completed', progress: 100, status: 'No speech found — using scene pieces' + }) + return + } + await setTask(threadId, assetId, 'transcript', { + state: 'completed', progress: 100, status: `${derived} speech segments` + }) + return + } + + await setTask(threadId, assetId, 'transcript', { state: 'completed', progress: 100 }) +} + +// Cap total strip frames per asset so a multi-hour source stays one bounded +// pass (PRD §5.5/§8): interval = max(1s, duration/300). +const FILMSTRIP_MAX_FRAMES = 300 + +async function runFilmstripStep(threadId: string, assetId: string, signal: AbortSignal) { + const asset = getAsset(threadId, assetId)! + if (asset.filmstrip?.length) { + await setTask(threadId, assetId, 'filmstrip', { state: 'completed', progress: 100 }) + return + } + + await setTask(threadId, assetId, 'filmstrip', { state: 'running', status: 'Generating filmstrip…', progress: 0 }) + const reportFilmstrip = makeProgressReporter(threadId, assetId, 'filmstrip') + + const thread = threadManager.getThread(threadId)! + const stripDir = path.join(getAssetDir(thread, assetId), ASSET_DIRS.FRAMES, 'strip') + if (!fs.existsSync(stripDir)) fs.mkdirSync(stripDir, { recursive: true }) + + const duration = asset.metadata?.duration || 0 + const intervalSec = Math.max(1, duration / FILMSTRIP_MAX_FRAMES) + const videoPath = asset.proxyPath || asset.originalPath + + const filmstrip = await withHeavySlot(() => + ffmpegAdapter.generateFilmstrip(videoPath, stripDir, intervalSec, signal, (p) => reportFilmstrip(p)) + ) + + await patchAsset(threadId, assetId, { filmstrip }) + await setTask(threadId, assetId, 'filmstrip', { + state: 'completed', + progress: 100, + status: `${filmstrip.length} frames` + }) +} + +/** `HH:MM:SS,mmm` (or `MM:SS,mmm`) → seconds. Mirrors enrichment.ts timeToSeconds. */ +function srtToSeconds(t: string): number { + const clean = t.trim().replace(',', '.') + const [timePart, milliPart = '0'] = clean.split('.') + const parts = timePart.split(':').map(Number) + let seconds = 0 + if (parts.length === 3) seconds = parts[0] * 3600 + parts[1] * 60 + parts[2] + else if (parts.length === 2) seconds = parts[0] * 60 + parts[1] + else seconds = parts[0] || 0 + return seconds + parseFloat(`0.${milliPart}`) +} + +/** + * Populate each Clip.text with the transcript excerpt overlapping [in, out]. + * Mirrors the clip.visual merge in runDescriptionsStep, but keyed by time + * overlap rather than scene index (transcript segments don't align to scenes). + */ +async function mergeTranscriptIntoClips(threadId: string, assetId: string) { + const transcriptPath = getAsset(threadId, assetId)?.preprocessing.transcriptPath + if (!exists(transcriptPath)) return + + const items: { start: string; end: string; text: string }[] = + JSON.parse(fs.readFileSync(transcriptPath!, 'utf-8')) + const ranges = items.map((it) => ({ + start: srtToSeconds(it.start), + end: srtToSeconds(it.end), + text: (it.text || '').trim() + })) + + await patchAsset(threadId, assetId, (current) => ({ + clips: (current.clips || []).map((clip) => { + // Overlap test: item[s,e] intersects clip[in,out] when s < out && e > in. + const text = ranges + .filter((r) => r.start < clip.out && r.end > clip.in && r.text && r.text !== '[Silence]') + .map((r) => r.text) + .join(' ') + .trim() + return text ? { ...clip, text } : clip + }) + })) +} + +// ===== Orchestrator ===== + +export async function preprocessMediaAsset( + threadId: string, + assetId: string, + options?: { steps?: PreprocessStep[]; threshold?: number } +): Promise { + const asset = getAsset(threadId, assetId) + if (!asset) return + if (isAssetPreprocessing(threadId, assetId)) return // already running + + const steps = options?.steps?.length + ? options.steps + : asset.kind === 'audio' ? AUDIO_STEPS : DEFAULT_STEPS + const controller = new AbortController() + // Every ffmpeg call (one per scene thumbnail) attaches an abort listener to + // this shared signal — lift Node's default cap of 10 to avoid leak warnings. + setMaxListeners(0, controller.signal) + abortControllers.set(abortKey(threadId, assetId), controller) + const { signal } = controller + + await patchAsset(threadId, assetId, { preprocessState: 'running', preprocessError: undefined }) + + // Pre-register every step as pending so the UI shows the full checklist + // immediately and each bar visibly transitions pending -> running -> done. + for (const step of steps) { + await setTask(threadId, assetId, step, { state: 'pending', progress: 0, status: undefined, error: undefined }) + } + + let currentStep: PreprocessStep | null = null + try { + // A threshold re-run must re-detect scenes even if outputs exist + if (options?.threshold !== undefined) { + await patchAssetPreprocessing(threadId, assetId, { sceneTimesPath: undefined }) + } + + for (const step of steps) { + if (signal.aborted) throw new Error('Aborted') + currentStep = step + try { + switch (step) { + case 'proxy': await runProxyStep(threadId, assetId, signal); break + case 'scenes': await runScenesStep(threadId, assetId, signal, options?.threshold); break + case 'thumbnails': await runThumbnailsStep(threadId, assetId, signal); break + case 'descriptions': await runDescriptionsStep(threadId, assetId, signal); break + case 'audio': await runAudioStep(threadId, assetId, signal); break + case 'transcript': await runTranscriptStep(threadId, assetId, signal); break + case 'filmstrip': await runFilmstripStep(threadId, assetId, signal); break + case 'segment': await runSegmentStep(threadId, assetId, signal); break + } + } catch (stepError: any) { + // Audio/transcript are best-effort in the default chain: a missing + // Gemini key or quota error falls back to scene pieces instead of + // bricking the import. Everything else still hard-fails. + if (signal.aborted || !SOFT_FAIL_STEPS.includes(step)) throw stepError + const message = stepError?.message || `${step} failed` + console.warn(`[editor] ${step} failed for asset ${assetId} — continuing without it:`, message) + await setTask(threadId, assetId, step, { state: 'error', error: message }) + } + } + + // An audio asset with no transcript (no Gemini key / quota) derives zero + // pieces from the chain above — synthesize a single whole-file clip so it + // is always placeable on the audio track. + await ensureAudioHasClip(threadId, assetId) + + await patchAsset(threadId, assetId, { preprocessState: 'completed' }) + } catch (error: any) { + if (signal.aborted) { + // Asset removed or run cancelled — leave whatever state the removal left + console.log(`[editor] Preprocessing aborted for asset ${assetId}`) + return + } + const message = error?.message || 'Preprocessing failed' + console.error(`[editor] Preprocessing failed for asset ${assetId} (step: ${currentStep}):`, error) + // Mark the failed step's task and the asset errored; other assets are unaffected. + if (currentStep) { + await setTask(threadId, assetId, currentStep, { state: 'error', error: message }) + } + await patchAsset(threadId, assetId, { + preprocessState: 'error', + preprocessError: message === SCENEDETECT_MISSING + ? `${SCENEDETECT_MISSING}: PySceneDetect is not installed — scene splitting is unavailable.` + : message + }) + } finally { + abortControllers.delete(abortKey(threadId, assetId)) + } +} diff --git a/src/main/editor/prompt.ts b/src/main/editor/prompt.ts new file mode 100644 index 0000000..ca43f1e --- /dev/null +++ b/src/main/editor/prompt.ts @@ -0,0 +1,390 @@ +import { BrowserWindow } from 'electron' +import { v4 as uuidv4 } from 'uuid' +import type { + EditorDocument, EditorOps, EditorPersona, PromptTurn, Thread, TimelineDiff, TimelineItem +} from '@shared/types' +import { computeContentEnd, itemEnd, TIMELINE_DIFF_SCHEMA_VERSION } from '@shared/timeline' +import { threadManager } from '../threads' +import { settingsManager } from '../settings' +import { GeminiAdapter } from '../gemini/adapter' +import { GEMINI_MODEL_2_5_FLASH } from '../constants/gemini' +import { BUILTIN_PERSONAS, DEFAULT_PERSONA_ID } from '../constants/personas' +import { buildPromptContext } from './context' + +/** + * AI prompt orchestrator for the timeline editor (PRD §5.7). + * One structured Gemini call per turn: persona systemPrompt + fixed editor + * contract + windowed context -> EditorOps -> TimelineDiff (opsToDiff). + * Streams turn state over the dedicated `editor-turn-update` event. + * The base document is NEVER mutated here — the renderer applies the diff + * only on user accept (commitStep with origin 'ai'). + */ + +// ===== Ops schema (constrained on purpose — see EditorOps in shared/types) ===== +export const EDITOR_OPS_SCHEMA = { + type: 'object', + properties: { + answer: { + type: 'string', + description: 'Only when the request is a question — answer it and propose no operations' + }, + rationale: { + type: 'string', + description: 'Plain-language explanation of the proposed edit, citing scene descriptions' + }, + removeItemIds: { type: 'array', items: { type: 'string' } }, + updateItems: { + type: 'array', + items: { + type: 'object', + properties: { + id: { type: 'string' }, + timelineStart: { type: 'number' }, + in: { type: 'number' }, + out: { type: 'number' }, + speed: { type: 'number' }, + label: { type: 'string' } + }, + required: ['id'] + } + }, + addClips: { + type: 'array', + items: { + type: 'object', + properties: { + assetId: { type: 'string' }, + sceneIndex: { type: 'integer', description: "A scene # from that asset's AVAILABLE SCENES list" }, + in: { type: 'number' }, + out: { type: 'number' }, + atSec: { type: 'number' }, + afterItemId: { type: 'string' }, + label: { type: 'string' } + }, + required: ['assetId'] + } + }, + addMarkers: { + type: 'array', + items: { + type: 'object', + properties: { + atSec: { type: 'number' }, + label: { type: 'string' } + }, + required: ['atSec', 'label'] + } + } + } +} + +// ===== System instruction composition ===== +export function composeSystemInstruction(persona: EditorPersona): string { + const defaults = persona.defaults || {} + const durationLine = defaults.targetDurationSec != null + ? `Target output duration: about ${defaults.targetDurationSec} seconds. Cut toward this target.` + : 'No length target — PRESERVE the full runtime. Make editorial improvements (remove dead air, tighten with retime, reorder, chapter) without shrinking the substantive content.' + + return [ + persona.systemPrompt, + '', + '=== EDITOR CONTRACT (always applies) ===', + 'You are proposing an edit to a video timeline in a non-linear editor. You receive the', + "current timeline items (each with a stable id), the project's media assets with their", + 'detected scenes, and a user request.', + '- If the request is a QUESTION about the project, put the answer in `answer` and', + ' propose no operations.', + '- Otherwise propose ONE coherent edit:', + ' - Reference existing timeline items ONLY by their exact `id` from the items list.', + ' - Add new material ONLY via `addClips`, referencing an `assetId` plus a scene `#`', + " from that asset's AVAILABLE SCENES list (preferred), or explicit in/out seconds", + " within the asset's duration.", + ' - Never invent ids or scene numbers. All times are seconds.', + ' - `updateItems` may change timelineStart, in, out, speed (0.25-4.0), label.', + ' Never set durations — they are derived from (out - in) / speed.', + ' - Only modify items listed in the items section.', + '- Always include a short `rationale` describing what you changed and why, citing', + ' scene descriptions where available.', + '', + '=== ACTIVE PERSONA DEFAULTS ===', + `Tone: ${persona.tone || 'neutral'}. Pacing: ${defaults.pacing || 'balanced'}.`, + durationLine, + defaults.aspectRatio ? `Target aspect ratio: ${defaults.aspectRatio}.` : '' + ].filter(Boolean).join('\n') +} + +// ===== Ops -> TimelineDiff mapping ===== +export function opsToDiff( + ops: EditorOps, + doc: EditorDocument +): { diff: TimelineDiff; addMarkers: { time: number; label: string }[]; droppedOps: string[] } { + const droppedOps: string[] = [] + const diff: TimelineDiff = { schemaVersion: TIMELINE_DIFF_SCHEMA_VERSION } + const itemIds = new Set(doc.timeline.map((i) => i.id)) + + // Removals + if (ops.removeItemIds?.length) { + const known = ops.removeItemIds.filter((id) => itemIds.has(id)) + for (const id of ops.removeItemIds) { + if (!itemIds.has(id)) droppedOps.push(`remove: unknown item ${id}`) + } + if (known.length) diff.removeItemIds = known + } + + // Updates (whitelist fields; validation happens in applyTimelineDiff) + if (ops.updateItems?.length) { + const updates: NonNullable = [] + for (const update of ops.updateItems) { + if (!itemIds.has(update.id)) { + droppedOps.push(`update: unknown item ${update.id}`) + continue + } + const clean: { id: string } & Partial = { id: update.id } + if (typeof update.timelineStart === 'number') clean.timelineStart = update.timelineStart + if (typeof update.in === 'number') clean.in = update.in + if (typeof update.out === 'number') clean.out = update.out + if (typeof update.speed === 'number') clean.speed = update.speed + if (typeof update.label === 'string') clean.label = update.label + if (Object.keys(clean).length > 1) updates.push(clean) + } + if (updates.length) diff.updateItems = updates + } + + // Adds: resolve asset + scene references; ids generated HERE, never by the model + if (ops.addClips?.length) { + const adds: TimelineItem[] = [] + // Sequential placement cursor for adds without explicit position + let appendCursor = computeContentEnd(doc.timeline) + const targetTrack = [...doc.tracks] + .sort((a, b) => a.order - b.order) + .find((t) => t.kind === 'video' && !t.locked && !t.hidden) + + for (const add of ops.addClips) { + const asset = doc.media.find((a) => a.id === add.assetId) + if (!asset) { + droppedOps.push(`add: unknown asset ${add.assetId}`) + continue + } + if (!targetTrack) { + droppedOps.push('add: no unlocked video track available') + break + } + + let sourceIn: number | undefined + let sourceOut: number | undefined + let sourceClipId: string | undefined + let masterSegmentIndex: number | undefined + let label = add.label + + if (typeof add.sceneIndex === 'number') { + const clip = asset.clips.find((c) => c.index === add.sceneIndex) + if (!clip) { + droppedOps.push(`add: unknown scene #${add.sceneIndex} of ${asset.name}`) + continue + } + sourceIn = clip.in + sourceOut = clip.out + sourceClipId = clip.id + masterSegmentIndex = clip.masterSegmentIndex + label = label || clip.visual?.slice(0, 40) || `Piece #${clip.index}` + } else if (typeof add.in === 'number' && typeof add.out === 'number') { + const assetDuration = asset.metadata?.duration ?? Number.POSITIVE_INFINITY + if (!(add.in >= 0 && add.out > add.in && add.out <= assetDuration + 0.01)) { + droppedOps.push(`add: invalid range ${add.in}-${add.out} for ${asset.name}`) + continue + } + sourceIn = add.in + sourceOut = add.out + label = label || asset.name + } else { + droppedOps.push(`add: neither sceneIndex nor in/out given for ${asset.name}`) + continue + } + + // Placement: explicit atSec > after a known item > append at end + let timelineStart: number + if (typeof add.atSec === 'number' && add.atSec >= 0) { + timelineStart = add.atSec + } else if (add.afterItemId && itemIds.has(add.afterItemId)) { + const anchor = doc.timeline.find((i) => i.id === add.afterItemId)! + timelineStart = itemEnd(anchor) + } else { + timelineStart = appendCursor + } + + const duration = sourceOut! - sourceIn! + adds.push({ + id: uuidv4(), + trackId: targetTrack.id, + sourceAssetId: asset.id, + sourceClipId, + masterSegmentIndex, + timelineStart, + in: sourceIn!, + out: sourceOut!, + speed: 1.0, + preservePitch: true, + duration, + label + }) + appendCursor = Math.max(appendCursor, timelineStart + duration) + } + if (adds.length) diff.addItems = adds + } + + const addMarkers = (ops.addMarkers || []) + .filter((m) => typeof m.atSec === 'number' && m.atSec >= 0 && m.label) + .map((m) => ({ time: m.atSec, label: m.label })) + + return { diff, addMarkers, droppedOps } +} + +// ===== Turn lifecycle ===== +const turnControllers = new Map() + +function emitTurnUpdate(payload: { + threadId: string + turn: PromptTurn + addMarkers?: { time: number; label: string }[] + thinContext?: boolean + truncated?: boolean +}) { + BrowserWindow.getAllWindows().forEach((win) => { + win.webContents.send('editor-turn-update', payload) + }) +} + +async function persistTurn(threadId: string, turn: PromptTurn): Promise { + await threadManager.updateThreadWith(threadId, (thread) => { + if (!thread.editor) return null + const turns = [...thread.editor.turns] + const index = turns.findIndex((t) => t.id === turn.id) + if (index === -1) turns.push(turn) + else turns[index] = turn + return { editor: { ...thread.editor, turns } } + }) +} + +function resolvePersona(doc: EditorDocument, personaId: string): EditorPersona { + return ( + doc.customPersonas?.find((p) => p.id === personaId) || + settingsManager.getPersonas().find((p) => p.id === personaId) || + BUILTIN_PERSONAS.find((p) => p.id === personaId) || + BUILTIN_PERSONAS.find((p) => p.id === DEFAULT_PERSONA_ID)! + ) +} + +export function abortEditorPrompt(turnId: string): void { + const controller = turnControllers.get(turnId) + if (controller) { + controller.abort() + turnControllers.delete(turnId) + } +} + +/** + * Runs one prompt turn. Returns the turnId immediately; work continues async + * and streams over `editor-turn-update`. + */ +export function runEditorPrompt(options: { + threadId: string + personaId: string + prompt: string + baseStepId: string + selectedItemIds: string[] + playheadSec: number + widen?: 'chapter' | 'full' +}): { turnId: string } { + const { threadId, personaId, prompt, baseStepId } = options + const turnId = uuidv4() + + const turn: PromptTurn = { + id: turnId, + personaId, + prompt, + baseStepId, + status: 'running', + createdAt: Date.now() + } + + // Async body — errors land on the turn record, never thrown to the caller + void (async () => { + const controller = new AbortController() + turnControllers.set(turnId, controller) + try { + const thread: Thread | null = threadManager.getThread(threadId) + if (!thread || thread.type !== 'editor' || !thread.editor) { + throw new Error('Not an editor project') + } + if (thread.editor.media.length === 0) { + throw new Error('Import media before prompting') + } + const doc = thread.editor + + await persistTurn(threadId, turn) + emitTurnUpdate({ threadId, turn }) + + const persona = resolvePersona(doc, personaId) + const context = buildPromptContext(doc, prompt, { + selectedItemIds: options.selectedItemIds, + playheadSec: options.playheadSec, + widen: options.widen + }) + + const modelSettings = settingsManager.getModelSettings() + const modelName = modelSettings.selection['editor-edit'] || GEMINI_MODEL_2_5_FLASH + + const adapter = GeminiAdapter.create() + const { data: ops, record } = await adapter.generateStructuredText( + modelName, + context.contextText, + EDITOR_OPS_SCHEMA, + composeSystemInstruction(persona), + controller.signal + ) + + // Record usage/cost at project level + await threadManager.updateThreadWith(threadId, (t) => ({ + usageHistory: [...(t.usageHistory || []), { ...record, timestamp: Date.now() }] + })) + + if (controller.signal.aborted) throw new Error('Cancelled') + + // Map ops against a FRESH read (doc may have advanced during the call); + // the renderer re-validates against ITS live doc anyway. + const freshDoc = threadManager.getThread(threadId)?.editor || doc + const { diff, addMarkers, droppedOps } = opsToDiff(ops || {}, freshDoc) + + const completed: PromptTurn = { + ...turn, + status: 'completed', + diff, + rationale: ops?.rationale, + answer: ops?.answer, + droppedOps: droppedOps.length ? droppedOps : undefined, + scopeLabel: context.scope.label, + usage: record.usage, + cost: record.cost + } + await persistTurn(threadId, completed) + emitTurnUpdate({ + threadId, + turn: completed, + addMarkers, + thinContext: context.thinContext, + truncated: context.truncated + }) + } catch (error: any) { + const message = controller.signal.aborted ? 'Cancelled' : (error?.message || 'Prompt failed') + const failed: PromptTurn = { ...turn, status: 'error', error: message } + try { + await persistTurn(threadId, failed) + } catch { /* best effort */ } + emitTurnUpdate({ threadId, turn: failed }) + } finally { + turnControllers.delete(turnId) + } + })() + + return { turnId } +} diff --git a/src/main/editor/render.ts b/src/main/editor/render.ts new file mode 100644 index 0000000..e19d59f --- /dev/null +++ b/src/main/editor/render.ts @@ -0,0 +1,502 @@ +import { BrowserWindow } from 'electron' +import ffmpeg from 'fluent-ffmpeg' +import fs from 'fs' +import path from 'path' +import process from 'node:process' +import { v4 as uuidv4 } from 'uuid' +import type { + EditorDocument, EditorRenderProgress, ExportQuality, MediaAsset, + TimelineItem, TimelineSegment, Track +} from '@shared/types' +import { itemEnd } from '@shared/timeline' +import { threadManager } from '../threads' +import { assembleVideo, sanitizeFilename } from '../ffmpeg' +import { THREAD_DIRS } from '../constants/paths' + +/** + * Export render engine (PRD §5.9 / §7 option C — segment-then-concat). + * + * Fast path: a single-source, speed-1, unmuted, gapless timeline maps to + * TimelineSegment[] and reuses assembleVideo unchanged. + * + * Region path: the sequence is sliced at clip boundaries into regions + * (clip | gap); each region renders to a UNIFORM mp4 intermediate + * (h264 + aac 48kHz stereo, normalized W×H/FPS, both streams always + * present), then the concat demuxer stitches with -c copy. Per-item + * speed renders via setpts + chained atempo (or asetrate when + * preservePitch=false). Gaps render black + silence. + * + * The region model carries audioSources[] (0..1 entries today) so the + * P2 multi-source amix graph slots in without reshaping. + */ + +const IS_MAC = process.platform === 'darwin' +const GAP_EPS = 0.05 + +// ===== Region model ===== +// Regions are sliced at the UNION of every video- AND audio-item boundary, so +// each region is fully covered (or not) by each item — never partially. A +// region carries at most one video slice (the primary video track) plus any +// number of active audio sources (the video's own audio + audio-track items), +// mixed to one stereo output at render time (§5.9). + +interface VideoSlice { + srcPath: string + in: number + out: number + speed: number +} + +interface AudioSource { + srcPath: string + in: number + out: number + speed: number + preservePitch: boolean + gain: number +} + +interface Region { + duration: number + video: VideoSlice | null // null → black frames of `duration` + audioSources: AudioSource[] // empty → silence of `duration` +} + +interface RegionPlan { + regions: Region[] + width: number + height: number + fps: number +} + +/** Map a covering item's source range onto a sub-region [t0,t1) of the timeline. */ +function sliceSource(item: TimelineItem, t0: number, t1: number): { in: number; out: number; speed: number } { + const speed = item.speed || 1 + return { + in: item.in + (t0 - item.timelineStart) * speed, + out: item.in + (t1 - item.timelineStart) * speed, + speed + } +} + +export function computeRegions(doc: EditorDocument, quality: ExportQuality): RegionPlan { + const videoTrack = doc.tracks + .filter((t) => t.kind === 'video' && !t.hidden) + .sort((a, b) => a.order - b.order)[0] || null + const audioTrackById = new Map( + doc.tracks.filter((t) => t.kind === 'audio' && !t.hidden).map((t) => [t.id, t]) + ) + + const videoItems = videoTrack + ? doc.timeline.filter((i) => i.trackId === videoTrack.id).sort((a, b) => a.timelineStart - b.timelineStart) + : [] + const audioItems = doc.timeline + .filter((i) => audioTrackById.has(i.trackId)) + .sort((a, b) => a.timelineStart - b.timelineStart) + + if (videoItems.length === 0 && audioItems.length === 0) { + throw new Error('Nothing to export — the timeline is empty.') + } + + const assetById = new Map(doc.media.map((a) => [a.id, a])) + const srcFor = (asset: MediaAsset) => + quality === 'preview' ? (asset.proxyPath || asset.originalPath) : asset.originalPath + + // Normalization target: derived from the VIDEO assets in use (audio assets + // carry no picture). Defaults keep an audio-only export at a sane 720p. + const usedVideoAssets = [...new Set(videoItems.map((i) => i.sourceAssetId))] + .map((id) => assetById.get(id)) + .filter((a): a is MediaAsset => !!a) + const even = (n: number) => Math.max(2, Math.floor(n / 2) * 2) + const largest = usedVideoAssets.reduce((best, a) => { + const area = (a.metadata?.width || 0) * (a.metadata?.height || 0) + return area > best.area ? { area, w: a.metadata!.width, h: a.metadata!.height } : best + }, { area: 0, w: 1280, h: 720 }) + const width = even(largest.w) + const height = even(largest.h) + const fps = Math.min(60, Math.max(10, + usedVideoAssets.reduce((max, a) => Math.max(max, a.metadata?.fps || 0), 0) || 30 + )) + + // Boundary set = every video- and audio-item edge, clamped to [0, end]. + const allItems = [...videoItems, ...audioItems] + const end = allItems.reduce((max, it) => Math.max(max, itemEnd(it)), 0) + const cuts = new Set([0, end]) + for (const it of allItems) { + cuts.add(Math.min(Math.max(it.timelineStart, 0), end)) + cuts.add(Math.min(Math.max(itemEnd(it), 0), end)) + } + const points = [...cuts].sort((a, b) => a - b) + + // A video item's own audio contributes when unmuted and the source has audio. + const videoAudioActive = (item: TimelineItem, asset: MediaAsset) => + !item.muted && !(videoTrack?.muted) && asset.metadata?.hasAudio !== false + // An audio-track item contributes when neither it nor its track is muted. + const audioItemActive = (item: TimelineItem) => + !item.muted && !audioTrackById.get(item.trackId)?.muted + + const covers = (item: TimelineItem, t0: number, t1: number) => + item.timelineStart <= t0 + GAP_EPS && itemEnd(item) >= t1 - GAP_EPS + + const regions: Region[] = [] + for (let i = 0; i < points.length - 1; i++) { + const t0 = points[i] + const t1 = points[i + 1] + const duration = t1 - t0 + if (duration <= GAP_EPS) continue + + // ---- Video: the single covering video-track item (if any) ---- + let video: VideoSlice | null = null + const vItem = videoItems.find((it) => covers(it, t0, t1)) + if (vItem) { + const asset = assetById.get(vItem.sourceAssetId) + if (!asset) throw new Error(`Missing media asset for clip "${vItem.label || vItem.id}".`) + const slice = sliceSource(vItem, t0, t1) + video = { srcPath: srcFor(asset), in: slice.in, out: slice.out, speed: slice.speed } + } + + // ---- Audio: video's own audio + every covering audio-track item ---- + const audioSources: AudioSource[] = [] + if (vItem) { + const asset = assetById.get(vItem.sourceAssetId)! + if (videoAudioActive(vItem, asset)) { + const slice = sliceSource(vItem, t0, t1) + audioSources.push({ + srcPath: srcFor(asset), in: slice.in, out: slice.out, + speed: slice.speed, preservePitch: vItem.preservePitch !== false, gain: vItem.gain ?? 1 + }) + } + } + for (const aItem of audioItems) { + if (!covers(aItem, t0, t1) || !audioItemActive(aItem)) continue + const asset = assetById.get(aItem.sourceAssetId) + if (!asset) continue + const slice = sliceSource(aItem, t0, t1) + audioSources.push({ + srcPath: srcFor(asset), in: slice.in, out: slice.out, + speed: slice.speed, preservePitch: aItem.preservePitch !== false, gain: aItem.gain ?? 1 + }) + } + + regions.push({ duration, video, audioSources }) + } + + return { regions, width, height, fps } +} + +// ===== Fast path ===== + +export function isFastPathEligible(items: TimelineItem[], track: Track, hasAudioItems = false): boolean { + if (items.length === 0) return false + if (hasAudioItems) return false // audio-track items need the mixing region path + if (track.muted || track.hidden) return false // assembleVideo can't drop audio + const srcId = items[0].sourceAssetId + if (!items.every((i) => i.sourceAssetId === srcId)) return false + if (!items.every((i) => (i.speed ?? 1) === 1)) return false // no setpts in assembleVideo + if (items.some((i) => i.muted)) return false + if (Math.abs(items[0].timelineStart) > GAP_EPS) return false // leading gap needs black + for (let i = 1; i < items.length; i++) { + if (Math.abs(items[i].timelineStart - itemEnd(items[i - 1])) > GAP_EPS) return false + } + return true +} + +// ===== ffmpeg building blocks ===== + +const vnorm = (w: number, h: number, fps: number) => + `scale=${w}:${h}:force_original_aspect_ratio=decrease,` + + `pad=${w}:${h}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=${fps},format=yuv420p` + +/** Chain atempo stages so each stays within ffmpeg's 0.5–2.0 range. */ +export function atempoChain(speed: number): string { + const stages: number[] = [] + let factor = speed + while (factor > 2) { stages.push(2); factor /= 2 } + while (factor < 0.5) { stages.push(0.5); factor /= 0.5 } + stages.push(factor) + return stages.map((s) => `atempo=${s.toFixed(4).replace(/0+$/, '').replace(/\.$/, '')}`).join(',') +} + +// Uniform intermediates: identical codec/res/fps/rate/layout is REQUIRED for +// concat -c copy. Forced h264+aac mp4 on all platforms (incl. webm sources). +const intermediateOpts = (): string[] => [ + ...(IS_MAC + ? ['-c:v', 'h264_videotoolbox', '-b:v', '8M', '-realtime', 'true'] + : ['-c:v', 'libx264', '-preset', 'veryfast', '-crf', '18']), + '-c:a', 'aac', '-b:a', '192k', '-ar', '48000', '-ac', '2', + '-video_track_timescale', '90000', + '-threads', '0' +] + +function runCommand( + command: ffmpeg.FfmpegCommand, + outputPath: string, + signal: AbortSignal, + onProgress?: (percent: number) => void +): Promise { + return new Promise((resolve, reject) => { + if (signal.aborted) return reject(new Error('aborted')) + signal.addEventListener('abort', () => command.kill('SIGKILL')) + command + .output(outputPath) + .on('start', (cmd) => console.log('[render] ffmpeg:', cmd)) + .on('progress', (progress) => { + if (onProgress && progress.percent) onProgress(Math.min(100, Math.round(progress.percent))) + }) + .on('end', () => resolve()) + .on('error', (err, _stdout, stderr) => { + if (signal.aborted) return reject(new Error('aborted')) + console.error('[render] ffmpeg failed:', err?.message, stderr?.slice(-800)) + reject(new Error(`Render step failed: ${err?.message || 'ffmpeg error'}`)) + }) + .run() + }) +} + +/** + * Renders ONE region to a uniform intermediate: a video slice (or black) plus + * the region's mixed audio. 0 audio sources → silence; 1 → that stream; k>1 → + * amix (normalize=0, gains already applied) so overlapping audio-track items + * and a video's own soundtrack mix correctly and stay region-length (§5.9). + */ +async function renderRegion( + region: Region, + index: number, + plan: RegionPlan, + workDir: string, + signal: AbortSignal, + onProgress: (percent: number) => void +): Promise { + const duration = region.duration + const dur = duration.toFixed(3) + const outputPath = path.join(workDir, `region-${String(index).padStart(3, '0')}.mp4`) + + const command = ffmpeg() + const filters: string[] = [] + let inputIdx = 0 + + // ---- Video: source slice, or black of exactly `duration` ---- + if (region.video) { + const v = region.video + const cut = (v.out - v.in).toFixed(3) + command.input(v.srcPath).inputOptions(['-ss', v.in.toFixed(3), '-t', cut]) + filters.push(`[${inputIdx++}:v]setpts=(PTS-STARTPTS)/${v.speed},${vnorm(plan.width, plan.height, plan.fps)}[v]`) + } else { + command.input(`color=black:size=${plan.width}x${plan.height}:rate=${plan.fps}`) + .inputFormat('lavfi').inputOptions(['-t', dur]) + filters.push(`[${inputIdx++}:v]setsar=1,format=yuv420p[v]`) + } + + // ---- Audio: silence, single stream, or amix of k streams ---- + let audioLabel = '[a]' + if (region.audioSources.length === 0) { + command.input('anullsrc=r=48000:cl=stereo').inputFormat('lavfi').inputOptions(['-t', dur]) + filters.push(`[${inputIdx++}:a]atrim=0:${dur},asetpts=PTS-STARTPTS[a]`) + } else { + const labels: string[] = [] + region.audioSources.forEach((a, k) => { + const cut = (a.out - a.in).toFixed(3) + command.input(a.srcPath).inputOptions(['-ss', a.in.toFixed(3), '-t', cut]) + const retime = a.speed === 1 + ? '' + : a.preservePitch ? `${atempoChain(a.speed)},` : `asetrate=48000*${a.speed},` + const gain = a.gain !== 1 ? `volume=${a.gain},` : '' + const label = `[a${k}]` + // apad + atrim pin every stream to EXACTLY the region duration so amix + // aligns them and the intermediate stays A/V frame-aligned for concat. + filters.push( + `[${inputIdx++}:a]asetpts=PTS-STARTPTS,${retime}${gain}aresample=48000,` + + `aformat=sample_fmts=fltp:channel_layouts=stereo,apad=whole_dur=${dur},atrim=0:${dur}${label}` + ) + labels.push(label) + }) + if (labels.length === 1) { + audioLabel = labels[0] + } else { + filters.push(`${labels.join('')}amix=inputs=${labels.length}:normalize=0[a]`) + } + } + + command + .complexFilter(filters.join(';')) + .map('[v]') + .map(audioLabel) + .outputOptions([...intermediateOpts(), '-t', dur]) + + await runCommand(command, outputPath, signal, onProgress) + return outputPath +} + +async function concatRegions( + regionFiles: string[], + workDir: string, + finalPath: string, + signal: AbortSignal, + reencode: boolean +): Promise { + const listPath = path.join(workDir, 'list.txt') + const list = regionFiles + .map((f) => `file '${path.basename(f).replace(/'/g, "'\\''")}'`) + .join('\n') + fs.writeFileSync(listPath, list) + + const command = ffmpeg(listPath) + .inputOptions(['-f', 'concat', '-safe', '0']) + .outputOptions( + reencode + ? [...intermediateOpts(), '-movflags', '+faststart'] + : ['-c', 'copy', '-movflags', '+faststart'] + ) + await runCommand(command, finalPath, signal) +} + +// ===== Orchestrator ===== + +const activeRenders = new Map() + +function emitRenderProgress(payload: EditorRenderProgress) { + BrowserWindow.getAllWindows().forEach((win) => { + win.webContents.send('editor-render-progress', payload) + }) +} + +export function abortEditorRender(renderId: string): void { + activeRenders.get(renderId)?.controller.abort() +} + +export function abortRendersForThread(threadId: string): void { + // renderIds are opaque; abort everything registered under this thread by + // checking workDir containment (workdirs live under the thread's tempDir). + const thread = threadManager.getThread(threadId) + if (!thread) return + for (const [, entry] of activeRenders) { + if (entry.workDir && entry.workDir.startsWith(thread.tempDir)) { + entry.controller.abort() + } + } +} + +export function startEditorRender(options: { threadId: string; quality: ExportQuality }): { renderId: string } { + const { threadId, quality } = options + const thread = threadManager.getThread(threadId) + if (!thread || thread.type !== 'editor' || !thread.editor) { + throw new Error('Not an editor project') + } + const doc = thread.editor + + // ---- Pre-flight (synchronous — a bad timeline rejects the invoke + // immediately instead of producing a ghost render) ---- + const plan = computeRegions(doc, quality) + const srcPaths = [...new Set( + plan.regions.flatMap((r) => [ + ...(r.video ? [r.video.srcPath] : []), + ...r.audioSources.map((a) => a.srcPath) + ]) + )] + const missing = srcPaths.filter((p) => !fs.existsSync(p)) + if (missing.length > 0) { + throw new Error(`Missing source file(s): ${missing.map((p) => path.basename(p)).join(', ')}`) + } + + const renderId = uuidv4() + const exportsDir = path.join(thread.tempDir, THREAD_DIRS.EXPORTS) + const workDir = path.join(exportsDir, renderId) + fs.mkdirSync(workDir, { recursive: true }) + + const controller = new AbortController() + activeRenders.set(renderId, { controller, workDir }) + const { signal } = controller + + const progress = (percent: number, phase: EditorRenderProgress['phase']) => + emitRenderProgress({ threadId, renderId, percent: Math.round(percent), phase }) + + // ---- Async render body ---- + void (async () => { + const baseName = sanitizeFilename(thread.title || 'export') + let finalPath = path.join(exportsDir, `${baseName}_${renderId.slice(0, 8)}.mp4`) + try { + progress(0, 'rendering') + + const videoTrack = doc.tracks + .filter((t) => t.kind === 'video' && !t.hidden) + .sort((a, b) => a.order - b.order)[0] || null + const items = videoTrack + ? doc.timeline + .filter((i) => i.trackId === videoTrack.id) + .sort((a, b) => a.timelineStart - b.timelineStart) + : [] + const audioTrackIds = new Set( + doc.tracks.filter((t) => t.kind === 'audio' && !t.hidden).map((t) => t.id) + ) + const hasAudioItems = doc.timeline.some((i) => audioTrackIds.has(i.trackId)) + + if (videoTrack && isFastPathEligible(items, videoTrack, hasAudioItems)) { + // ---- Fast path: single-source trim+concat via assembleVideo ---- + const asset = doc.media.find((a) => a.id === items[0].sourceAssetId)! + const srcPath = quality === 'preview' + ? (asset.proxyPath || asset.originalPath) + : asset.originalPath + const segments: TimelineSegment[] = items.map((item, i) => ({ + index: i + 1, + start: item.in.toFixed(3), // NEVER String(): exponent forms break timeToSeconds + end: item.out.toFixed(3), + text: '', + duration: item.out - item.in + })) + finalPath = await assembleVideo( + srcPath, + segments, + exportsDir, + renderId.slice(0, 8), + (percent) => progress(percent * 0.98, 'rendering'), + signal + ) + } else { + // ---- Region path: segment-then-concat ---- + const total = plan.regions.reduce((sum, r) => sum + r.duration, 0) || 1 + let doneWeight = 0 + let lastEmit = -1 + const regionFiles: string[] = [] + + for (let i = 0; i < plan.regions.length; i++) { + if (signal.aborted) throw new Error('aborted') + const region = plan.regions[i] + const weight = region.duration / total + const onRegionProgress = (pct: number) => { + const overall = (doneWeight + weight * (pct / 100)) * 96 + if (overall - lastEmit >= 1) { + lastEmit = overall + progress(overall, 'rendering') + } + } + const file = await renderRegion(region, i, plan, workDir, signal, onRegionProgress) + regionFiles.push(file) + doneWeight += weight + progress(doneWeight * 96, 'rendering') + } + + progress(96, 'stitching') + try { + await concatRegions(regionFiles, workDir, finalPath, signal, false) + } catch (error) { + if (signal.aborted) throw error + console.warn('[render] copy-concat failed, retrying with re-encode:', error) + await concatRegions(regionFiles, workDir, finalPath, signal, true) + } + } + + emitRenderProgress({ threadId, renderId, percent: 100, phase: 'done', outputPath: finalPath }) + } catch (error: any) { + const message = signal.aborted ? 'aborted' : (error?.message || 'Export failed') + if (!signal.aborted) console.error('[render] Export failed:', error) + // Remove a partial final file + try { if (fs.existsSync(finalPath)) fs.unlinkSync(finalPath) } catch { /* ignore */ } + emitRenderProgress({ threadId, renderId, percent: 0, phase: 'error', error: message }) + } finally { + try { fs.rmSync(workDir, { recursive: true, force: true }) } catch { /* ignore */ } + activeRenders.delete(renderId) + } + })() + + return { renderId } +} diff --git a/src/main/editor/revisions.ts b/src/main/editor/revisions.ts new file mode 100644 index 0000000..d6ed17a --- /dev/null +++ b/src/main/editor/revisions.ts @@ -0,0 +1,94 @@ +import { app } from 'electron' +import fs from 'fs' +import path from 'path' +import type { EditorRevision, EditorRevisionsFile } from '@shared/types' + +/** + * Revision-tree sidecar persistence (mirrors editor/history.ts). + * Revisions are coarse, permanent checkpoints holding FULL timeline + * snapshots — kept OUT of threads/{id}.json so the debounced doc autosave + * stays O(1), and OUT of the capped undo ring so they never dangle. + * The renderer owns the tree logic; this module only load/push/delete/persists. + */ + +const MAX_REVISIONS = 100 + +function revisionsDir(): string { + return path.join(app.getPath('userData'), 'editor-revisions') +} + +function revisionsPath(threadId: string): string { + return path.join(revisionsDir(), `${threadId}.json`) +} + +function emptyFile(threadId: string): EditorRevisionsFile { + return { threadId, schemaVersion: 1, revisionCounter: 0, revisions: [] } +} + +function writeRevisions(file: EditorRevisionsFile) { + const dir = revisionsDir() + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }) + const target = revisionsPath(file.threadId) + const temp = `${target}.tmp` + fs.writeFileSync(temp, JSON.stringify(file)) + fs.renameSync(temp, target) +} + +export function loadRevisions(threadId: string): EditorRevisionsFile { + const filePath = revisionsPath(threadId) + if (!fs.existsSync(filePath)) return emptyFile(threadId) + try { + const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8')) as EditorRevisionsFile + if (!Array.isArray(parsed.revisions)) return emptyFile(threadId) + return { ...emptyFile(threadId), ...parsed, threadId } + } catch (error) { + console.error(`[editor-revisions] Failed to read revisions for ${threadId}:`, error) + return emptyFile(threadId) + } +} + +/** + * Appends a revision: assigns a monotonic seq (max-ever — V numbers never + * reused after deletes) and enforces the cap by pruning the OLDEST LEAF + * (no children) that is neither the root nor the just-pushed revision. + * Leaf-only pruning guarantees no surviving revision loses an ancestor. + */ +export function pushRevision(threadId: string, revision: EditorRevision): { seq: number; count: number } { + const file = loadRevisions(threadId) + const seq = ++file.revisionCounter + const stored: EditorRevision = { ...revision, seq } + file.revisions.push(stored) + + while (file.revisions.length > MAX_REVISIONS) { + const hasChildren = new Set(file.revisions.map((r) => r.parentId).filter(Boolean)) + const candidate = file.revisions + .filter((r) => r.parentId !== null && r.id !== stored.id && !hasChildren.has(r.id)) + .sort((a, b) => a.createdAt - b.createdAt)[0] + if (!candidate) break // pathological pure chain — allow exceeding the cap + file.revisions = file.revisions.filter((r) => r.id !== candidate.id) + } + + writeRevisions(file) + return { seq, count: file.revisions.length } +} + +/** Deletes the given ids (the renderer sends the fully collected subtree). */ +export function deleteRevisions(threadId: string, ids: string[]): boolean { + const file = loadRevisions(threadId) + const remove = new Set(ids) + // Never delete the root + const root = file.revisions.find((r) => r.parentId === null) + if (root) remove.delete(root.id) + file.revisions = file.revisions.filter((r) => !remove.has(r.id)) + writeRevisions(file) + return true +} + +export function deleteRevisionsFile(threadId: string): void { + const filePath = revisionsPath(threadId) + try { + if (fs.existsSync(filePath)) fs.unlinkSync(filePath) + } catch (error) { + console.error(`[editor-revisions] Failed to delete revisions for ${threadId}:`, error) + } +} diff --git a/src/main/ffmpeg/index.ts b/src/main/ffmpeg/index.ts index 2e0fc12..71130ff 100644 --- a/src/main/ffmpeg/index.ts +++ b/src/main/ffmpeg/index.ts @@ -1,9 +1,10 @@ import ffmpeg from 'fluent-ffmpeg' import ffmpegPath from 'ffmpeg-static' import ffprobePath from 'ffprobe-static' +import fs from 'fs' import { join, basename, extname } from 'path' import process from 'node:process' -import { TimelineSegment } from '../../shared/types' +import { TimelineSegment, SilenceRegion } from '../../shared/types' const IS_MAC = process.platform === 'darwin' @@ -130,6 +131,33 @@ export async function getVideoMetadata(filePath: string): Promise { + return new Promise((resolve, reject) => { + ffmpeg.ffprobe(filePath, (err, metadata) => { + if (err) return reject(err) + const audioStream = metadata.streams.find((s) => s.codec_type === 'audio') + if (!audioStream) { + return reject(new Error('No audio stream found')) + } + resolve({ + duration: metadata.format.duration || 0, + width: 0, + height: 0, + size: metadata.format.size || 0, + codec: audioStream.codec_name || 'unknown', + fps: 0, + format: metadata.format.format_name || 'unknown', + hasAudio: true + }) + }) + }) +} + /** * Returns true if the video resolution is 480p or lower. */ @@ -146,11 +174,16 @@ export async function isVideoLowResolution(filePath: string): Promise { /** * Converts video to low resolution (480p). */ +// A proxy for scrubbing / thumbnails / scene detection never needs more than +// this; halving a 60fps source roughly halves proxy encode time. +const PROXY_MAX_FPS = 30 + export async function toLowResolution( filePath: string, outputDir: string, onProgress?: (percent: number) => void, - signal?: AbortSignal + signal?: AbortSignal, + opts?: { sourceFps?: number } ): Promise { const ext = extname(filePath).toLowerCase() const rawFilename = basename(filePath, extname(filePath)) @@ -161,12 +194,23 @@ export async function toLowResolution( throw new Error('FFmpeg downscaling aborted by user before start') } - return new Promise((resolve, reject) => { - const isWebm = ext === '.webm' + const isWebm = ext === '.webm' + + // Cap fps only when the source is above the ceiling — applying fps=30 to a + // <=30fps source would DUPLICATE frames (slower + larger), not drop any. + const capFps = !!opts?.sourceFps && opts.sourceFps > PROXY_MAX_FPS + const vf = capFps ? `fps=${PROXY_MAX_FPS},scale=-2:480` : 'scale=-2:480' + // Hardware-accelerated DECODE (Mac) is a clear win for decode-bound proxies + // (~-25%). But when we're already dropping frames via the fps filter, + // benchmarks show the per-frame GPU→CPU download makes it slower than plain + // software decode — so only use hw decode when NOT capping fps. + const preferHwDecode = IS_MAC && !isWebm && !capFps + const encode = (hwDecode: boolean): Promise => new Promise((resolve, reject) => { const command = ffmpeg(filePath) - .outputOptions(['-vf', 'scale=-2:480']) + if (hwDecode) command.inputOptions(['-hwaccel', 'videotoolbox']) + command.outputOptions(['-vf', vf]) if (isWebm) { // WebM (VP8/VP9) optimizations @@ -179,8 +223,7 @@ export async function toLowResolution( '-b:v', '1M' ]) } else if (IS_MAC) { - // Mac hardware acceleration - + // Mac hardware encode command.outputOptions([ '-c:v', 'h264_videotoolbox', '-b:v', '2M', @@ -199,10 +242,7 @@ export async function toLowResolution( } if (signal) { - signal.addEventListener('abort', () => { - console.log('FFmpeg toLowResolution aborted by signal') - command.kill('SIGKILL') - }) + signal.addEventListener('abort', () => command.kill('SIGKILL')) } command @@ -223,6 +263,18 @@ export async function toLowResolution( }) .run() }) + + try { + return await encode(preferHwDecode) + } catch (err) { + // Hardware decode can reject some inputs — fall back to software decode + // once rather than failing the whole proxy step. + if (!signal?.aborted && preferHwDecode) { + console.warn('[ffmpeg] hw-decode proxy failed, retrying with software decode:', (err as Error).message) + return await encode(false) + } + throw err + } } /** @@ -468,3 +520,126 @@ export async function extractFrame( command.run() }) } + +/** + * Batched filmstrip extractor (PRD §5.5). ONE ffmpeg pass samples a frame + * every `intervalSec` seconds (`fps=1/interval`) at 120px height — unlike + * extractFrame's process-per-frame, a multi-hour source stays a single + * bounded run. Returns entries mapping each frame to its source time. + */ +export async function generateFilmstrip( + videoPath: string, + outputDir: string, + intervalSec: number, + signal?: AbortSignal, + onProgress?: (percent: number) => void +): Promise<{ time: number; thumbnailPath: string }[]> { + if (signal?.aborted) { + throw new Error('Filmstrip generation aborted before start') + } + + const pattern = join(outputDir, 'strip_%05d.jpg') + + await new Promise((resolve, reject) => { + const command = ffmpeg(videoPath) + .outputOptions([ + '-vf', `fps=1/${intervalSec},scale=-2:120`, + '-q:v', '5' + ]) + .output(pattern) + .on('progress', (progress) => { + if (onProgress && progress.percent) { + onProgress(Math.round(progress.percent)) + } + }) + .on('end', () => resolve()) + .on('error', (err) => { + if (signal?.aborted) { + return reject(new Error('Filmstrip generation aborted by user')) + } + console.error('Error generating filmstrip:', err) + reject(err) + }) + + if (signal) { + signal.addEventListener('abort', () => command.kill('SIGKILL')) + } + + command.run() + }) + + return fs.readdirSync(outputDir) + .filter((f) => /^strip_\d+\.jpg$/.test(f)) + .sort() + .map((f, i) => ({ + // fps=1/N emits the frame representing window [iN, (i+1)N) — stamp it + // at the window centre so nearest-entry lookup lands inside the window. + time: (i + 0.5) * intervalSec, + thumbnailPath: join(outputDir, f) + })) +} + +/** + * Assistive silence/dead-air finder (PRD §5.6). Runs ffmpeg's `silencedetect` + * audio filter over the source and parses the `silence_start`/`silence_end` + * markers off stderr into source-time regions. Read-only analysis — it never + * mutates media; the caller reviews the ranges before applying any cut. + * + * Requires an audio stream — guard with `metadata.hasAudio` before calling. + */ +export async function detectSilence( + videoPath: string, + opts?: { noiseDb?: number; minDurationSec?: number }, + signal?: AbortSignal, + onProgress?: (percent: number) => void +): Promise { + const noiseDb = opts?.noiseDb ?? -30 + const minDurationSec = opts?.minDurationSec ?? 0.5 + + if (signal?.aborted) { + throw new Error('Silence detection aborted before start') + } + + return new Promise((resolve, reject) => { + const regions: SilenceRegion[] = [] + let pendingStart: number | null = null + + const command = ffmpeg(videoPath) + .audioFilters(`silencedetect=noise=${noiseDb}dB:d=${minDurationSec}`) + .outputOptions(['-f', 'null']) + .output(process.platform === 'win32' ? 'NUL' : '/dev/null') + .on('progress', (progress) => { + if (onProgress && progress.percent) { + onProgress(Math.round(progress.percent)) + } + }) + .on('stderr', (line: string) => { + const startMatch = line.match(/silence_start:\s*(-?[\d.]+)/) + if (startMatch) { + pendingStart = parseFloat(startMatch[1]) + return + } + const endMatch = line.match(/silence_end:\s*(-?[\d.]+)/) + if (endMatch) { + const end = parseFloat(endMatch[1]) + const start = Math.max(0, pendingStart ?? 0) + if (Number.isFinite(end) && end > start) regions.push({ start, end }) + pendingStart = null + } + }) + .on('end', () => resolve(regions)) + .on('error', (err) => { + if (signal?.aborted) { + return reject(new Error('Silence detection aborted by user')) + } + console.error('Error during silence detection:', err) + reject(err) + }) + + if (signal) { + signal.addEventListener('abort', () => command.kill('SIGKILL')) + } + + command.run() + }) +} diff --git a/src/main/index.ts b/src/main/index.ts index 9659da5..31fe4a1 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -18,10 +18,18 @@ import * as imageGeneration from './pipeline/phases/image-generation' import { backgroundTaskManager } from './tasks' import { GeminiAdapter } from './gemini/adapter' -import { checkFFmpegAvailability, getVideoMetadata } from './ffmpeg' +import { checkFFmpegAvailability, getVideoMetadata, detectSilence } from './ffmpeg' import { checkScenedetectAvailability } from './scenedetect' import { checkYtDlpAvailability, downloadVideo, getVideoFormats } from './ytdlp' import { dependencyManager } from './dependencies/manager' +import * as editorAssets from './editor/assets' +import * as editorPreprocess from './editor/preprocess' +import * as editorHistory from './editor/history' +import * as editorPrompt from './editor/prompt' +import * as editorRender from './editor/render' +import * as editorRevisions from './editor/revisions' +import { BUILTIN_PERSONAS } from './constants/personas' +import { v4 as uuidv4 } from 'uuid' import { THREAD_DIRS } from './constants/paths' import { GEMINI_MODEL_2_5_FLASH, MODEL_METADATA } from './constants/gemini' import { electronApp, optimizer, is } from '@electron-toolkit/utils' @@ -111,7 +119,11 @@ app.whenReady().then(() => { ipcMain.handle('select-video', async () => { const result = await dialog.showOpenDialog({ properties: ['openFile'], - filters: [{ name: 'Videos', extensions: ['mp4', 'avi', 'mov', 'webm'] }] + filters: [ + { name: 'Media', extensions: ['mp4', 'avi', 'mov', 'webm', 'mp3', 'wav', 'm4a', 'aac', 'flac', 'ogg', 'opus', 'wma'] }, + { name: 'Videos', extensions: ['mp4', 'avi', 'mov', 'webm'] }, + { name: 'Audio', extensions: ['mp3', 'wav', 'm4a', 'aac', 'flac', 'ogg', 'opus', 'wma'] } + ] }) if (result.canceled || result.filePaths.length === 0) { @@ -329,6 +341,16 @@ app.whenReady().then(() => { return MODEL_METADATA }) + // Editor personas: built-ins (code) merged with the user's global library + ipcMain.handle('get-personas', () => { + return [...BUILTIN_PERSONAS, ...settingsManager.getPersonas()] + }) + + ipcMain.handle('set-personas', (_event, personas: any[]) => { + const saved = settingsManager.setPersonas(personas) + return [...BUILTIN_PERSONAS, ...saved] + }) + // Thread Management ipcMain.handle('create-thread', async (_event, { videoPath, videoName, imagePaths }) => { const newThread = await threadManager.createThread(videoPath, videoName, imagePaths) @@ -345,6 +367,270 @@ app.whenReady().then(() => { return true }) + // ===== Timeline Video Editor ===== + + // Creates an editor project thread. Deliberately does NOT auto-start + // preprocessing (media is imported per-asset inside the editor). + ipcMain.handle('create-editor-project', async (_event, { title }: { title?: string }) => { + return await threadManager.createEditorThread(title || 'Untitled Project') + }) + + ipcMain.handle('rename-editor-project', async (_event, { threadId, title }: { + threadId: string, title: string + }) => { + const clean = (title || '').trim().slice(0, 120) + if (!clean) return null + return await threadManager.updateThread(threadId, { title: clean }) + }) + + // Autosave of RENDERER-OWNED editor fields only. Main owns preprocessing-derived + // asset state (preprocessing/proxyPath/metadata/preprocessState/clips content); + // the renderer owns selection, clip-selected flags, tracks, timeline, meta, personas. + // This ownership split is what prevents autosave/thread-updated echo clobbering. + ipcMain.handle('save-editor-doc', async (_event, { threadId, patch }: { + threadId: string + patch: { + tracks?: any[] + timeline?: any[] + timelineMeta?: any + selection?: any + activePersonaId?: string + customPersonas?: any[] + clipSelections?: Record + historyRef?: any + markers?: any[] + currentRevisionId?: string + } + }) => { + return await threadManager.updateThreadWith(threadId, (thread) => { + if (thread.type !== 'editor' || !thread.editor) return null + + const editor = { ...thread.editor } + if (patch.tracks) editor.tracks = patch.tracks + if (patch.timeline) editor.timeline = patch.timeline + if (patch.timelineMeta) editor.timelineMeta = patch.timelineMeta + if (patch.selection) editor.selection = patch.selection + if (patch.activePersonaId !== undefined) editor.activePersonaId = patch.activePersonaId + if (patch.customPersonas) editor.customPersonas = patch.customPersonas + // historyRef travels WITH the doc so the undo pointer and the timeline + // state land in one atomic write (crash-consistent by construction). + if (patch.historyRef) editor.historyRef = patch.historyRef + if (patch.markers) editor.markers = patch.markers + if (patch.currentRevisionId !== undefined) editor.currentRevisionId = patch.currentRevisionId + + // Fold clip-selected flags into the matching clips (the one renderer-owned + // field living inside main-owned MediaAsset records). + if (patch.clipSelections) { + editor.media = editor.media.map(asset => ({ + ...asset, + clips: asset.clips.map(clip => + patch.clipSelections![clip.id] !== undefined + ? { ...clip, selected: patch.clipSelections![clip.id] } + : clip + ) + })) + } + + return { editor } + }) + }) + + // Import a local file as a MediaAsset and start per-asset preprocessing. + ipcMain.handle('add-media-asset', async (_event, { threadId, filePath, name }: { + threadId: string, filePath: string, name?: string + }) => { + const asset = await editorAssets.createMediaAsset(threadId, { sourcePath: filePath, name, referenceInPlace: true }) + if (asset && asset.preprocessState !== 'error') { + // Fire-and-forget: progress streams via background-task-update + editorPreprocess.preprocessMediaAsset(threadId, asset.id).catch((error) => { + console.error(`[editor] preprocess failed for ${asset.id}:`, error) + }) + } + return asset + }) + + // Import from URL/YouTube: download straight into the asset's source dir, + // with per-asset progress (the legacy 'download-progress' event is keyless + // and breaks under concurrent imports). + ipcMain.handle('import-media-url', async (event, { threadId, url, resolution }: { + threadId: string, url: string, resolution?: string + }) => { + const thread = threadManager.getThread(threadId) + if (!thread || thread.type !== 'editor') throw new Error('Not an editor project') + + const assetId = uuidv4() + const sourceDir = join(thread.tempDir, 'media', assetId, 'source') + fs.mkdirSync(sourceDir, { recursive: true }) + + try { + const result = await downloadVideo(url, sourceDir, resolution, (percent) => { + event.sender.send('editor-import-progress', { threadId, assetId, url, percent }) + }) + + const asset = await editorAssets.createMediaAsset(threadId, { + sourcePath: result.path, + name: result.name, + assetId + }) + if (asset && asset.preprocessState !== 'error') { + editorPreprocess.preprocessMediaAsset(threadId, asset.id).catch((error) => { + console.error(`[editor] preprocess failed for ${asset.id}:`, error) + }) + } + return asset + } catch (error) { + // Failed import: remove the partial asset dir so nothing dangles + try { + fs.rmSync(join(thread.tempDir, 'media', assetId), { recursive: true, force: true }) + } catch { /* best effort */ } + throw error + } + }) + + ipcMain.handle('remove-media-asset', async (_event, { threadId, assetId }: { + threadId: string, assetId: string + }) => { + return await editorAssets.removeAsset(threadId, assetId) + }) + + // Retry / re-run / opt-in steps (e.g. ['descriptions']) for one asset. + ipcMain.handle('preprocess-media', async (_event, { threadId, assetId, steps, threshold }: { + threadId: string, assetId: string, steps?: string[], threshold?: number + }) => { + editorPreprocess.preprocessMediaAsset(threadId, assetId, { + steps: steps as any, + threshold + }).catch((error) => { + console.error(`[editor] preprocess retry failed for ${assetId}:`, error) + }) + return true + }) + + // Remove derived Gemini data (transcript / scene descriptions) from an asset. + ipcMain.handle('clear-asset-data', (_event, { threadId, assetId, kind }: { + threadId: string, assetId: string, kind: 'transcript' | 'descriptions' + }) => { + return editorAssets.clearAssetData(threadId, assetId, kind) + }) + + // Scene-piece corrections (§5.2): merge adjacent pieces / split further. + ipcMain.handle('merge-clips', (_event, { threadId, assetId, clipIds }: { + threadId: string, assetId: string, clipIds: string[] + }) => { + return editorAssets.mergeClips(threadId, assetId, clipIds) + }) + + ipcMain.handle('split-clip', (_event, { threadId, assetId, clipId, atSec }: { + threadId: string, assetId: string, clipId: string, atSec?: number + }) => { + return editorAssets.splitClip(threadId, assetId, clipId, atSec) + }) + + // Assistive silence/dead-air finder (§5.6). Read-only analysis over the + // asset's proxy/original — returns candidate source-time ranges the user + // reviews before applying as ripple-deletes. Never mutates media. + ipcMain.handle('find-silence', async (event, { threadId, assetId, noiseDb, minDurationSec }: { + threadId: string, assetId: string, noiseDb?: number, minDurationSec?: number + }) => { + const asset = threadManager.getThread(threadId)?.editor?.media.find((a) => a.id === assetId) + if (!asset) throw new Error('Asset not found') + if (asset.metadata && asset.metadata.hasAudio === false) return [] + const source = asset.proxyPath || asset.originalPath + let lastPercent = -1 + return await detectSilence(source, { noiseDb, minDurationSec }, undefined, (percent) => { + if (percent - lastPercent < 2 && percent !== 100) return + lastPercent = percent + if (!event.sender.isDestroyed()) { + event.sender.send('editor-silence-progress', { assetId, percent }) + } + }) + }) + + // AI prompt turns (M3): one structured call per turn, streamed via + // editor-turn-update; the base doc is never mutated until user accept. + ipcMain.handle('run-editor-prompt', (_event, options: { + threadId: string, personaId: string, prompt: string, baseStepId: string, + selectedItemIds: string[], playheadSec: number, widen?: 'chapter' | 'full' + }) => { + return editorPrompt.runEditorPrompt(options) + }) + + ipcMain.handle('abort-editor-prompt', (_event, { turnId }: { threadId: string, turnId: string }) => { + editorPrompt.abortEditorPrompt(turnId) + return true + }) + + // Renderer stamps the applied history step / revision back onto the turn. + ipcMain.handle('update-editor-turn', async (_event, { threadId, turnId, patch }: { + threadId: string, turnId: string, patch: { resultStepId?: string, revisionId?: string } + }) => { + return await threadManager.updateThreadWith(threadId, (thread) => { + if (!thread.editor) return null + const turns = thread.editor.turns.map((t) => + t.id === turnId + ? { + ...t, + resultStepId: patch.resultStepId ?? t.resultStepId, + revisionId: patch.revisionId ?? t.revisionId + } + : t + ) + return { editor: { ...thread.editor, turns } } + }) + }) + + // Export / render (M4): fast path via assembleVideo, region path via + // segment-then-concat; progress streams over editor-render-progress. + ipcMain.handle('export-editor-timeline', (_event, { threadId, quality }: { + threadId: string, quality: 'original' | 'preview' + }) => { + return editorRender.startEditorRender({ threadId, quality }) + }) + + ipcMain.handle('abort-editor-render', (_event, { renderId }: { renderId: string }) => { + editorRender.abortEditorRender(renderId) + return true + }) + + // Undo/redo history sidecar (renderer-authoritative; main persists) + ipcMain.handle('get-editor-history', (_event, threadId: string) => { + return editorHistory.loadHistory(threadId) + }) + + ipcMain.handle('push-editor-step', async (_event, { threadId, step, keyframe }: { + threadId: string, step: any, keyframe?: any + }) => { + return await editorHistory.pushStep(threadId, step, keyframe) + }) + + ipcMain.handle('set-editor-history-pointer', (_event, { threadId, currentStepId }: { + threadId: string, currentStepId: string + }) => { + return editorHistory.setPointer(threadId, currentStepId) + }) + + ipcMain.handle('clear-editor-history', (_event, { threadId }: { threadId: string }) => { + editorHistory.clearHistory(threadId) + return true + }) + + // Revision tree sidecar (renderer owns tree logic; main persists) + ipcMain.handle('get-editor-revisions', (_event, threadId: string) => { + return editorRevisions.loadRevisions(threadId) + }) + + ipcMain.handle('push-editor-revision', (_event, { threadId, revision }: { + threadId: string, revision: any + }) => { + return editorRevisions.pushRevision(threadId, revision) + }) + + ipcMain.handle('delete-editor-revisions', (_event, { threadId, ids }: { + threadId: string, ids: string[] + }) => { + return editorRevisions.deleteRevisions(threadId, ids) + }) + ipcMain.handle('get-all-threads', () => { return threadManager.getAllThreads() }) @@ -354,6 +640,17 @@ app.whenReady().then(() => { }) ipcMain.handle('delete-thread', (_event, id) => { + // Editor threads: abort any live per-asset preprocessing and renders, + // and remove the history sidecar before deletion + const thread = threadManager.getThread(id) + if (thread?.type === 'editor' && thread.editor) { + for (const asset of thread.editor.media) { + editorPreprocess.abortAssetPreprocessing(id, asset.id) + } + editorRender.abortRendersForThread(id) + editorHistory.deleteHistory(id) + editorRevisions.deleteRevisionsFile(id) + } return threadManager.deleteThread(id) }) diff --git a/src/main/scenedetect/index.ts b/src/main/scenedetect/index.ts index 107a15a..b5e18e0 100644 --- a/src/main/scenedetect/index.ts +++ b/src/main/scenedetect/index.ts @@ -173,11 +173,16 @@ export class SceneDetector { * @returns Array of detected scenes sorted by start time. * @throws If the CLI exits non-zero, the CSV is missing, or values cannot be parsed. */ - async detectScenes(videoPath: string, signal?: AbortSignal): Promise { + async detectScenes( + videoPath: string, + signal?: AbortSignal, + threshold: number = CONTENT_THRESHOLD, + onProgress?: (percent: number) => void + ): Promise { const tempDir = await fs.mkdtemp(join(tmpdir(), 'scenedetect-')) try { - await this.runScenedetect(videoPath, tempDir, signal) + await this.runScenedetect(videoPath, tempDir, signal, threshold, onProgress) const csvPath = await this.locateCsvFile(tempDir, videoPath) const csvContent = await fs.readFile(csvPath, 'utf-8') return this.parseCsv(csvContent) @@ -194,14 +199,20 @@ export class SceneDetector { /** * Execute the scenedetect CLI process. */ - private async runScenedetect(videoPath: string, outputDir: string, signal?: AbortSignal): Promise { + private async runScenedetect( + videoPath: string, + outputDir: string, + signal?: AbortSignal, + threshold: number = CONTENT_THRESHOLD, + onProgress?: (percent: number) => void + ): Promise { const pathOrRef = await resolveScenedetectPath() - + return new Promise((resolve, reject) => { const scenedetectArgs = [ '-i', videoPath, 'detect-content', - '-t', String(CONTENT_THRESHOLD), + '-t', String(threshold), 'list-scenes', '-o', outputDir, '-f', CSV_FILENAME, @@ -219,6 +230,17 @@ export class SceneDetector { args = scenedetectArgs } + const parseProgress = (chunk: Buffer | string) => { + // scenedetect's tqdm writes "Detected: N | Progress: 45%|…" to + // stderr even without a tty — surface the last percent per chunk. + if (!onProgress) return + const matches = String(chunk).match(/Progress:\s*(\d{1,3})%/g) + if (matches?.length) { + const last = matches[matches.length - 1].match(/(\d{1,3})/) + if (last) onProgress(Number(last[1])) + } + } + const child = execFile(cmd, args, (error, _stdout, stderr) => { if (error) { if (signal?.aborted) { @@ -235,6 +257,8 @@ export class SceneDetector { resolve() }) + child.stderr?.on('data', parseProgress) + if (signal) { signal.addEventListener('abort', () => { console.log('scenedetect process killed by signal') diff --git a/src/main/settings.ts b/src/main/settings.ts index 45621ba..f652e9a 100644 --- a/src/main/settings.ts +++ b/src/main/settings.ts @@ -2,13 +2,16 @@ import { app } from 'electron' import { join } from 'path' import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync } from 'fs' import os from 'os' -import { ModelSettings } from '../shared/types' +import { EditorPersona, ModelSettings } from '../shared/types' import { DEFAULT_MODEL_SETTINGS } from './constants/gemini' +import { BUILTIN_PERSONAS } from './constants/personas' interface Settings { tempDir: string geminiApiKey?: string modelSettings?: ModelSettings + /** User-defined editor personas ONLY — built-ins are merged at read time. */ + personas?: EditorPersona[] } class SettingsManager { @@ -52,7 +55,8 @@ class SettingsManager { return { tempDir: parsed.tempDir || this.defaultTempDir, geminiApiKey: parsed.geminiApiKey, - modelSettings: this.mergeModelSettings(parsed.modelSettings || DEFAULT_MODEL_SETTINGS) + modelSettings: this.mergeModelSettings(parsed.modelSettings || DEFAULT_MODEL_SETTINGS), + personas: Array.isArray(parsed.personas) ? parsed.personas : [] } } } catch (error) { @@ -151,6 +155,24 @@ class SettingsManager { this.saveSettings() return DEFAULT_MODEL_SETTINGS } + + /** User-defined personas only (built-ins live in constants/personas.ts). */ + getPersonas(): EditorPersona[] { + return this.settings.personas || [] + } + + /** + * Persists user personas. Built-in entries and any entry whose id collides + * with a built-in are filtered out — built-ins are code, not data. + */ + setPersonas(personas: EditorPersona[]): EditorPersona[] { + const builtinIds = new Set(BUILTIN_PERSONAS.map((p) => p.id)) + this.settings.personas = (personas || []).filter( + (p) => !p.builtin && !builtinIds.has(p.id) + ) + this.saveSettings() + return this.settings.personas + } } export const settingsManager = new SettingsManager() diff --git a/src/main/threads/index.ts b/src/main/threads/index.ts index 5889779..03f5129 100644 --- a/src/main/threads/index.ts +++ b/src/main/threads/index.ts @@ -3,7 +3,7 @@ import fs from 'fs' import path from 'path' import { v4 as uuidv4 } from 'uuid' import { MessageRole, FileType } from '@shared/types' -import type { Message, Thread, Usage, UsageRecord, VideoMetadata } from '@shared/types' +import type { EditorDocument, Message, Thread, Track, Usage, UsageRecord, VideoMetadata } from '@shared/types' import { settingsManager } from '../settings' import { getVideoMetadata } from '../ffmpeg' import { THREAD_DIRS } from '../constants/paths' @@ -137,6 +137,58 @@ class ThreadManager { return thread } + // Create a new timeline-editor project thread. + // Unlike createThread, no videoPath is required and NO preprocessing auto-starts — + // media is imported per-asset inside the editor (see src/main/editor/). + async createEditorThread(title: string): Promise { + const id = uuidv4() + const tempDir = settingsManager.getThreadTempDir(id) + + const seedTrack = (kind: Track['kind'], name: string, order: number): Track => ({ + id: uuidv4(), + kind, + name, + order, + muted: false, + locked: false, + hidden: false, + height: 64 + }) + + const editor: EditorDocument = { + schemaVersion: 1, + media: [], + tracks: [ + seedTrack('video', 'V1', 0), + seedTrack('audio', 'A1', 1), + seedTrack('overlay', 'OV', 2) + ], + timeline: [], + timelineMeta: { fps: 30, width: 1920, height: 1080, duration: 0 }, + activePersonaId: '', + turns: [], + historyRef: { currentStepId: '', stepCount: 0 }, + selection: {} + } + + const thread: Thread = { + id, + title, + type: 'editor', + preprocessing: {}, + tempDir, + messages: [], + versionCounter: 0, + usageHistory: [], + editor, + createdAt: Date.now(), + updatedAt: Date.now() + } + + this.saveThread(thread) + return thread + } + // Helper to normalize paths (handles symlinks like /var vs /private/var on macOS) private normalize(p: string | undefined): string | undefined { @@ -209,6 +261,30 @@ class ThreadManager { return msg }) + // Update editor document media paths (timeline editor threads) + if (thread.editor?.media) { + for (const asset of thread.editor.media) { + asset.originalPath = fixPath(asset.originalPath) || asset.originalPath + asset.proxyPath = fixPath(asset.proxyPath) + if (asset.preprocessing) { + for (const key of Object.keys(asset.preprocessing) as Array) { + const value = asset.preprocessing[key] + if (typeof value === 'string') { + (asset.preprocessing as any)[key] = fixPath(value) + } else if (Array.isArray(value)) { + (asset.preprocessing as any)[key] = value.map(fixPath) + } + } + } + for (const clip of asset.clips || []) { + clip.thumbnailPath = fixPath(clip.thumbnailPath) + } + for (const entry of asset.filmstrip || []) { + entry.thumbnailPath = fixPath(entry.thumbnailPath) || entry.thumbnailPath + } + } + } + // Save the repaired thread back to metadata and mirror it this.saveThread(thread) } @@ -320,14 +396,20 @@ class ThreadManager { } } - // Update a thread atomically - updateThread(id: string, updates: Partial): Promise { + // Update a thread atomically via a mutator that runs INSIDE the queued closure. + // The mutator receives the freshest thread state and returns the partial update + // (or null to skip). This makes concurrent read-modify-write patterns safe + // (e.g. multiple assets patching thread.editor.media in parallel). + updateThreadWith(id: string, mutator: (thread: Thread) => Partial | null): Promise { const existingQueue = this.updateQueues.get(id) || Promise.resolve() const nextUpdate = existingQueue.then(async () => { const thread = this.getThread(id) if (!thread) return null + const updates = mutator(thread) + if (!updates) return thread + const updatedThread = { ...thread, ...updates, @@ -359,6 +441,11 @@ class ThreadManager { return nextUpdate } + // Update a thread atomically with a precomputed partial. + updateThread(id: string, updates: Partial): Promise { + return this.updateThreadWith(id, () => updates) + } + private deleteFile(filePath: string) { if (!filePath) return const cleanPath = filePath.replace('file://', '') diff --git a/src/preload/index.ts b/src/preload/index.ts index f96f709..89865c8 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -48,6 +48,72 @@ const api = { getModelSettings: () => ipcRenderer.invoke('get-model-settings'), setModelSettings: (settings: any) => ipcRenderer.invoke('set-model-settings', settings), resetModelSettings: () => ipcRenderer.invoke('reset-model-settings'), + // Timeline Video Editor + createEditorProject: (title?: string) => ipcRenderer.invoke('create-editor-project', { title }), + renameEditorProject: (data: { threadId: string, title: string }) => + ipcRenderer.invoke('rename-editor-project', data), + saveEditorDoc: (data: { threadId: string, patch: any }) => + ipcRenderer.invoke('save-editor-doc', data), + addMediaAsset: (data: { threadId: string, filePath: string, name?: string }) => + ipcRenderer.invoke('add-media-asset', data), + importMediaUrl: (data: { threadId: string, url: string, resolution?: string }) => + ipcRenderer.invoke('import-media-url', data), + removeMediaAsset: (data: { threadId: string, assetId: string }) => + ipcRenderer.invoke('remove-media-asset', data), + preprocessMedia: (data: { threadId: string, assetId: string, steps?: string[], threshold?: number }) => + ipcRenderer.invoke('preprocess-media', data), + findSilence: (data: { threadId: string, assetId: string, noiseDb?: number, minDurationSec?: number }) => + ipcRenderer.invoke('find-silence', data), + clearAssetData: (data: { threadId: string, assetId: string, kind: 'transcript' | 'descriptions' }) => + ipcRenderer.invoke('clear-asset-data', data), + mergeClips: (data: { threadId: string, assetId: string, clipIds: string[] }) => + ipcRenderer.invoke('merge-clips', data), + splitClip: (data: { threadId: string, assetId: string, clipId: string, atSec?: number }) => + ipcRenderer.invoke('split-clip', data), + onEditorImportProgress: (callback: (data: { threadId: string, assetId: string, url?: string, percent: number }) => void) => { + const listener = (_event: any, data: any) => callback(data) + ipcRenderer.on('editor-import-progress', listener) + return () => ipcRenderer.removeListener('editor-import-progress', listener) + }, + onEditorSilenceProgress: (callback: (data: { assetId: string, percent: number }) => void) => { + const listener = (_event: any, data: any) => callback(data) + ipcRenderer.on('editor-silence-progress', listener) + return () => ipcRenderer.removeListener('editor-silence-progress', listener) + }, + getPersonas: () => ipcRenderer.invoke('get-personas'), + setPersonas: (personas: any[]) => ipcRenderer.invoke('set-personas', personas), + exportEditorTimeline: (data: { threadId: string, quality: 'original' | 'preview' }) => + ipcRenderer.invoke('export-editor-timeline', data), + abortEditorRender: (data: { renderId: string }) => + ipcRenderer.invoke('abort-editor-render', data), + onEditorRenderProgress: (callback: (data: any) => void) => { + const listener = (_event: any, data: any) => callback(data) + ipcRenderer.on('editor-render-progress', listener) + return () => ipcRenderer.removeListener('editor-render-progress', listener) + }, + runEditorPrompt: (data: { threadId: string, personaId: string, prompt: string, baseStepId: string, selectedItemIds: string[], playheadSec: number, widen?: string }) => + ipcRenderer.invoke('run-editor-prompt', data), + abortEditorPrompt: (data: { threadId: string, turnId: string }) => + ipcRenderer.invoke('abort-editor-prompt', data), + updateEditorTurn: (data: { threadId: string, turnId: string, patch: { resultStepId?: string, revisionId?: string } }) => + ipcRenderer.invoke('update-editor-turn', data), + onEditorTurnUpdate: (callback: (data: any) => void) => { + const listener = (_event: any, data: any) => callback(data) + ipcRenderer.on('editor-turn-update', listener) + return () => ipcRenderer.removeListener('editor-turn-update', listener) + }, + getEditorHistory: (threadId: string) => ipcRenderer.invoke('get-editor-history', threadId), + pushEditorStep: (data: { threadId: string, step: any, keyframe?: any }) => + ipcRenderer.invoke('push-editor-step', data), + setEditorHistoryPointer: (data: { threadId: string, currentStepId: string }) => + ipcRenderer.invoke('set-editor-history-pointer', data), + clearEditorHistory: (data: { threadId: string }) => + ipcRenderer.invoke('clear-editor-history', data), + getEditorRevisions: (threadId: string) => ipcRenderer.invoke('get-editor-revisions', threadId), + pushEditorRevision: (data: { threadId: string, revision: any }) => + ipcRenderer.invoke('push-editor-revision', data), + deleteEditorRevisions: (data: { threadId: string, ids: string[] }) => + ipcRenderer.invoke('delete-editor-revisions', data), // Thread Management createThread: (data: { videoPath?: string, videoName: string, imagePaths?: string[] }) => ipcRenderer.invoke('create-thread', data), diff --git a/src/renderer/src/assets/main.css b/src/renderer/src/assets/main.css index 798b0da..a66fc44 100644 --- a/src/renderer/src/assets/main.css +++ b/src/renderer/src/assets/main.css @@ -64,3 +64,38 @@ @apply h-full min-h-[50vh] !resize-none; } } + +/* Reduced motion (PRD §5.11): suppress decorative animation — fades, pulses, + spinners' easing flair, hover lifts, press-scales, and smooth scrolling. + Functional motion (video playback, the moving playhead, progress bars' + width changes) is untouched: none of it is driven by CSS animation. */ +@media (prefers-reduced-motion: reduce) { + html { + /* !important: a bundled third-party sheet re-declares this after us */ + scroll-behavior: auto !important; + } + + .animate-fade-in-up, + .animate-menu, + .animate-pulse-soft, + .animate-indeterminate { + animation: none !important; + } + + /* Keep spinners visible (they convey "busy") but slow their spin. */ + .animate-spin { + animation-duration: 2.5s !important; + } + + /* Neutralize transition-driven decoration: hover lifts and press scales. */ + *, + ::before, + ::after { + transition-duration: 0.01ms !important; + } + .active\:scale-95:active, + .hover\:scale-125:hover, + [class*="hover:-translate-y"]:hover { + transform: none !important; + } +} diff --git a/src/renderer/src/components/graph/GraphHeader.vue b/src/renderer/src/components/graph/GraphHeader.vue index db682cc..3bc2621 100644 --- a/src/renderer/src/components/graph/GraphHeader.vue +++ b/src/renderer/src/components/graph/GraphHeader.vue @@ -11,7 +11,19 @@ -

+ + +

{{ title || 'Graph Task Manager' }}

@@ -48,13 +60,40 @@ diff --git a/src/renderer/src/editor/VideoEditorPage.vue b/src/renderer/src/editor/VideoEditorPage.vue new file mode 100644 index 0000000..6d57f7a --- /dev/null +++ b/src/renderer/src/editor/VideoEditorPage.vue @@ -0,0 +1,207 @@ + + + diff --git a/src/renderer/src/editor/components/AiResultCard.vue b/src/renderer/src/editor/components/AiResultCard.vue new file mode 100644 index 0000000..876b31f --- /dev/null +++ b/src/renderer/src/editor/components/AiResultCard.vue @@ -0,0 +1,98 @@ + + + diff --git a/src/renderer/src/editor/components/AssetRow.vue b/src/renderer/src/editor/components/AssetRow.vue new file mode 100644 index 0000000..96c033a --- /dev/null +++ b/src/renderer/src/editor/components/AssetRow.vue @@ -0,0 +1,166 @@ + + + diff --git a/src/renderer/src/editor/components/ChatPanel.vue b/src/renderer/src/editor/components/ChatPanel.vue new file mode 100644 index 0000000..6de4a5b --- /dev/null +++ b/src/renderer/src/editor/components/ChatPanel.vue @@ -0,0 +1,156 @@ + + + diff --git a/src/renderer/src/editor/components/ClipTile.vue b/src/renderer/src/editor/components/ClipTile.vue new file mode 100644 index 0000000..e2b643f --- /dev/null +++ b/src/renderer/src/editor/components/ClipTile.vue @@ -0,0 +1,85 @@ + + + diff --git a/src/renderer/src/editor/components/ClipTray.vue b/src/renderer/src/editor/components/ClipTray.vue new file mode 100644 index 0000000..b47e449 --- /dev/null +++ b/src/renderer/src/editor/components/ClipTray.vue @@ -0,0 +1,113 @@ + + + diff --git a/src/renderer/src/editor/components/ExportDialog.vue b/src/renderer/src/editor/components/ExportDialog.vue new file mode 100644 index 0000000..bfbc1f6 --- /dev/null +++ b/src/renderer/src/editor/components/ExportDialog.vue @@ -0,0 +1,160 @@ + + + + + diff --git a/src/renderer/src/editor/components/ImportMediaModal.vue b/src/renderer/src/editor/components/ImportMediaModal.vue new file mode 100644 index 0000000..51d3fa1 --- /dev/null +++ b/src/renderer/src/editor/components/ImportMediaModal.vue @@ -0,0 +1,136 @@ + + + diff --git a/src/renderer/src/editor/components/InspectorPanel.vue b/src/renderer/src/editor/components/InspectorPanel.vue new file mode 100644 index 0000000..b019b89 --- /dev/null +++ b/src/renderer/src/editor/components/InspectorPanel.vue @@ -0,0 +1,299 @@ + + + diff --git a/src/renderer/src/editor/components/MediaPanel.vue b/src/renderer/src/editor/components/MediaPanel.vue new file mode 100644 index 0000000..c34b7b5 --- /dev/null +++ b/src/renderer/src/editor/components/MediaPanel.vue @@ -0,0 +1,109 @@ + + + diff --git a/src/renderer/src/editor/components/PersonaEditorModal.vue b/src/renderer/src/editor/components/PersonaEditorModal.vue new file mode 100644 index 0000000..8f63ffc --- /dev/null +++ b/src/renderer/src/editor/components/PersonaEditorModal.vue @@ -0,0 +1,190 @@ + + + diff --git a/src/renderer/src/editor/components/RevisionGraphModal.vue b/src/renderer/src/editor/components/RevisionGraphModal.vue new file mode 100644 index 0000000..88b516d --- /dev/null +++ b/src/renderer/src/editor/components/RevisionGraphModal.vue @@ -0,0 +1,100 @@ + + + + + diff --git a/src/renderer/src/editor/components/RevisionGraphNode.vue b/src/renderer/src/editor/components/RevisionGraphNode.vue new file mode 100644 index 0000000..0c34984 --- /dev/null +++ b/src/renderer/src/editor/components/RevisionGraphNode.vue @@ -0,0 +1,61 @@ + + + diff --git a/src/renderer/src/editor/components/RevisionListItem.vue b/src/renderer/src/editor/components/RevisionListItem.vue new file mode 100644 index 0000000..ec04251 --- /dev/null +++ b/src/renderer/src/editor/components/RevisionListItem.vue @@ -0,0 +1,75 @@ + + + diff --git a/src/renderer/src/editor/components/RevisionsPanel.vue b/src/renderer/src/editor/components/RevisionsPanel.vue new file mode 100644 index 0000000..0d20482 --- /dev/null +++ b/src/renderer/src/editor/components/RevisionsPanel.vue @@ -0,0 +1,64 @@ + + + diff --git a/src/renderer/src/editor/components/SaveRevisionButton.vue b/src/renderer/src/editor/components/SaveRevisionButton.vue new file mode 100644 index 0000000..730c20b --- /dev/null +++ b/src/renderer/src/editor/components/SaveRevisionButton.vue @@ -0,0 +1,73 @@ + + + diff --git a/src/renderer/src/editor/components/SilenceFinder.vue b/src/renderer/src/editor/components/SilenceFinder.vue new file mode 100644 index 0000000..26629a5 --- /dev/null +++ b/src/renderer/src/editor/components/SilenceFinder.vue @@ -0,0 +1,94 @@ + + + diff --git a/src/renderer/src/editor/components/TimelinePanel.vue b/src/renderer/src/editor/components/TimelinePanel.vue new file mode 100644 index 0000000..5ccdf27 --- /dev/null +++ b/src/renderer/src/editor/components/TimelinePanel.vue @@ -0,0 +1,97 @@ + + + diff --git a/src/renderer/src/editor/components/timeline/TimelineClip.vue b/src/renderer/src/editor/components/timeline/TimelineClip.vue new file mode 100644 index 0000000..39ead36 --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TimelineClip.vue @@ -0,0 +1,155 @@ + + + + + diff --git a/src/renderer/src/editor/components/timeline/TimelineMinimap.vue b/src/renderer/src/editor/components/timeline/TimelineMinimap.vue new file mode 100644 index 0000000..7c744db --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TimelineMinimap.vue @@ -0,0 +1,90 @@ + + + diff --git a/src/renderer/src/editor/components/timeline/TimelinePlayhead.vue b/src/renderer/src/editor/components/timeline/TimelinePlayhead.vue new file mode 100644 index 0000000..ea7fe33 --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TimelinePlayhead.vue @@ -0,0 +1,29 @@ + + + diff --git a/src/renderer/src/editor/components/timeline/TimelineRuler.vue b/src/renderer/src/editor/components/timeline/TimelineRuler.vue new file mode 100644 index 0000000..fb59142 --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TimelineRuler.vue @@ -0,0 +1,68 @@ + + + diff --git a/src/renderer/src/editor/components/timeline/TimelineToolbar.vue b/src/renderer/src/editor/components/timeline/TimelineToolbar.vue new file mode 100644 index 0000000..4c25d66 --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TimelineToolbar.vue @@ -0,0 +1,124 @@ + + + diff --git a/src/renderer/src/editor/components/timeline/TrackHeader.vue b/src/renderer/src/editor/components/timeline/TrackHeader.vue new file mode 100644 index 0000000..56348cc --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TrackHeader.vue @@ -0,0 +1,37 @@ + + + diff --git a/src/renderer/src/editor/components/timeline/TrackLane.vue b/src/renderer/src/editor/components/timeline/TrackLane.vue new file mode 100644 index 0000000..00f9b3c --- /dev/null +++ b/src/renderer/src/editor/components/timeline/TrackLane.vue @@ -0,0 +1,104 @@ + + + diff --git a/src/renderer/src/editor/composables/useEditorShortcuts.ts b/src/renderer/src/editor/composables/useEditorShortcuts.ts new file mode 100644 index 0000000..bd386b2 --- /dev/null +++ b/src/renderer/src/editor/composables/useEditorShortcuts.ts @@ -0,0 +1,98 @@ +import { onMounted, onUnmounted } from 'vue' +import { useEditorStore } from '../../stores/editorStore' + +/** + * Global keyboard shortcuts for the editor page (PRD §5.11). + * Never fires while the user is typing in an input/textarea/select/ + * contenteditable — that guard is the first line. + * + * Space play/pause M add marker at playhead + * ← / → frame-step playhead S or Cmd/Ctrl+B split at playhead + * Shift+←/→ nudge selection 1f Delete/Backsp ripple-delete selection + * Alt+Shift+←/→ nudge 1s Shift+Delete delete leaving gap + * Home / End sequence start/end Cmd/Ctrl+Z undo + * + / - zoom Cmd/Ctrl+Shift+Z redo + * Esc clear selection + */ +export function useEditorShortcuts(zoomBy?: (factor: number) => void) { + const store = useEditorStore() + + const frameStep = () => 1 / (store.doc?.timelineMeta.fps || 30) + + const onKeyDown = (e: KeyboardEvent) => { + const target = e.target as HTMLElement | null + if (target?.closest('input, textarea, select, [contenteditable="true"]')) return + if (!store.doc) return + + const meta = e.metaKey || e.ctrlKey + + // Undo / redo + if (meta && e.key.toLowerCase() === 'z') { + e.preventDefault() + if (e.shiftKey) store.redo() + else store.undo() + return + } + + // Split + if ((meta && e.key.toLowerCase() === 'b') || (!meta && (e.key === 's' || e.key === 'S'))) { + e.preventDefault() + store.splitAtPlayhead() + return + } + + switch (e.key) { + case ' ': + e.preventDefault() + store.isPlaying = !store.isPlaying + return + case 'ArrowLeft': + case 'ArrowRight': { + e.preventDefault() + const direction = e.key === 'ArrowRight' ? 1 : -1 + if (e.shiftKey && store.selectedItemIds.length) { + store.nudgeItems(store.selectedItemIds, direction * (e.altKey ? 1 : frameStep())) + } else { + store.seekTo(store.playheadSec + direction * (e.altKey ? 1 : frameStep())) + } + return + } + case 'Home': + e.preventDefault() + store.seekTo(0) + return + case 'End': + e.preventDefault() + store.seekTo(store.contentEnd) + return + case 'Delete': + case 'Backspace': + if (store.selectedItemIds.length) { + e.preventDefault() + store.deleteItems(store.selectedItemIds, { ripple: !e.shiftKey && !e.altKey }) + } + return + case 'm': + case 'M': + e.preventDefault() + store.addMarker(store.playheadSec) + return + case '+': + case '=': + e.preventDefault() + zoomBy?.(1.25) + return + case '-': + case '_': + e.preventDefault() + zoomBy?.(0.8) + return + case 'Escape': + store.clearItemSelection() + return + } + } + + onMounted(() => window.addEventListener('keydown', onKeyDown)) + onUnmounted(() => window.removeEventListener('keydown', onKeyDown)) +} diff --git a/src/renderer/src/editor/composables/useEdlPlayback.ts b/src/renderer/src/editor/composables/useEdlPlayback.ts new file mode 100644 index 0000000..037214f --- /dev/null +++ b/src/renderer/src/editor/composables/useEdlPlayback.ts @@ -0,0 +1,322 @@ +import { computed, onUnmounted, ref, watch, type Ref } from 'vue' +import { useEditorStore } from '../../stores/editorStore' + +interface EdlSegment { + itemId: string + tStart: number + tEnd: number + src: string + sourceIn: number + speed: number + muted: boolean + gain?: number + trackId?: string +} + +/** + * EDL playback engine (PRD §5.3): plays the COMPOSED timeline through an + * A/B pair of