From 27676ac9ffbf2b39620ce570a785869d82342bd1 Mon Sep 17 00:00:00 2001
From: Yu-ga <74749461+yuga-hashimoto@users.noreply.github.com>
Date: Thu, 25 Jun 2026 20:02:08 +0900
Subject: [PATCH 1/4] feat: add end-to-end Video Studio automation
---
.gitignore | 1 +
docs/video-studio/oss-research.md | 82 +++
package.json | 1 +
packages/dashboard/src/index.ts | 46 +-
packages/gateway/src/tools/index.ts | 2 +
packages/gateway/src/tools/video-studio.ts | 629 +++++++++++++++++++++
packages/mcp/src/http-server.ts | 26 +
packages/shared/src/tool-profiles.ts | 20 +
scripts/video-studio-e2e.ts | 83 +++
tests/video-studio.test.ts | 149 +++++
10 files changed, 1038 insertions(+), 1 deletion(-)
create mode 100644 docs/video-studio/oss-research.md
create mode 100644 packages/gateway/src/tools/video-studio.ts
create mode 100644 scripts/video-studio-e2e.ts
create mode 100644 tests/video-studio.test.ts
diff --git a/.gitignore b/.gitignore
index a278c33..b8519d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,5 @@ examples/skills/*/dist/
coverage/
*.log
.tmp-tests/
+.tmp-video-studio-e2e/
.commandcode/
diff --git a/docs/video-studio/oss-research.md b/docs/video-studio/oss-research.md
new file mode 100644
index 0000000..b2df8c4
--- /dev/null
+++ b/docs/video-studio/oss-research.md
@@ -0,0 +1,82 @@
+# OSS Research
+
+## Compared projects
+
+- name: Remotion
+- url: https://github.com/remotion-dev/remotion and https://www.remotion.dev/docs/
+- stars if available: about 51.2k on GitHub when checked on 2026-06-25
+- last activity if available: latest release v4.0.482 on 2026-06-22 when checked
+- license: custom Remotion license, with company-license requirements in some cases
+- stack: TypeScript, React, browser rendering, programmatic composition
+- useful ideas: composition objects, props-driven rendering, preview/render separation, reusable templates
+- risks: too heavy for the first LocalAnt implementation and license constraints make direct dependency undesirable
+
+- name: MoviePy
+- url: https://github.com/Zulko/moviepy and https://zulko.github.io/moviepy/
+- stars if available: GitHub project is a widely used Python video editing library
+- last activity if available: active public repository when checked
+- license: MIT
+- stack: Python, FFmpeg-backed clip composition
+- useful ideas: timeline-style clip composition, audio/video/text overlay separation, cross-platform fallback
+- risks: adds Python runtime dependency and package installation surface to LocalAnt
+
+- name: WhisperX
+- url: https://github.com/m-bain/whisperX
+- stars if available: public GitHub project
+- last activity if available: active public repository when checked
+- license: BSD-style license in repository
+- stack: Python, Whisper, alignment models
+- useful ideas: word-level timestamp structure, future speech-to-caption alignment, diarization-ready JSON
+- risks: large ML/runtime dependencies; not suitable as required default
+
+- name: FFmpeg
+- url: https://ffmpeg.org/
+- stars if available: not applicable
+- last activity if available: active upstream project
+- license: LGPL/GPL depending on build options
+- stack: native CLI
+- useful ideas: final render, audio muxing, ASS subtitle burn-in, thumbnail extraction, ffprobe validation
+- risks: local install required; codec/filter availability varies by build
+
+- name: Aegisub / ASS subtitle design
+- url: https://github.com/TypesettingTools/Aegisub
+- stars if available: public GitHub project
+- last activity if available: active public repository when checked
+- license: BSD-style license in repository
+- stack: C++, ASS subtitle editor
+- useful ideas: style/timing separation, safe area, outline/shadow readability, karaoke-style future extension
+- risks: editor code is not needed; LocalAnt should emit ASS directly
+
+- name: OpenShorts
+- url: https://github.com/mutonby/openshorts
+- stars if available: GitHub badge present; exact count can drift
+- last activity if available: active enough to have current docs/site when checked
+- license: MIT
+- stack: self-hosted Docker app, short-video workflows
+- useful ideas: clip generation, AI shorts, YouTube Studio style workflow grouping
+- risks: full platform scope is larger than LocalAnt's first pass
+
+- name: gyoridavid/short-video-maker
+- url: https://github.com/gyoridavid/short-video-maker
+- stars if available: public GitHub project
+- last activity if available: active public repository when checked
+- license: repository license should be checked before copying any code
+- stack: MCP/REST, Kokoro TTS, Whisper, Pexels, Remotion
+- useful ideas: text to TTS to captions to Remotion render pipeline, MCP-compatible public surface
+- risks: Pexels/API-backed media search and ML dependencies are not acceptable as mandatory defaults here
+
+## Design decisions for LocalAnt
+
+- what to adopt: Remotion's scene/composition/props model, MoviePy's separation of clips/audio/overlays, WhisperX's word-timing JSON shape, FFmpeg as the required first renderer, ASS subtitles for readable Shorts/Reels captions, and a browser-upload-assist provider that stops before submit.
+- what not to adopt: paid external video generation APIs, cloud text-to-video APIs, mandatory Remotion dependency, mandatory Python/ML stack, API-only publishers as the default path, or any copied OSS source.
+- why: the product requirement is a free local fallback that can generate a real uploadable MP4 from ChatGPT/MCP. Optional paid or reviewed APIs can exist later, but they cannot be required for video generation.
+
+## Final architecture
+
+- renderer: `builtin-ffmpeg` first. It creates scene images, burns ASS captions, muxes local narration when available, extracts a thumbnail, and validates with ffprobe.
+- script generator: deterministic template generator that works without an LLM API.
+- asset generator: free local generated visuals. It writes per-scene PNGs using FFmpeg color/image generation when available, with SVG placeholders retained for inspection.
+- audio generator: free local narration. On macOS it uses `say`; otherwise FFmpeg silence is used when possible, with setup guidance only if neither local option works.
+- caption system: SRT, ASS, and `words.json` are generated from scene timings, with the JSON shaped for future WhisperX word alignment.
+- publisher system: browser upload assist and dry-run metadata first. Official APIs are readiness-checked but not required.
+- dashboard integration: dashboard routes call the same MCP tools so ChatGPT and the local UI share one implementation.
diff --git a/package.json b/package.json
index 74e3455..e35596d 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
"test": "vitest run",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage",
+ "video-studio:e2e": "tsx scripts/video-studio-e2e.ts",
"validate": "pnpm build && pnpm test",
"start": "node packages/cli/dist/bin.js start",
"setup": "node packages/cli/dist/bin.js setup",
diff --git a/packages/dashboard/src/index.ts b/packages/dashboard/src/index.ts
index 684932b..ec4c102 100644
--- a/packages/dashboard/src/index.ts
+++ b/packages/dashboard/src/index.ts
@@ -96,7 +96,7 @@ export function dashboardHtml(token = ""): string {