From 27676ac9ffbf2b39620ce570a785869d82342bd1 Mon Sep 17 00:00:00 2001 From: Yu-ga <74749461+yuga-hashimoto@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:02:08 +0900 Subject: [PATCH 1/4] feat: add end-to-end Video Studio automation --- .gitignore | 1 + docs/video-studio/oss-research.md | 82 +++ package.json | 1 + packages/dashboard/src/index.ts | 46 +- packages/gateway/src/tools/index.ts | 2 + packages/gateway/src/tools/video-studio.ts | 629 +++++++++++++++++++++ packages/mcp/src/http-server.ts | 26 + packages/shared/src/tool-profiles.ts | 20 + scripts/video-studio-e2e.ts | 83 +++ tests/video-studio.test.ts | 149 +++++ 10 files changed, 1038 insertions(+), 1 deletion(-) create mode 100644 docs/video-studio/oss-research.md create mode 100644 packages/gateway/src/tools/video-studio.ts create mode 100644 scripts/video-studio-e2e.ts create mode 100644 tests/video-studio.test.ts diff --git a/.gitignore b/.gitignore index a278c33..b8519d8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ examples/skills/*/dist/ coverage/ *.log .tmp-tests/ +.tmp-video-studio-e2e/ .commandcode/ diff --git a/docs/video-studio/oss-research.md b/docs/video-studio/oss-research.md new file mode 100644 index 0000000..b2df8c4 --- /dev/null +++ b/docs/video-studio/oss-research.md @@ -0,0 +1,82 @@ +# OSS Research + +## Compared projects + +- name: Remotion +- url: https://github.com/remotion-dev/remotion and https://www.remotion.dev/docs/ +- stars if available: about 51.2k on GitHub when checked on 2026-06-25 +- last activity if available: latest release v4.0.482 on 2026-06-22 when checked +- license: custom Remotion license, with company-license requirements in some cases +- stack: TypeScript, React, browser rendering, programmatic composition +- useful ideas: composition objects, props-driven rendering, preview/render separation, reusable templates +- risks: too heavy for the first LocalAnt implementation and license constraints make direct dependency undesirable + +- name: MoviePy +- url: https://github.com/Zulko/moviepy and https://zulko.github.io/moviepy/ +- stars if available: GitHub project is a widely used Python video editing library +- last activity if available: active public repository when checked +- license: MIT +- stack: Python, FFmpeg-backed clip composition +- useful ideas: timeline-style clip composition, audio/video/text overlay separation, cross-platform fallback +- risks: adds Python runtime dependency and package installation surface to LocalAnt + +- name: WhisperX +- url: https://github.com/m-bain/whisperX +- stars if available: public GitHub project +- last activity if available: active public repository when checked +- license: BSD-style license in repository +- stack: Python, Whisper, alignment models +- useful ideas: word-level timestamp structure, future speech-to-caption alignment, diarization-ready JSON +- risks: large ML/runtime dependencies; not suitable as required default + +- name: FFmpeg +- url: https://ffmpeg.org/ +- stars if available: not applicable +- last activity if available: active upstream project +- license: LGPL/GPL depending on build options +- stack: native CLI +- useful ideas: final render, audio muxing, ASS subtitle burn-in, thumbnail extraction, ffprobe validation +- risks: local install required; codec/filter availability varies by build + +- name: Aegisub / ASS subtitle design +- url: https://github.com/TypesettingTools/Aegisub +- stars if available: public GitHub project +- last activity if available: active public repository when checked +- license: BSD-style license in repository +- stack: C++, ASS subtitle editor +- useful ideas: style/timing separation, safe area, outline/shadow readability, karaoke-style future extension +- risks: editor code is not needed; LocalAnt should emit ASS directly + +- name: OpenShorts +- url: https://github.com/mutonby/openshorts +- stars if available: GitHub badge present; exact count can drift +- last activity if available: active enough to have current docs/site when checked +- license: MIT +- stack: self-hosted Docker app, short-video workflows +- useful ideas: clip generation, AI shorts, YouTube Studio style workflow grouping +- risks: full platform scope is larger than LocalAnt's first pass + +- name: gyoridavid/short-video-maker +- url: https://github.com/gyoridavid/short-video-maker +- stars if available: public GitHub project +- last activity if available: active public repository when checked +- license: repository license should be checked before copying any code +- stack: MCP/REST, Kokoro TTS, Whisper, Pexels, Remotion +- useful ideas: text to TTS to captions to Remotion render pipeline, MCP-compatible public surface +- risks: Pexels/API-backed media search and ML dependencies are not acceptable as mandatory defaults here + +## Design decisions for LocalAnt + +- what to adopt: Remotion's scene/composition/props model, MoviePy's separation of clips/audio/overlays, WhisperX's word-timing JSON shape, FFmpeg as the required first renderer, ASS subtitles for readable Shorts/Reels captions, and a browser-upload-assist provider that stops before submit. +- what not to adopt: paid external video generation APIs, cloud text-to-video APIs, mandatory Remotion dependency, mandatory Python/ML stack, API-only publishers as the default path, or any copied OSS source. +- why: the product requirement is a free local fallback that can generate a real uploadable MP4 from ChatGPT/MCP. Optional paid or reviewed APIs can exist later, but they cannot be required for video generation. + +## Final architecture + +- renderer: `builtin-ffmpeg` first. It creates scene images, burns ASS captions, muxes local narration when available, extracts a thumbnail, and validates with ffprobe. +- script generator: deterministic template generator that works without an LLM API. +- asset generator: free local generated visuals. It writes per-scene PNGs using FFmpeg color/image generation when available, with SVG placeholders retained for inspection. +- audio generator: free local narration. On macOS it uses `say`; otherwise FFmpeg silence is used when possible, with setup guidance only if neither local option works. +- caption system: SRT, ASS, and `words.json` are generated from scene timings, with the JSON shaped for future WhisperX word alignment. +- publisher system: browser upload assist and dry-run metadata first. Official APIs are readiness-checked but not required. +- dashboard integration: dashboard routes call the same MCP tools so ChatGPT and the local UI share one implementation. diff --git a/package.json b/package.json index 74e3455..e35596d 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "video-studio:e2e": "tsx scripts/video-studio-e2e.ts", "validate": "pnpm build && pnpm test", "start": "node packages/cli/dist/bin.js start", "setup": "node packages/cli/dist/bin.js setup", diff --git a/packages/dashboard/src/index.ts b/packages/dashboard/src/index.ts index 684932b..ec4c102 100644 --- a/packages/dashboard/src/index.ts +++ b/packages/dashboard/src/index.ts @@ -96,7 +96,7 @@ export function dashboardHtml(token = ""): string {