From 5370c8c0b8f6a817074b65f30eb16deee9c12adb Mon Sep 17 00:00:00 2001 From: Yu-ga <74749461+yuga-hashimoto@users.noreply.github.com> Date: Mon, 29 Jun 2026 01:51:05 +0900 Subject: [PATCH] feat: add FlowKit LocalAnt skill --- docs/skills.md | 1 + examples/skills/flowkit/README.md | 21 ++ examples/skills/flowkit/package.json | 6 + examples/skills/flowkit/skill.json | 124 +++++++++ examples/skills/flowkit/src/index.ts | 285 ++++++++++++++++++++ examples/skills/flowkit/tests/index.test.ts | 128 +++++++++ pnpm-lock.yaml | 2 + 7 files changed, 567 insertions(+) create mode 100644 examples/skills/flowkit/README.md create mode 100644 examples/skills/flowkit/package.json create mode 100644 examples/skills/flowkit/skill.json create mode 100644 examples/skills/flowkit/src/index.ts create mode 100644 examples/skills/flowkit/tests/index.test.ts diff --git a/docs/skills.md b/docs/skills.md index 45f8b71..e536c3e 100644 --- a/docs/skills.md +++ b/docs/skills.md @@ -106,6 +106,7 @@ Ready-to-read references live in [`examples/skills/`](../examples/skills): | `file-organizer` | Filesystem writes — sort a folder by type/date. A "local hands" chore ChatGPT can't do itself. | | `local-backup` | Shell allowlist (`tar` only) — timestamped `.tar.gz` snapshots. | | `article-publisher` | Network + secrets + git — publish to Zenn/Qiita/note. | +| `flowkit` | Local REST API — control a running FlowKit video-generation server. | ## Generating a skill from ChatGPT diff --git a/examples/skills/flowkit/README.md b/examples/skills/flowkit/README.md new file mode 100644 index 0000000..fabaadc --- /dev/null +++ b/examples/skills/flowkit/README.md @@ -0,0 +1,21 @@ +# FlowKit + +LocalAnt skill for controlling a running FlowKit server. + +Prerequisites: + +1. Start FlowKit locally, usually with `python -m agent.main`. +2. Load the FlowKit Chrome extension and open Google Flow. +3. Confirm `flowkit_health` returns `extension_connected: true`. + +The skill defaults to `http://127.0.0.1:8100`. Pass `baseUrl` to any tool when FlowKit is running elsewhere. + +Typical flow: + +1. `flowkit_create_workflow` +2. `flowkit_generate_references` +3. `flowkit_generate_images` +4. `flowkit_generate_videos` +5. `flowkit_poll` + +Generation tools submit batch requests; FlowKit handles throttling internally. diff --git a/examples/skills/flowkit/package.json b/examples/skills/flowkit/package.json new file mode 100644 index 0000000..8cc0ba2 --- /dev/null +++ b/examples/skills/flowkit/package.json @@ -0,0 +1,6 @@ +{ + "name": "@localant-skill/flowkit", + "version": "0.1.0", + "private": true, + "type": "module" +} diff --git a/examples/skills/flowkit/skill.json b/examples/skills/flowkit/skill.json new file mode 100644 index 0000000..b9bc59d --- /dev/null +++ b/examples/skills/flowkit/skill.json @@ -0,0 +1,124 @@ +{ + "name": "flowkit", + "displayName": "FlowKit", + "version": "0.1.0", + "description": "Control a local FlowKit video-generation server through LocalAnt tools.", + "author": "LocalAnt", + "license": "MIT", + "entry": "src/index.ts", + "riskLevel": 3, + "permissions": { + "filesystem": { "mode": "none", "allowedDirectories": [] }, + "shell": { "mode": "none", "allowedCommands": [] }, + "network": { "mode": "allowlist", "allowedHosts": ["127.0.0.1", "localhost"] }, + "secrets": [], + "browser": "none", + "adb": "none", + "git": "none", + "agent": "none" + }, + "tools": [ + { + "name": "flowkit_health", + "description": "Check FlowKit API health and Chrome extension connection status.", + "riskLevel": 0, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" } + } + } + }, + { + "name": "flowkit_status", + "description": "Summarize FlowKit health, projects, and request status.", + "riskLevel": 0, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" }, + "projectId": { "type": "string" }, + "limit": { "type": "number" } + } + } + }, + { + "name": "flowkit_create_workflow", + "description": "Create a FlowKit project, video, and scenes from a structured workflow spec.", + "riskLevel": 3, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" }, + "project": { "type": "object" }, + "video": { "type": "object" }, + "scenes": { "type": "array", "items": { "type": "object" } } + }, + "required": ["project", "video", "scenes"] + } + }, + { + "name": "flowkit_generate_references", + "description": "Submit batch requests to generate reference images for project entities.", + "riskLevel": 3, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" }, + "projectId": { "type": "string" }, + "characterIds": { "type": "array", "items": { "type": "string" } } + }, + "required": ["projectId"] + } + }, + { + "name": "flowkit_generate_images", + "description": "Submit batch requests to generate FlowKit scene images.", + "riskLevel": 3, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" }, + "projectId": { "type": "string" }, + "videoId": { "type": "string" }, + "sceneIds": { "type": "array", "items": { "type": "string" } }, + "orientation": { "type": "string", "enum": ["VERTICAL", "HORIZONTAL"] } + }, + "required": ["projectId", "videoId"] + } + }, + { + "name": "flowkit_generate_videos", + "description": "Submit batch requests to generate FlowKit scene video clips.", + "riskLevel": 3, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" }, + "projectId": { "type": "string" }, + "videoId": { "type": "string" }, + "sceneIds": { "type": "array", "items": { "type": "string" } }, + "orientation": { "type": "string", "enum": ["VERTICAL", "HORIZONTAL"] } + }, + "required": ["projectId", "videoId"] + } + }, + { + "name": "flowkit_poll", + "description": "Poll FlowKit batch status for project/video request completion.", + "riskLevel": 0, + "inputSchema": { + "type": "object", + "properties": { + "baseUrl": { "type": "string" }, + "projectId": { "type": "string" }, + "videoId": { "type": "string" }, + "type": { "type": "string" }, + "orientation": { "type": "string" }, + "timeoutSeconds": { "type": "number" }, + "intervalSeconds": { "type": "number" } + } + } + } + ] +} diff --git a/examples/skills/flowkit/src/index.ts b/examples/skills/flowkit/src/index.ts new file mode 100644 index 0000000..1b5742d --- /dev/null +++ b/examples/skills/flowkit/src/index.ts @@ -0,0 +1,285 @@ +const DEFAULT_BASE_URL = "http://127.0.0.1:8100"; + +type RequestLike = { type?: string; status?: string }; +type BatchRequestType = "GENERATE_CHARACTER_IMAGE" | "GENERATE_IMAGE" | "GENERATE_VIDEO"; +type SkillContext = { log: (message: string, extra?: unknown) => void }; +type Parser = { parse: (input: unknown) => T }; +type BaseInput = { baseUrl?: string }; +type Orientation = "VERTICAL" | "HORIZONTAL"; + +function ensureRecord(input: unknown): Record { + if (input === undefined || input === null) return {}; + if (typeof input !== "object" || Array.isArray(input)) throw new Error("Expected an object input."); + return input as Record; +} + +function optionalString(value: unknown, name: string): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== "string") throw new Error(`${name} must be a string.`); + return value; +} + +function requiredString(value: unknown, name: string): string { + const text = optionalString(value, name); + if (!text) throw new Error(`${name} is required.`); + return text; +} + +function optionalPositiveInt(value: unknown, name: string, fallback: number): number { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer.`); + return value; +} + +function optionalStringArray(value: unknown, name: string): string[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) throw new Error(`${name} must be an array of strings.`); + return value; +} + +function optionalOrientation(value: unknown): Orientation { + if (value === undefined) return "VERTICAL"; + if (value !== "VERTICAL" && value !== "HORIZONTAL") throw new Error("orientation must be VERTICAL or HORIZONTAL."); + return value; +} + +function base(input: Record): BaseInput { + return { baseUrl: optionalString(input.baseUrl, "baseUrl") }; +} + +function schema(parse: (input: Record) => T): Parser { + return { parse: (input: unknown) => parse(ensureRecord(input)) }; +} + +const emptySchema = schema((input) => base(input)); +const statusSchema = schema((input) => ({ + ...base(input), + projectId: optionalString(input.projectId, "projectId"), + limit: optionalPositiveInt(input.limit, "limit", 10), +})); +const workflowSchema = schema((input) => { + const project = input.project; + const video = input.video; + const scenes = input.scenes; + if (typeof project !== "object" || project === null || Array.isArray(project)) throw new Error("project is required."); + if (typeof video !== "object" || video === null || Array.isArray(video)) throw new Error("video is required."); + if (!Array.isArray(scenes) || scenes.some((scene) => typeof scene !== "object" || scene === null || Array.isArray(scene))) { + throw new Error("scenes must be an array of objects."); + } + return { ...base(input), project: project as Record, video: video as Record, scenes: scenes as Record[] }; +}); +const referenceSchema = schema((input) => ({ + ...base(input), + projectId: requiredString(input.projectId, "projectId"), + characterIds: optionalStringArray(input.characterIds, "characterIds"), +})); +const sceneGenerationSchema = schema((input) => ({ + ...base(input), + projectId: requiredString(input.projectId, "projectId"), + videoId: requiredString(input.videoId, "videoId"), + sceneIds: optionalStringArray(input.sceneIds, "sceneIds"), + orientation: optionalOrientation(input.orientation), +})); +const pollSchema = schema((input) => ({ + ...base(input), + projectId: optionalString(input.projectId, "projectId"), + videoId: optionalString(input.videoId, "videoId"), + type: optionalString(input.type, "type"), + orientation: optionalString(input.orientation, "orientation"), + timeoutSeconds: optionalPositiveInt(input.timeoutSeconds, "timeoutSeconds", 900), + intervalSeconds: optionalPositiveInt(input.intervalSeconds, "intervalSeconds", 10), +})); + +function normalizeBaseUrl(baseUrl?: string): string { + return (baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ""); +} + +async function api(baseUrl: string | undefined, method: string, path: string, body?: unknown): Promise { + const url = `${normalizeBaseUrl(baseUrl)}${path}`; + const response = await fetch(url, { + method, + headers: body === undefined ? { Accept: "application/json" } : { Accept: "application/json", "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const text = await response.text(); + const data = text ? JSON.parse(text) : null; + if (!response.ok) { + throw new Error(`${method} ${url} failed: HTTP ${response.status}: ${text}`); + } + return data as T; +} + +function query(params: Record): string { + const search = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) search.set(key, String(value)); + } + const text = search.toString(); + return text ? `?${text}` : ""; +} + +export function summarizeRequests(rows: RequestLike[]): { + total: number; + byStatus: Record; + byType: Record>; +} { + const byStatus: Record = {}; + const byType: Record> = {}; + for (const row of rows) { + const status = row.status ?? "UNKNOWN"; + const type = row.type ?? "UNKNOWN"; + byStatus[status] = (byStatus[status] ?? 0) + 1; + byType[type] ??= {}; + byType[type]![status] = (byType[type]![status] ?? 0) + 1; + } + return { total: rows.length, byStatus, byType }; +} + +export function buildBatchPayload( + type: BatchRequestType, + input: { + projectId: string; + videoId?: string; + characterIds?: string[]; + sceneIds?: string[]; + orientation?: "VERTICAL" | "HORIZONTAL"; + }, +): { requests: Record[] } { + const requests: Record[] = []; + for (const characterId of input.characterIds ?? []) { + requests.push({ type, project_id: input.projectId, character_id: characterId }); + } + for (const sceneId of input.sceneIds ?? []) { + requests.push({ + type, + project_id: input.projectId, + video_id: input.videoId, + scene_id: sceneId, + orientation: input.orientation ?? "VERTICAL", + }); + } + return { requests }; +} + +async function listProjectCharacterIds(baseUrl: string | undefined, projectId: string): Promise { + const rows = await api<{ id: string }[]>(baseUrl, "GET", `/api/projects/${projectId}/characters`); + return rows.map((row) => row.id); +} + +async function listVideoSceneIds(baseUrl: string | undefined, videoId: string): Promise { + const rows = await api<{ id: string; display_order?: number }[]>(baseUrl, "GET", `/api/scenes${query({ video_id: videoId })}`); + return rows.sort((a, b) => (a.display_order ?? 0) - (b.display_order ?? 0)).map((row) => row.id); +} + +async function submitBatch(baseUrl: string | undefined, payload: { requests: Record[] }) { + if (payload.requests.length === 0) { + throw new Error("No FlowKit requests to submit."); + } + return api(baseUrl, "POST", "/api/requests/batch", payload); +} + +const skill = { + name: "flowkit", + displayName: "FlowKit", + description: "Control a local FlowKit video-generation server through LocalAnt tools.", + version: "0.1.0", + tools: { + flowkit_health: { + description: "Check FlowKit API health and Chrome extension connection status.", + riskLevel: 0, + inputSchema: emptySchema, + handler: async ({ baseUrl }: BaseInput) => api(baseUrl, "GET", "/health"), + }, + flowkit_status: { + description: "Summarize FlowKit health, projects, and request status.", + riskLevel: 0, + inputSchema: statusSchema, + handler: async ({ baseUrl, projectId, limit }) => { + const [health, projects, requests] = await Promise.all([ + api(baseUrl, "GET", "/health"), + api(baseUrl, "GET", "/api/projects"), + api(baseUrl, "GET", `/api/requests${query({ project_id: projectId })}`), + ]); + const result: Record = { + health, + projects: projects.slice(0, limit), + requestSummary: summarizeRequests(requests), + }; + if (projectId) { + result.project = await api(baseUrl, "GET", `/api/projects/${projectId}`); + result.characters = await api(baseUrl, "GET", `/api/projects/${projectId}/characters`); + result.videos = await api(baseUrl, "GET", `/api/videos${query({ project_id: projectId })}`); + } + return result; + }, + }, + flowkit_create_workflow: { + description: "Create a FlowKit project, video, and scenes from a structured workflow spec.", + riskLevel: 3, + inputSchema: workflowSchema, + handler: async ({ baseUrl, project, video, scenes }) => { + const createdProject = await api<{ id: string }>(baseUrl, "POST", "/api/projects", project); + const createdVideo = await api<{ id: string }>(baseUrl, "POST", "/api/videos", { ...video, project_id: createdProject.id }); + const createdScenes: unknown[] = []; + let previousSceneId: string | undefined; + for (let index = 0; index < scenes.length; index += 1) { + const scene = scenes[index]!; + const body: Record = { video_id: createdVideo.id, display_order: index, ...scene }; + if (body.parent_scene_id === "__previous__") body.parent_scene_id = previousSceneId; + const createdScene = await api<{ id: string }>(baseUrl, "POST", "/api/scenes", body); + previousSceneId = createdScene.id; + createdScenes.push(createdScene); + } + return { project: createdProject, video: createdVideo, scenes: createdScenes }; + }, + }, + flowkit_generate_references: { + description: "Submit batch requests to generate reference images for project entities.", + riskLevel: 3, + inputSchema: referenceSchema, + handler: async ({ baseUrl, projectId, characterIds }) => { + const ids = characterIds ?? (await listProjectCharacterIds(baseUrl, projectId)); + return submitBatch(baseUrl, buildBatchPayload("GENERATE_CHARACTER_IMAGE", { projectId, characterIds: ids })); + }, + }, + flowkit_generate_images: { + description: "Submit batch requests to generate FlowKit scene images.", + riskLevel: 3, + inputSchema: sceneGenerationSchema, + handler: async ({ baseUrl, projectId, videoId, sceneIds, orientation }) => { + const ids = sceneIds ?? (await listVideoSceneIds(baseUrl, videoId)); + return submitBatch(baseUrl, buildBatchPayload("GENERATE_IMAGE", { projectId, videoId, sceneIds: ids, orientation })); + }, + }, + flowkit_generate_videos: { + description: "Submit batch requests to generate FlowKit scene video clips.", + riskLevel: 3, + inputSchema: sceneGenerationSchema, + handler: async ({ baseUrl, projectId, videoId, sceneIds, orientation }) => { + const ids = sceneIds ?? (await listVideoSceneIds(baseUrl, videoId)); + return submitBatch(baseUrl, buildBatchPayload("GENERATE_VIDEO", { projectId, videoId, sceneIds: ids, orientation })); + }, + }, + flowkit_poll: { + description: "Poll FlowKit batch status for project/video request completion.", + riskLevel: 0, + inputSchema: pollSchema, + handler: async ({ baseUrl, projectId, videoId, type, orientation, timeoutSeconds, intervalSeconds }) => { + const deadline = Date.now() + timeoutSeconds * 1000; + let last: Record = {}; + do { + last = await api>( + baseUrl, + "GET", + `/api/requests/batch-status${query({ project_id: projectId, video_id: videoId, type, orientation })}`, + ); + if (last.done) return last; + await new Promise((resolve) => setTimeout(resolve, intervalSeconds * 1000)); + } while (Date.now() < deadline); + return { ...last, timedOut: true }; + }, + }, + }, +}; + +export default skill; diff --git a/examples/skills/flowkit/tests/index.test.ts b/examples/skills/flowkit/tests/index.test.ts new file mode 100644 index 0000000..f87ade5 --- /dev/null +++ b/examples/skills/flowkit/tests/index.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it, vi, afterEach } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createGateway } from "@localant/gateway"; +import skill, { buildBatchPayload, summarizeRequests } from "../src/index"; + +const ctx = { getSecret: async () => undefined, workspaceDir: ".", log: () => {} }; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("flowkit skill", () => { + it("exposes LocalAnt FlowKit tools", () => { + expect(skill.name).toBe("flowkit"); + expect(skill.tools.flowkit_health).toBeDefined(); + expect(skill.tools.flowkit_status).toBeDefined(); + expect(skill.tools.flowkit_create_workflow).toBeDefined(); + expect(skill.tools.flowkit_generate_references).toBeDefined(); + expect(skill.tools.flowkit_generate_images).toBeDefined(); + expect(skill.tools.flowkit_generate_videos).toBeDefined(); + expect(skill.tools.flowkit_poll).toBeDefined(); + }); + + it("is discoverable and valid through the LocalAnt gateway", async () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), "flowkit-gateway-")); + try { + const gateway = createGateway(base); + gateway.saveConfig({ ...gateway.config(), tools: { profile: "full" } }); + const list = await gateway.executeTool("skill_list", {}, { caller: "test" }); + expect(list.ok).toBe(true); + const skills = list.data as { name: string; valid: boolean; tools: string[] }[]; + const flowkit = skills.find((item) => item.name === "flowkit"); + expect(flowkit).toBeTruthy(); + expect(flowkit!.valid).toBe(true); + expect(flowkit!.tools).toContain("flowkit_health"); + + const validation = await gateway.executeTool("skill_validate", { name: "flowkit" }, { caller: "test" }); + expect(validation.ok).toBe(true); + expect(validation.data).toEqual({ valid: true, errors: [] }); + } finally { + fs.rmSync(base, { recursive: true, force: true }); + } + }); + + it("builds batch payloads for scene generation", () => { + expect( + buildBatchPayload("GENERATE_IMAGE", { + projectId: "pid", + videoId: "vid", + sceneIds: ["s1", "s2"], + orientation: "VERTICAL", + }), + ).toEqual({ + requests: [ + { type: "GENERATE_IMAGE", project_id: "pid", video_id: "vid", scene_id: "s1", orientation: "VERTICAL" }, + { type: "GENERATE_IMAGE", project_id: "pid", video_id: "vid", scene_id: "s2", orientation: "VERTICAL" }, + ], + }); + }); + + it("summarizes requests by status and type", () => { + expect( + summarizeRequests([ + { type: "GENERATE_IMAGE", status: "COMPLETED" }, + { type: "GENERATE_IMAGE", status: "FAILED" }, + { type: "GENERATE_VIDEO", status: "PROCESSING" }, + ]), + ).toEqual({ + total: 3, + byStatus: { COMPLETED: 1, FAILED: 1, PROCESSING: 1 }, + byType: { + GENERATE_IMAGE: { COMPLETED: 1, FAILED: 1 }, + GENERATE_VIDEO: { PROCESSING: 1 }, + }, + }); + }); + + it("calls FlowKit health endpoint", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (url: string) => { + expect(url).toBe("http://127.0.0.1:8100/health"); + return new Response(JSON.stringify({ status: "ok", extension_connected: true }), { status: 200 }); + }), + ); + + const result = await skill.tools.flowkit_health.handler({}, ctx); + expect(result).toEqual({ status: "ok", extension_connected: true }); + }); + + it("creates a workflow project, video, and scenes", async () => { + const calls: { url: string; body: unknown }[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init?: RequestInit) => { + calls.push({ url, body: init?.body ? JSON.parse(String(init.body)) : undefined }); + if (url.endsWith("/api/projects")) { + return new Response(JSON.stringify({ id: "pid", name: "Demo" }), { status: 200 }); + } + if (url.endsWith("/api/videos")) { + return new Response(JSON.stringify({ id: "vid", project_id: "pid" }), { status: 200 }); + } + if (url.endsWith("/api/scenes")) { + return new Response(JSON.stringify({ id: `scene-${calls.length}` }), { status: 200 }); + } + throw new Error(`unexpected URL ${url}`); + }), + ); + + const result = await skill.tools.flowkit_create_workflow.handler( + { + project: { name: "Demo", story: "story" }, + video: { title: "Episode 1" }, + scenes: [ + { prompt: "root", character_names: [], chain_type: "ROOT" }, + { prompt: "next", character_names: [], chain_type: "CONTINUATION", parent_scene_id: "__previous__" }, + ], + }, + ctx, + ); + + expect(result).toMatchObject({ project: { id: "pid" }, video: { id: "vid" } }); + expect(calls[1]!.body).toEqual({ title: "Episode 1", project_id: "pid" }); + expect(calls[3]!.body).toMatchObject({ parent_scene_id: "scene-3" }); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9e6f9d3..27fb138 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,6 +100,8 @@ importers: specifier: workspace:* version: link:../../../packages/skill-sdk + examples/skills/flowkit: {} + examples/skills/hello-world: dependencies: '@localant/skill-sdk':