Skip to content

api-reference.md

Latest commit

 

History

History
874 lines (675 loc) · 21.4 KB

api-reference.md

File metadata and controls

874 lines (675 loc) · 21.4 KB
Raw

API Reference -- Lucy

Table of Contents

WebSocket Protocol

Bidirectional JSON messages between the Host (Electron app) and the Agent (Python, runs in VM).

Connection: ws://127.0.0.1:<port> (port is per-agent, starting at 8765) Max payload: 50 MB Ping/Pong: 30s interval, 10s timeout

Message Format

Every message is a JSON object with two fields:

{
  "type": "<message_type>",
  "payload": { ... }
}

Host -> Agent Messages

ping

Health check. Agent responds with pong.

{ "type": "ping", "payload": {} }

execute_task

Start a new task. Rejected if a task is already running.

{
  "type": "execute_task",
  "payload": {
    "description": "Open LibreOffice Writer and write a letter"
  }
}

cancel_task

Cancel the currently running task.

{ "type": "cancel_task", "payload": {} }

replay_task

Replay a saved recipe. Accepts a full path or bare task ID.

{
  "type": "replay_task",
  "payload": {
    "path": "a1b2c3d4e5f6",
    "mode": "intelligent"
  }
}
Field Values
path Task ID or full path to YAML file
mode "intelligent" or "deterministic"

install_software

Install software in the VM. Internally executed as an agent task.

{
  "type": "install_software",
  "payload": { "name": "vlc" }
}

list_tasks

Request all saved recipes. Agent responds with task_list.

{ "type": "list_tasks", "payload": {} }

delete_task

Delete a saved recipe by ID.

{
  "type": "delete_task",
  "payload": { "id": "a1b2c3d4e5f6" }
}

update_task

Update a recipe's description.

{
  "type": "update_task",
  "payload": {
    "id": "a1b2c3d4e5f6",
    "description": "Updated description"
  }
}

update_task_instructions

Set per-recipe instructions.

{
  "type": "update_task_instructions",
  "payload": {
    "id": "a1b2c3d4e5f6",
    "instructions": "Use filename report.pdf"
  }
}

export_task

Export a recipe as raw YAML. Agent responds with task_export.

{
  "type": "export_task",
  "payload": { "id": "a1b2c3d4e5f6" }
}

import_task

Import a recipe from YAML content. Agent responds with task_imported.

{
  "type": "import_task",
  "payload": { "yaml": "task:\n  id: ..." }
}

screenshot

Request a single screenshot. Agent responds with screenshot_result.

{ "type": "screenshot", "payload": {} }

add_lesson

Add a learned lesson.

{
  "type": "add_lesson",
  "payload": {
    "task_id": "a1b2c3d4e5f6",
    "feedback": "Always use apt instead of snap"
  }
}

list_lessons

Request all lessons. Agent responds with lessons_list.

{ "type": "list_lessons", "payload": {} }

delete_lesson

Delete a lesson by ID.

{
  "type": "delete_lesson",
  "payload": { "id": "abc12345" }
}

Agent -> Host Messages

pong

Response to ping.

{ "type": "pong", "payload": {} }

event

Real-time event during task execution. Streamed continuously.

{
  "type": "event",
  "payload": {
    "event_type": "action",
    "task_id": "a1b2c3d4e5f6",
    "step": 5,
    "data": {
      "tool": "computer",
      "input": { "action": "left_click", "coordinate": [640, 400] },
      "result": "Clicked at (640, 400)"
    },
    "timestamp": 1700000005.0
  }
}

Event types:

event_type data fields Description
screenshot base64 (string) Screenshot taken (base64 PNG)
action tool, input, result Tool action executed
thinking text (string) Claude's reasoning text
status message, optionally description Task started / cancelled
error message, optionally tool, input Error during execution
complete message, total_steps Task finished successfully

task_list

Response to list_tasks.

{
  "type": "task_list",
  "payload": {
    "tasks": [
      {
        "id": "a1b2c3d4e5f6",
        "description": "Open LibreOffice...",
        "instructions": "",
        "status": "completed",
        "duration_seconds": 32.5,
        "created_at": 1700000000.0
      }
    ]
  }
}

task_deleted

Confirmation of delete_task.

{
  "type": "task_deleted",
  "payload": { "id": "a1b2c3d4e5f6" }
}

task_updated

Confirmation of update_task.

{
  "type": "task_updated",
  "payload": { "id": "a1b2c3d4e5f6", "description": "Updated" }
}

task_instructions_updated

Confirmation of update_task_instructions.

{
  "type": "task_instructions_updated",
  "payload": { "id": "a1b2c3d4e5f6" }
}

task_export

Response to export_task.

{
  "type": "task_export",
  "payload": {
    "id": "a1b2c3d4e5f6",
    "yaml": "task:\n  id: a1b2c3d4e5f6\n  ..."
  }
}

task_imported

Response to import_task. Returns the imported task's metadata.

{
  "type": "task_imported",
  "payload": {
    "id": "a1b2c3d4e5f6",
    "description": "...",
    "instructions": "",
    "status": "completed",
    "duration_seconds": 32.5,
    "created_at": 1700000000.0
  }
}

screenshot_result

Response to screenshot.

{
  "type": "screenshot_result",
  "payload": {
    "base64": "<base64-encoded PNG>",
    "width": 1280,
    "height": 800
  }
}

lesson_added

Confirmation of add_lesson.

{
  "type": "lesson_added",
  "payload": {
    "id": "abc12345",
    "task_id": "a1b2c3d4e5f6",
    "feedback": "Always use apt instead of snap"
  }
}

lessons_list

Response to list_lessons.

{
  "type": "lessons_list",
  "payload": {
    "lessons": [
      {
        "id": "abc12345",
        "task_id": "a1b2c3d4e5f6",
        "feedback": "Always use apt instead of snap",
        "created_at": 1700000000.0
      }
    ]
  }
}

lesson_deleted

Confirmation of delete_lesson.

{
  "type": "lesson_deleted",
  "payload": { "id": "abc12345" }
}

error

Error response for any failed request.

{
  "type": "error",
  "payload": {
    "message": "Eine Aufgabe lauft bereits"
  }
}

Constraints

  • One task at a time. A second execute_task while one is running returns an error.
  • install_software is internally converted to a task, so the same one-at-a-time constraint applies.
  • Screenshots in events are embedded as base64 inside event payloads. These can be several MB.
  • Max 100 steps per task (configurable via SA_MAX_AGENT_STEPS).

Electron IPC Channels

Invoke channels (renderer -> main, request/response)

Channel Arguments Return Description
agents:create opts: { name, apiKey, model, apiBaseUrl? } AgentInstance Create a new agent
agents:remove agentId: string void Remove agent and container
agents:list -- AgentInstance[] List all agents
agents:connect agentId: string void Connect WebSocket
agents:disconnect agentId: string void Disconnect WebSocket
agents:update agentId, updates: { name?, apiKey?, model?, apiBaseUrl? } AgentInstance | null Update agent settings
agents:get agentId: string AgentInstance | null Get single agent
agents:state agentId: string "running" | "stopped" | "not_found" Docker container state
task:execute agentId, description: string void Start a task
task:cancel agentId: string void Cancel running task
task:list agentId: string Array<Record> List saved recipes
task:replay agentId, path, mode: string void Replay a recipe
task:delete agentId, id: string void Delete a recipe
task:update agentId, id, description: string void Update recipe description
task:updateInstructions agentId, id, instructions: string void Update recipe instructions
task:export agentId, id: string { id, yaml } Export recipe YAML
task:import agentId, yaml: string void Import recipe YAML
software:install agentId, name: string void Install software
screenshot agentId: string { base64, width, height } Request screenshot
lessons:add agentId, taskId, feedback: string void Add a lesson
lessons:list agentId: string Array<Record> List lessons
lessons:delete agentId, id: string void Delete a lesson
config:load -- AppConfig Load full config
config:update partial: Partial<AppConfig> AppConfig Update config
schedule:list agentId?: string Schedule[] List schedules
schedule:add { recipeId, agentId, cronExpr, mode } Schedule Add schedule
schedule:update id, updates Schedule | null Update schedule
schedule:remove id: string void Remove schedule

Send channels (main -> renderer, push notifications)

Channel Arguments Description
agent:event agentId: string, event: AgentEvent Real-time agent event
agent:connected agentId: string Agent WebSocket connected
agent:disconnected agentId: string, reason: string Agent WebSocket disconnected
schedule:executed scheduleId: string, agentId: string Scheduled recipe executed
schedule:missed scheduleId: string, agentId: string Scheduled run skipped (agent disconnected)

Preload API (window.lucy)

The preload script exposes a typed API on window.lucy via Electron's contextBridge. This is the only way renderer code communicates with the main process.

window.lucy.agents

agents: {
  create(opts: { name: string; apiKey: string; model: string; apiBaseUrl?: string }): Promise<AgentInstance>;
  remove(agentId: string): Promise<void>;
  list(): Promise<AgentInstance[]>;
  connect(agentId: string): Promise<void>;
  disconnect(agentId: string): Promise<void>;
  state(agentId: string): Promise<"running" | "stopped" | "not_found">;
  update(agentId: string, updates: { name?: string; apiKey?: string; model?: string; apiBaseUrl?: string }): Promise<AgentInstance | null>;
  get(agentId: string): Promise<AgentInstance | null>;
}

window.lucy.task

task: {
  execute(agentId: string, description: string): Promise<void>;
  cancel(agentId: string): Promise<void>;
  list(agentId: string): Promise<TaskMeta[]>;
  replay(agentId: string, path: string, mode: "intelligent" | "deterministic"): Promise<void>;
  delete(agentId: string, id: string): Promise<void>;
  update(agentId: string, id: string, description: string): Promise<void>;
  updateInstructions(agentId: string, id: string, instructions: string): Promise<void>;
  export(agentId: string, id: string): Promise<{ id: string; yaml: string }>;
  import(agentId: string, yaml: string): Promise<void>;
}

window.lucy.software

software: {
  install(agentId: string, name: string): Promise<void>;
}

window.lucy.lessons

lessons: {
  add(agentId: string, taskId: string, feedback: string): Promise<void>;
  list(agentId: string): Promise<Lesson[]>;
  delete(agentId: string, id: string): Promise<void>;
}

window.lucy.config

config: {
  load(): Promise<AppConfig>;
  update(partial: Partial<AppConfig>): Promise<AppConfig>;
}

window.lucy.schedule

schedule: {
  list(agentId?: string): Promise<Schedule[]>;
  add(data: { recipeId: string; agentId: string; cronExpr: string; mode: string }): Promise<Schedule>;
  update(id: string, updates: Partial<Schedule>): Promise<Schedule | null>;
  remove(id: string): Promise<void>;
}

window.lucy.screenshot

screenshot(agentId: string): Promise<{ base64: string; width: number; height: number }>;

window.lucy.on (event listeners)

Each returns an unsubscribe function.

on: {
  agentEvent(cb: (agentId: string, event: AgentEvent) => void): () => void;
  agentConnected(cb: (agentId: string) => void): () => void;
  agentDisconnected(cb: (agentId: string, reason: string) => void): () => void;
}

Agent Python API

AgentSettings (agent/config.py)

Pydantic settings loaded from environment variables with SA_ prefix.

class AgentSettings(BaseSettings):
    anthropic_api_key: str          # Required
    claude_model: str               # Default: "claude-sonnet-4-20250514"
    max_tokens: int                 # Default: 4096
    max_agent_steps: int            # Default: 100
    display: str                    # Default: ":1"
    screen_width: int               # Default: 1280
    screen_height: int              # Default: 800
    screenshot_max_dim: int         # Default: 1280
    ws_host: str                    # Default: "0.0.0.0"
    ws_port: int                    # Default: 8765
    record_all_tasks: bool          # Default: True
    tasks_dir: Path                 # Default: Path("/home/agent/tasks")
    lessons_file: Path              # Default: Path("/home/agent/tasks/lessons.yaml")
    log_level: str                  # Default: "INFO"

Agent (agent/core.py)

The main orchestrator class.

class Agent:
    def __init__(self, settings: AgentSettings, lessons: LessonsStore | None = None) -> None: ...

    @property
    def busy(self) -> bool: ...

    def cancel(self) -> None:
        """Request cancellation of the currently running task."""

    def force_reset(self) -> None:
        """Force-reset the busy flag."""

    async def execute_task(self, description: str) -> AsyncIterator[AgentEvent]:
        """Run a task. Yields AgentEvent objects for real-time progress."""

AgentEvent (agent/core.py)

@dataclass
class AgentEvent:
    type: EventType        # screenshot, action, thinking, status, error, complete
    task_id: str
    step: int
    data: dict[str, Any]
    timestamp: float

ClaudeClient (agent/claude.py)

Manages conversation history and Claude API calls.

class ClaudeClient:
    def __init__(self, settings: AgentSettings, lessons_prompt: str = "") -> None: ...

    def reset(self) -> None:
        """Clear conversation history for a new task."""

    async def send(self, screenshot_b64: str | None = None) -> ParsedResponse:
        """Send conversation to Claude and return parsed response."""

    def add_user_message(self, text: str) -> None:
        """Add a plain-text user message."""

    def add_tool_results(self, results: list[BetaToolResultBlockParam]) -> None:
        """Add tool-result blocks as a user turn."""

    def update_lessons(self, lessons_prompt: str) -> None:
        """Update the lessons section in the system prompt."""

    @staticmethod
    def make_tool_result(
        tool_use_id: str,
        output: str | None = None,
        error: str | None = None,
        screenshot_b64: str | None = None,
    ) -> BetaToolResultBlockParam:
        """Build a tool_result block."""

ParsedResponse (agent/claude.py)

@dataclass
class ParsedResponse:
    text_blocks: list[ParsedTextBlock]
    tool_calls: list[ParsedToolCall]
    stop_reason: str | None
    task_complete: bool           # True if "AUFGABE ABGESCHLOSSEN" found in text

ComputerController (agent/computer.py)

Controls the VM desktop via X11 tools. All methods are async.

class ComputerController:
    def __init__(self, settings: AgentSettings) -> None: ...

    async def screenshot(self) -> tuple[str, ScalingInfo]:
        """Take screenshot, downscale, return (base64_png, scaling)."""

    async def mouse_click(self, x: int, y: int, button: int = 1) -> str:
    async def mouse_double_click(self, x: int, y: int, button: int = 1) -> str:
    async def mouse_drag(self, start_x: int, start_y: int, end_x: int, end_y: int, button: int = 1) -> str:
    async def scroll(self, x: int, y: int, direction: str, clicks: int = 3) -> str:

    async def type_text(self, text: str) -> str:
        """Type text. Falls back to xclip + Ctrl+V for non-ASCII."""

    async def key_press(self, key: str) -> str:
    async def hotkey(self, *keys: str) -> str:

    async def execute(self, command: str, timeout: float = 120.0) -> str:
        """Execute a shell command. Commands ending with '&' run in background."""

    async def wait_for_window(self, name: str, timeout: float = 30.0) -> str:
    async def get_clipboard(self) -> str:
    async def focus_window(self, name: str) -> str:

ScalingInfo (agent/computer.py)

@dataclass(frozen=True)
class ScalingInfo:
    real_width: int
    real_height: int
    scaled_width: int
    scaled_height: int

    def to_real(self, x: int, y: int) -> tuple[int, int]:
        """Convert Claude-visible coords to real screen coords."""

TaskRecorder (agent/recorder.py)

Records task executions as YAML playbooks.

class TaskRecorder:
    def __init__(self, settings: AgentSettings) -> None: ...

    def start(self, task_id: str, description: str) -> None:
    def add_step(self, tool: str, action: str, params: dict, result: str) -> None:
    def finish(self, status: str = "completed") -> Path | None:

    @staticmethod
    def load(path: Path) -> dict[str, Any]:

    def list_tasks(self) -> list[dict[str, Any]]:
    def delete_task(self, task_id: str) -> bool:
    def get_task_yaml(self, task_id: str) -> str | None:
    def import_task_yaml(self, yaml_content: str) -> dict[str, Any] | None:
    def update_task_instructions(self, task_id: str, instructions: str) -> bool:
    def update_task_description(self, task_id: str, description: str) -> bool:

LessonsStore (agent/lessons.py)

Manages persistent lessons for the agent.

class LessonsStore:
    def __init__(self, settings: AgentSettings) -> None: ...

    def add(self, task_id: str, feedback: str) -> Lesson:
    def remove(self, lesson_id: str) -> bool:
    def list_all(self) -> list[dict[str, Any]]:
    def build_prompt_section(self) -> str:
        """Build prompt section from all lessons. Returns empty string if none."""

MessageRouter (server/main.py)

Routes incoming WebSocket messages to handlers.

class MessageRouter:
    def __init__(self, settings: AgentSettings) -> None: ...

    async def handle(self, ws: ServerConnection, raw: str | bytes) -> None:
    async def cleanup(self) -> None:
        """Cancel running task on disconnect."""

Tool definitions (agent/tools.py)

def make_computer_tool(width: int, height: int, display_number: int = 1) -> BetaToolComputerUse20250124Param:
def get_tools(width: int, height: int) -> list[BetaToolUnionParam]:

BASH_TOOL: BetaToolBash20250124Param
TEXT_EDITOR_TOOL: BetaToolTextEditor20250124Param

Configuration Schema

AppConfig (TypeScript, host)

interface AppConfig {
  vm: VmConfig;
  agent: AgentConfig;
  anthropic: AnthropicConfig;
  window: WindowState;
  agents: AgentInstance[];
  nextPortOffset: number;
  theme: "dark" | "light";
  language: "de" | "en";
  schedules: Schedule[];
}

AgentInstance

interface AgentInstance {
  id: string;              // 8-char UUID prefix
  name: string;
  wsPort: number;
  vncPort: number;
  sshPort: number;
  containerName: string;   // "lucy-agent-<id>"
  createdAt: number;       // timestamp ms
  apiKey: string;
  model: string;
  apiBaseUrl: string;      // empty = Anthropic default
}

Schedule

interface Schedule {
  id: string;              // 8-char UUID prefix
  recipeId: string;        // task ID
  agentId: string;
  mode: "intelligent" | "deterministic";
  cronExpr: string;        // "daily:HH:MM" | "weekly:DOW:HH:MM" | "interval:MINUTES"
  enabled: boolean;
  lastRunAt: number | null;
  nextRunAt: number;       // timestamp ms
  createdAt: number;
}

VmConfig

interface VmConfig {
  ramMb: number;           // Default: 4096
  cpus: number;            // Default: 2
  diskSizeGb: number;      // Default: 20
  vncPassword: string;     // Default: "sandbox"
}

AgentConfig

interface AgentConfig {
  maxReconnectAttempts: number;  // Default: 20
  reconnectIntervalMs: number;   // Default: 3000
  maxAgentSteps: number;         // Default: 100
}

AnthropicConfig

interface AnthropicConfig {
  apiKey: string;          // Default: ""
  model: string;           // Default: "claude-sonnet-4-20250514"
  maxTokens: number;       // Default: 4096
}

Recipe YAML Schema

task:
  id: string               # 12-char hex UUID prefix
  description: string       # Natural language task description
  instructions: string      # Per-recipe replay instructions (can be empty)
  created_at: float         # Unix timestamp
  status: string            # "completed" | "cancelled" | "running"
  duration_seconds: float
steps:
  - step: int
    tool: string            # "computer" | "bash" | "str_replace_editor"
    action: string          # Tool-specific action name
    params: object          # Tool-specific parameters
    result: string          # Execution result text
    timestamp: float

Lessons YAML Schema

# /home/agent/tasks/lessons.yaml
- id: string                # 8-char hex UUID prefix
  task_id: string           # Associated task ID (or empty for manual lessons)
  feedback: string          # The lesson text
  created_at: float         # Unix timestamp