Feat/aloha screenshot mode

.env.example

@@ -0,0 +1,15 @@
+# Third-party Anthropic-compatible vendor (e.g. gpugeek). Leave unset to
+# fall back to the official Anthropic API + the CLAUDE_API_KEY in
+# Aloha_Act/config/api_keys.json.
+ANTHROPIC_BASE_URL="https://api.gpugeek.com"
+ANTHROPIC_AUTH_TOKEN="vendor-bearer-token-here"
+
+# Default Claude Computer Use model. Overrides claude_model from config.yaml
+# when set. The agent auto-selects the right beta header based on this name.
+ANTHROPIC_MODEL="Vendor2/Claude-4.6-Sonnet"
+
+# Other API keys can also be exported here instead of api_keys.json.
+# OPENAI_API_KEY=""
+# OPERATOR_OPENAI_API_KEY=""
+# CLAUDE_API_KEY=""
+# GOOGLE_API_KEY=""

.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+logs/
+Aloha_Act/trace_data
+inputs
+.env
+.cache
+.DS_Store

Aloha_Act/app_client.py

@@ -7,6 +7,10 @@
 import os
 import logging
 
+from dotenv import find_dotenv, load_dotenv
+
+load_dotenv(find_dotenv(usecwd=True), override=False)
+
 from flask import Flask, request, jsonify
 
 from ui_aloha.execute.executor.aloha_executor import AlohaExecutor
@@ -21,6 +25,8 @@ def __init__(self, args):
         self.trace_id = args.trace_id
         self.server_url = args.server_url
         self.max_steps = getattr(args, 'max_steps', 50)
+        # Set per /run_task request (JSON `mode` or `actor_model`); not a CLI flag.
+        self.mode: str | None = None
 
         self.is_processing = False
         self.should_stop = False
@@ -50,19 +56,30 @@ def process_input():
             trace_id=shared_state.trace_id,
             server_url=shared_state.server_url,
             max_steps=shared_state.max_steps,
+            mode=shared_state.mode,
         )
 
         for loop_msg in sampling_loop:
             if shared_state.should_stop or shared_state.stop_event.is_set():
                 break
 
-            # Log minimal progress for visibility
+            # Progress logs: full text; avoid dumping megabytes of base64 screens.
             try:
                 msg_type = loop_msg.get("type")
-                content_preview = str(loop_msg.get("content"))[:100]
-                logging.info(f"[loop_msg] type={msg_type} content={content_preview}")
+                raw = loop_msg.get("content")
+                if msg_type == "image_base64" and isinstance(raw, str):
+                    head = 120
+                    snippet = raw[:head] + ("..." if len(raw) > head else "")
+                    logging.info(
+                        "[loop_msg] type=%s content_len=%s content_prefix=%s",
+                        msg_type,
+                        len(raw),
+                        snippet,
+                    )
+                else:
+                    logging.info("[loop_msg] type=%s content=%s", msg_type, raw)
             except Exception:
-                logging.info(f"[loop_msg] {str(loop_msg)[:100]}")
+                logging.info("[loop_msg] %s", loop_msg)
 
             # light pacing to avoid busy loop in UI
             time.sleep(0.1)
@@ -98,12 +115,23 @@ def run_task():
     shared_state.trace_id = data.get("trace_id", shared_state.trace_id)
     shared_state.server_url = data.get("server_url", shared_state.server_url)
     shared_state.max_steps = data.get("max_steps", shared_state.max_steps)
+    # Accept either `mode` or `actor_mode` for the per-request actor override
+    # (e.g. "vanilla-claude" to skip trace/planner). Empty string clears it.
+    incoming_mode = data.get("mode", data.get("actor_mode", shared_state.mode))
+    if isinstance(incoming_mode, str):
+        incoming_mode = incoming_mode.strip() or None
+    shared_state.mode = incoming_mode
 
     shared_state.stop_event.clear()
     shared_state.processing_thread = threading.Thread(target=process_input, daemon=True)
     shared_state.processing_thread.start()
 
-    return jsonify({"status": "success", "message": "Task started", "task": shared_state.task})
+    return jsonify({
+        "status": "success",
+        "message": "Task started",
+        "task": shared_state.task,
+        "mode": shared_state.mode,
+    })
 
 
 @app.route("/stop", methods=["POST"])

Aloha_Act/app_server.py

@@ -2,6 +2,12 @@
 from datetime import datetime
 from typing import Any, Dict
 
+from dotenv import find_dotenv, load_dotenv
+
+# Load .env from project root before reading any env-driven config.
+# `override=False` lets pre-existing env vars win, matching standard precedence.
+load_dotenv(find_dotenv(usecwd=True), override=False)
+
 from flask import Flask, jsonify, request
 
 from config import config
@@ -52,6 +58,9 @@ def generate_action():
     screenshot = data.get("screenshot")
     query = data.get("query")
     action_history = data.get("action_history", [])
+    # Optional per-request actor override; when missing we fall back to
+    # `actor.model` (config.yaml: actor_model) inside the loop.
+    mode = data.get("mode") or data.get("actor_model")
 
     # Set up a per-request logging directory under log root
     log_dir = setup_logging_directory(task_id)
@@ -67,6 +76,7 @@ def generate_action():
         screenshot=screenshot,
         action_history=action_history,
         trace_name=trace_name,
+        mode=mode,
         log_dir=log_dir,
     )
 

Aloha_Act/config/api_keys_example.json

@@ -1,6 +1,8 @@
 {
-    "OPENAI_API_KEY": "sk-proj-123", 
+    "OPENAI_API_KEY": "sk-proj-123",
     "OPERATOR_OPENAI_API_KEY": "sk-proj-123",
-    "CLAUDE_API_KEY": "sk-ant-api03-123"
-}
+    "CLAUDE_API_KEY": "sk-ant-api03-123",
 
+    "ANTHROPIC_BASE_URL": "https://api.gpugeek.com",
+    "ANTHROPIC_AUTH_TOKEN": "vendor-bearer-token-here"
+}

Aloha_Act/config/config.yaml

@@ -3,6 +3,33 @@ log_dir: "./logs"
 trace_dir: "./trace_data"
 
 # model settings
-planner_model: "gpt-5"
-actor_model: "oai-operator"
-os_name: "windows"  # "windows", "mac", "linux"
+# planner_model is auto-routed by run_llm:
+#   - "claude*" / "*Sonnet*" / "*Opus*" / "*Haiku*" / "Vendor2/Claude-*" -> Anthropic
+#   - everything else -> OpenAI Responses API
+planner_model: "Vendor2/Claude-4.6-Opus"
+# actor_model options:
+#   "oai-operator"               OpenAI computer-use-preview
+#   "claude-computer-use"        Claude beta tool_use (computer_20250124/20251124).
+#                                Requires a vendor that *truly* implements the
+#                                Computer Use beta with stable input schemas.
+#   "claude-aloha-computer-use"  Claude with NO beta tool: system prompt enforces
+#                                the Aloha JSON schema directly; the model emits
+#                                a single JSON action per turn. Robust against
+#                                vendors that simulate Computer Use via prompt
+#                                engineering (e.g. gpugeek/Vendor2).
+#   "ui-tars"                    UI-TARS / Qwen-style grounding model.
+actor_model: "claude-aloha-computer-use"
+# os_name: "windows"  # "windows", "mac", "linux"
+os_name: "mac"  # "windows", "mac", "linux"
+
+# Claude model name used by both `claude-computer-use` and
+# `claude-aloha-computer-use` actor backends.
+# Precedence: ANTHROPIC_MODEL env / .env > this value > built-in default.
+# Examples:
+#   "claude-sonnet-4-5-20250929"   (official Anthropic API)
+#   "Vendor2/Claude-4.6-Sonnet"    (gpugeek third-party endpoint)
+#   "Vendor2/Claude-4.6-Opus"      (gpugeek third-party endpoint)
+# For `claude-computer-use` the agent auto-selects the right beta header / tool
+# type based on the version it parses out of the name.
+claude_model: "Vendor2/Claude-4.6-Sonnet"
+

Aloha_Act/scripts/aloha_run.py

@@ -26,6 +26,11 @@
 import logging
 from pathlib import Path
 
+from dotenv import find_dotenv, load_dotenv
+
+# Load project-root .env so env vars propagate to subprocesses we spawn.
+load_dotenv(find_dotenv(usecwd=True), override=False)
+
 # Configure logging at import time so it's available throughout the module
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 log = logging.getLogger(__name__)