diff --git a/backend/.env.example b/backend/.env.example index 5b342cc17..e3ec885a4 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -112,23 +112,37 @@ OTEL_EXPORTER_OTLP_TOKEN= # Web search via Exa. EXA_API_KEY= +# === Agents — optional === +# AGENT_DEPLOY: expose the descriptor-driven "Add agent" deploy flow (THU-743). +# Opt-in; defaults off. Haystack is the first deployable provider. +AGENT_DEPLOY=false + # === Haystack / Deepset Cloud — optional === -# Configure to expose Deepset RAG pipelines as managed ACP agents. +# Configure to expose Deepset pipelines as managed ACP agents. Pipelines are +# discovered live from the Deepset API (GET /agents lists DEPLOYED, prompt-capable +# pipelines, excluding Thunderbolt-deployed `tb-*` ones) — no static list to maintain. # HAYSTACK_BASE_URL: Deepset Cloud root (e.g. https://api.cloud.deepset.ai). # HAYSTACK_API_KEY: bearer token used as `Authorization: Bearer …`. # HAYSTACK_WORKSPACE: workspace slug. URLs become # `${HAYSTACK_BASE_URL}/api/v1/workspaces/${HAYSTACK_WORKSPACE}/...`. -# HAYSTACK_PIPELINES: JSON array. Each entry needs: -# - id public slug surfaced to the FE (`rag-chat`). -# - name human-readable label. -# - pipelineName Deepset URL slug used in /pipelines//chat-stream. -# - pipelineId Deepset UUID, sent as `pipeline_id` when bootstrapping -# a `search_session`. -# - icon (optional) Phosphor icon name shown in the agent picker. HAYSTACK_BASE_URL= HAYSTACK_API_KEY= HAYSTACK_WORKSPACE= -HAYSTACK_PIPELINES='[{"id":"rag-chat","name":"RAG Chat","pipelineName":"rag-chat-pipeline","pipelineId":"15cf8b39-0000-0000-0000-000000000000","icon":"book"}]' +# HAYSTACK_TEMPLATE_PIPELINE: name of an existing pipeline whose YAML is cloned when a +# user deploys a Haystack agent (requires AGENT_DEPLOY=true). Empty = not deployable. +HAYSTACK_TEMPLATE_PIPELINE= + +# === OpenClaw (sandbox-hosted agent) — optional === +# Configure to expose OpenClaw as a one-click deployable agent (requires AGENT_DEPLOY=true). +# Owner-managed: we spin a per-user E2B microVM from a prebuilt template (name hardcoded +# as `thunderbolt-openclaw`) and relay ACP frames to it. All three values must be set, +# else OpenClaw is not deployable. +# E2B_API_KEY: E2B account key used to create/connect/kill sandboxes. +# OPENCLAW_MODEL: "provider/model" the agent runs (e.g. openrouter/...). +# OPENCLAW_OPENROUTER_API_KEY: shared OpenRouter key injected into the sandbox for inference. +E2B_API_KEY= +OPENCLAW_MODEL= +OPENCLAW_OPENROUTER_API_KEY= # === Trusted proxy === # Controls which proxy headers are trusted for IP extraction in rate limiting. diff --git a/backend/bun.lock b/backend/bun.lock index fc3f5068d..3b00daf2f 100644 --- a/backend/bun.lock +++ b/backend/bun.lock @@ -23,6 +23,7 @@ "bun-types": "^1.2.22", "dotenv": "^17.2.3", "drizzle-orm": "^0.45.2", + "e2b": "^2.36.1", "elysia": "^1.4.28", "exa-js": "^2.16.0", "jose": "^6.2.2", @@ -110,8 +111,14 @@ "@borewit/text-codec": ["@borewit/text-codec@0.2.2", "", {}, "sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ=="], + "@bufbuild/protobuf": ["@bufbuild/protobuf@2.13.0", "", {}, "sha512-acq7c49vxfm1ggJ95P70TX7ABDM0vxr1SYD3BB0o0jnBLB4OAqeHyKuN+cD3w80gXEDQ2zxHpR6CUeA+O/aU9g=="], + "@cfworker/json-schema": ["@cfworker/json-schema@4.1.1", "", {}, "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og=="], + "@connectrpc/connect": ["@connectrpc/connect@2.1.2", "", { "peerDependencies": { "@bufbuild/protobuf": "^2.7.0" } }, "sha512-MXkBijtcX09R10Eb6sFeIetc6w6746eio6xtfuyVOH7oQAacT1X0GzMIQFux6Qy8cq3W/T5qX5Bei8YbFtmRGA=="], + + "@connectrpc/connect-web": ["@connectrpc/connect-web@2.1.2", "", { "peerDependencies": { "@bufbuild/protobuf": "^2.7.0", "@connectrpc/connect": "2.1.2" } }, "sha512-1tfaK85MU+gJjwwmL31d2rzdf0XCYX99chZf63uG89SGBUd4XuZ4ZzhGo2u79TPXOE6nLIZQ2okrpyey42PYdg=="], + "@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="], "@electric-sql/pglite": ["@electric-sql/pglite@0.5.4", "", {}, "sha512-yYZUyyXrHU7tPlCjwZQJ6hIG9DscdCCn7Uk0mYKwC1FeHX286AbcmFveMiRBEak8e9iPupjsoVImN3yJZVed2g=="], @@ -280,6 +287,8 @@ "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="], + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], @@ -530,8 +539,12 @@ "caniuse-lite": ["caniuse-lite@1.0.30001788", "", {}, "sha512-6q8HFp+lOQtcf7wBK+uEenxymVWkGKkjFpCvw5W25cmMwEDU45p1xQFBQv8JDlMMry7eNxyBaR+qxgmTUZkIRQ=="], + "chalk": ["chalk@5.6.2", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="], + "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], + "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], + "citty": ["citty@0.2.2", "", {}, "sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w=="], "cjs-module-lexer": ["cjs-module-lexer@1.4.3", "", {}, "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q=="], @@ -548,6 +561,8 @@ "commander": ["commander@13.1.0", "", {}, "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw=="], + "compare-versions": ["compare-versions@6.1.1", "", {}, "sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg=="], + "conf": ["conf@15.1.0", "", { "dependencies": { "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "atomically": "^2.0.3", "debounce-fn": "^6.0.0", "dot-prop": "^10.0.0", "env-paths": "^3.0.0", "json-schema-typed": "^8.0.1", "semver": "^7.7.2", "uint8array-extras": "^1.5.0" } }, "sha512-Uy5YN9KEu0WWDaZAVJ5FAmZoaJt9rdK6kH+utItPyGsCqCgaTKkrmZx3zoE0/3q6S3bcp3Ihkk+ZqPxWxFK5og=="], "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], @@ -582,6 +597,8 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "dockerfile-ast": ["dockerfile-ast@0.7.1", "", { "dependencies": { "vscode-languageserver-textdocument": "^1.0.8", "vscode-languageserver-types": "^3.17.3" } }, "sha512-oX/A4I0EhSkGqrFv0YuvPkBUSYp1XiY8O8zAKc8Djglx8ocz+JfOr8gP0ryRMC2myqvDLagmnZaU9ot1vG2ijw=="], + "dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="], "domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="], @@ -598,6 +615,8 @@ "drizzle-orm": ["drizzle-orm@0.45.2", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@tidbcloud/serverless": "*", "@types/better-sqlite3": "*", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=7", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "knex": "*", "kysely": "*", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@tidbcloud/serverless", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "knex", "kysely", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q=="], + "e2b": ["e2b@2.36.1", "", { "dependencies": { "@bufbuild/protobuf": "^2.12.1", "@connectrpc/connect": "^2.1.2", "@connectrpc/connect-web": "^2.1.2", "chalk": "^5.3.0", "compare-versions": "^6.1.0", "dockerfile-ast": "^0.7.1", "glob": "^13.0.6", "openapi-fetch": "^0.14.1", "platform": "^1.3.6", "tar": "^7.5.19", "undici": "^7.28.0" }, "optionalDependencies": { "undici8": "npm:undici@8.8.0" } }, "sha512-ERDTKfIM14mQX0cizRpfLCWDX4A83rGvbAbnUISDExM26WuMS4Jd9v2EWrY03LXfEKBst954SzFvKIZZEiKwng=="], + "ecdsa-sig-formatter": ["ecdsa-sig-formatter@1.0.11", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ=="], "elysia": ["elysia@1.4.28", "", { "dependencies": { "cookie": "^1.1.1", "exact-mirror": "^0.2.7", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-Vrx8sBnvq8squS/3yNBzR1jBXI+SgmnmvwawPjNuEHndUe5l1jV2Gp6JJ4ulDkEB8On6bWmmuyPpA+bq4t+WYg=="], @@ -816,6 +835,8 @@ "minipass": ["minipass@7.1.3", "", {}, "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A=="], + "minizlib": ["minizlib@3.1.0", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw=="], + "module-details-from-path": ["module-details-from-path@1.0.4", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -854,8 +875,12 @@ "openai": ["openai@6.34.0", "", { "peerDependencies": { "ws": "^8.18.0", "zod": "^3.25 || ^4.0" }, "optionalPeers": ["ws", "zod"], "bin": { "openai": "bin/cli" } }, "sha512-yEr2jdGf4tVFYG6ohmr3pF6VJuveP0EA/sS8TBx+4Eq5NT10alu5zg2dmxMXMgqpihRDQlFGpRt2XwsGj+Fyxw=="], + "openapi-fetch": ["openapi-fetch@0.14.1", "", { "dependencies": { "openapi-typescript-helpers": "^0.0.15" } }, "sha512-l7RarRHxlEZYjMLd/PR0slfMVse2/vvIAGm75/F7J6MlQ8/b9uUQmUF2kCPrQhJqMXSxmYWObVgeYXbFYzZR+A=="], + "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], + "openapi-typescript-helpers": ["openapi-typescript-helpers@0.0.15", "", {}, "sha512-opyTPaunsklCBpTK8JGef6mfPhLSnyy5a0IN9vKtx3+4aExf+KxEqYwIy3hqkedXIB97u357uLMJsOnm3GVjsw=="], + "optionator": ["optionator@0.9.4", "", { "dependencies": { "deep-is": "^0.1.3", "fast-levenshtein": "^2.0.6", "levn": "^0.4.1", "prelude-ls": "^1.2.1", "type-check": "^0.4.0", "word-wrap": "^1.2.5" } }, "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g=="], "p-finally": ["p-finally@1.0.0", "", {}, "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow=="], @@ -902,6 +927,8 @@ "pino-std-serializers": ["pino-std-serializers@7.1.0", "", {}, "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw=="], + "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "postal-mime": ["postal-mime@2.7.4", "", {}, "sha512-0WdnFQYUrPGGTFu1uOqD2s7omwua8xaeYGdO6rb88oD5yJ/4pPHDA4sdWqfD8wQVfCny563n/HQS7zTFft+f/g=="], "postcss": ["postcss@8.4.31", "", { "dependencies": { "nanoid": "^3.3.6", "picocolors": "^1.0.0", "source-map-js": "^1.0.2" } }, "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ=="], @@ -1028,6 +1055,8 @@ "tailwindcss": ["tailwindcss@4.2.4", "", {}, "sha512-HhKppgO81FQof5m6TEnuBWCZGgfRAWbaeOaGT00KOy/Pf/j6oUihdvBpA7ltCeAvZpFhW3j0PTclkxsd4IXYDA=="], + "tar": ["tar@7.5.22", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.1.0", "yallist": "^5.0.0" } }, "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA=="], + "thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="], "tinyexec": ["tinyexec@1.1.2", "", {}, "sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA=="], @@ -1060,14 +1089,22 @@ "uint8array-extras": ["uint8array-extras@1.5.0", "", {}, "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A=="], + "undici": ["undici@7.29.0", "", {}, "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw=="], + "undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="], + "undici8": ["undici@8.8.0", "", {}, "sha512-ubshXMXwF3MQIMF1y/WxZdNBnjEKeSg2wF5mcGUtU55YTw34tnVVpKRlLf7ruDXZ5344KokPVX4RBx1wJm64Bw=="], + "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], "uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], + "vscode-languageserver-textdocument": ["vscode-languageserver-textdocument@1.0.12", "", {}, "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA=="], + + "vscode-languageserver-types": ["vscode-languageserver-types@3.18.0", "", {}, "sha512-8TsGPNMIMiiBdkORgRSvLjuiEIiAFtO+KssmYWxQ+uSVvlf7RjK8YKCOjPzZ+YA04jXEV7+7LvkSmHkhpNS99g=="], + "web-streams-polyfill": ["web-streams-polyfill@3.3.3", "", {}, "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw=="], "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], @@ -1096,6 +1133,8 @@ "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], + "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], + "yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], diff --git a/backend/package.json b/backend/package.json index cf159241b..50c9c45e7 100644 --- a/backend/package.json +++ b/backend/package.json @@ -35,6 +35,7 @@ "bun-types": "^1.2.22", "dotenv": "^17.2.3", "drizzle-orm": "^0.45.2", + "e2b": "^2.36.1", "elysia": "^1.4.28", "exa-js": "^2.16.0", "jose": "^6.2.2", diff --git a/backend/src/agents/deployment-id.test.ts b/backend/src/agents/deployment-id.test.ts new file mode 100644 index 000000000..c08ab1752 --- /dev/null +++ b/backend/src/agents/deployment-id.test.ts @@ -0,0 +1,33 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { decodeDeploymentId, encodeDeploymentId } from './deployment-id' + +describe('deployment id codec', () => { + it('round-trips provider + ref', () => { + const id = encodeDeploymentId('haystack', 'tb-my-agent') + expect(id).toBe('haystack:tb-my-agent') + expect(decodeDeploymentId(id)).toEqual({ provider: 'haystack', ref: 'tb-my-agent' }) + }) + + it('splits on the first separator so refs may contain colons', () => { + expect(decodeDeploymentId('haystack:a:b:c')).toEqual({ provider: 'haystack', ref: 'a:b:c' }) + }) + + it('rejects a provider containing the separator', () => { + expect(() => encodeDeploymentId('hay:stack', 'ref')).toThrow() + }) + + it('rejects an empty provider or ref', () => { + expect(() => encodeDeploymentId('', 'ref')).toThrow() + expect(() => encodeDeploymentId('haystack', '')).toThrow() + }) + + it('rejects malformed ids', () => { + expect(() => decodeDeploymentId('haystack')).toThrow() + expect(() => decodeDeploymentId(':ref')).toThrow() + expect(() => decodeDeploymentId('haystack:')).toThrow() + }) +}) diff --git a/backend/src/agents/deployment-id.ts b/backend/src/agents/deployment-id.ts new file mode 100644 index 000000000..d954313c1 --- /dev/null +++ b/backend/src/agents/deployment-id.ts @@ -0,0 +1,33 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * A deployment id is `:` — self-describing so the deployment-status + * endpoint can resolve the owning provider and poll the host directly, with no + * server-side deployment table (THU-743). `provider` is the registry key (the + * agent kind); `ref` is the host-scoped identifier (e.g. a Deepset pipeline name). + */ + +const separator = ':' + +/** Build a deployment id from a provider id and a host reference. */ +export const encodeDeploymentId = (provider: string, ref: string): string => { + if (!provider || provider.includes(separator)) { + throw new Error(`invalid provider for deployment id: ${JSON.stringify(provider)}`) + } + if (!ref) { + throw new Error('deployment id requires a non-empty ref') + } + return `${provider}${separator}${ref}` +} + +/** Parse a deployment id back into its provider and ref. Splits on the first + * separator so a ref may itself contain `:`. */ +export const decodeDeploymentId = (deploymentId: string): { provider: string; ref: string } => { + const index = deploymentId.indexOf(separator) + if (index <= 0 || index === deploymentId.length - 1) { + throw new Error(`malformed deployment id: ${JSON.stringify(deploymentId)}`) + } + return { provider: deploymentId.slice(0, index), ref: deploymentId.slice(index + 1) } +} diff --git a/backend/src/agents/discovery.ts b/backend/src/agents/discovery.ts index dc8c1163d..d63a9e998 100644 --- a/backend/src/agents/discovery.ts +++ b/backend/src/agents/discovery.ts @@ -4,19 +4,54 @@ import type { Settings } from '@/config/settings' import type { RemoteAgentDescriptor } from '@shared/acp-types' +import type { + AgentDescriptor, + AgentSpec, + DeployResponse, + DeploymentStatusResponse, + UndeployResponse, +} from '@shared/agent-descriptors' /** - * An agent provider contributes one or more {@link RemoteAgentDescriptor} entries - * to the `GET /agents` response. The Haystack module calls {@link registerAgentProvider} - * at startup with its provider; future managed agents follow the same shape. + * Everything a provider needs to service a catalog/deploy/status request: the + * incoming request (WS URL host derivation), resolved settings, and the calling + * user's id (deployment namespacing). + */ +export type ProviderContext = { + request: Request + settings: Settings + userId: string +} + +/** + * An agent provider is the adapter for one agent "kind" (haystack/openclaw/…), + * keyed by {@link AgentProvider.id}. It always contributes discovery descriptors + * (`list`), and may optionally be *deployable* — exposing a creation `catalog` + * plus `deploy`/`status` lifecycle verbs (THU-743). The Haystack module calls + * {@link registerAgentProvider} at startup; future managed agents follow the same shape. */ export type AgentProvider = { - /** Stable identifier for the provider. Re-registering the same id is a no-op. */ + /** Stable identifier for the provider. Re-registering the same id is a no-op. + * Matches `AgentDescriptor.provider`, so deploy requests route back here. */ id: string /** Returns descriptors visible to the caller. May read settings or the request - * (e.g. for WS URL host derivation). Throwing here is isolated per-provider — - * the discovery route swallows the failure and continues. */ - list: (request: Request, settings: Settings) => RemoteAgentDescriptor[] + * (e.g. for WS URL host derivation) and may be async (e.g. a live host query). + * Throwing / rejecting here is isolated per-provider — the discovery route + * swallows the failure and continues. */ + list: (request: Request, settings: Settings) => RemoteAgentDescriptor[] | Promise + /** Curated creation descriptors (static). Absent → the provider is discovery-only + * and not deployable. */ + catalog?: (ctx: ProviderContext) => AgentDescriptor[] + /** Create + deploy an instance from an already-validated spec. The returned + * `deploymentId` encodes `provider:ref` (see `deployment-id.ts`). */ + deploy?: (spec: AgentSpec, ctx: ProviderContext) => Promise + /** Live deployment status for `ref`, fetched from the host — never stored. */ + status?: (ref: string, ctx: ProviderContext) => Promise + /** Trigger teardown of `ref` on the host (undeploy / kill). Returns once the + * teardown is accepted — it does NOT block on the full unmount. Idempotent: a + * foreign or already-gone deployment is a no-op so the client can still drop + * its local row. Absent → the provider's deployments can't be torn down. */ + undeploy?: (ref: string, ctx: ProviderContext) => Promise } /** @@ -38,6 +73,10 @@ export const registerAgentProvider = (provider: AgentProvider): void => { /** Return the current set of registered providers in registration order. */ export const getRegisteredProviders = (): AgentProvider[] => [...providers] +/** Look up a provider by id (its "kind"). Used to route deploy/status/catalog + * requests back to the owning provider. */ +export const getProviderById = (id: string): AgentProvider | undefined => providers.find((p) => p.id === id) + /** Test helper — clears all providers. Not exported from the module index. */ export const resetAgentProvidersForTesting = (): void => { providers.length = 0 diff --git a/backend/src/agents/index.ts b/backend/src/agents/index.ts index 4a23f3d00..189d4cc7a 100644 --- a/backend/src/agents/index.ts +++ b/backend/src/agents/index.ts @@ -3,5 +3,13 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ export { createAgentsRoutes } from './routes' -export { registerAgentProvider, getRegisteredProviders, buildWebSocketUrl, type AgentProvider } from './discovery' +export { + registerAgentProvider, + getRegisteredProviders, + getProviderById, + buildWebSocketUrl, + type AgentProvider, + type ProviderContext, +} from './discovery' +export { encodeDeploymentId, decodeDeploymentId } from './deployment-id' export type { AgentsErrorCode, AgentsErrorResponse } from './types' diff --git a/backend/src/agents/routes.test.ts b/backend/src/agents/routes.test.ts index 901f310aa..6c0aafc30 100644 --- a/backend/src/agents/routes.test.ts +++ b/backend/src/agents/routes.test.ts @@ -16,6 +16,7 @@ import type { Auth } from '@/auth/elysia-plugin' import { clearSettingsCache } from '@/config/settings' import type { RemoteAgentDescriptor } from '@shared/acp-types' +import { schemaVersionMismatch, type AgentDescriptor } from '@shared/agent-descriptors' import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { Elysia } from 'elysia' import { registerAgentProvider, resetAgentProvidersForTesting } from './discovery' @@ -163,3 +164,184 @@ describe('GET /agents', () => { expect(body.agents).toEqual([customDescriptor]) }) }) + +describe('agent deploy endpoints', () => { + const envKeys = ['AGENT_DEPLOY', 'ENABLED_AGENTS'] as const + let savedEnv: Partial> + + const descriptor: AgentDescriptor = { + id: 'haystack', + provider: 'haystack', + name: 'Haystack', + description: null, + icon: null, + schemaVersion: 3, + action: 'deploy', + steps: [ + { id: 'basics', title: 'Basics', fields: [{ key: 'name', label: 'Name', widget: 'text', required: true }] }, + ], + } + + /** A deployable provider whose verbs echo their inputs so tests can assert dispatch. */ + const registerDeployable = () => + registerAgentProvider({ + id: 'haystack', + list: () => [], + catalog: () => [descriptor], + deploy: (spec) => + Promise.resolve({ deploymentId: `haystack:tb-${String(spec.name)}`, status: 'pending', connection: null }), + status: (ref) => Promise.resolve({ deploymentId: `haystack:${ref}`, status: 'running', connection: null }), + undeploy: (ref) => Promise.resolve({ deploymentId: `haystack:${ref}`, status: 'gone' }), + }) + + const post = (app: Elysia, path: string, body: unknown) => + app.handle( + new Request(`http://localhost${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + ) + + beforeEach(() => { + resetAgentProvidersForTesting() + savedEnv = {} + for (const key of envKeys) { + savedEnv[key] = process.env[key] + delete process.env[key] + } + process.env.AGENT_DEPLOY = 'true' + clearSettingsCache() + }) + + afterEach(() => { + resetAgentProvidersForTesting() + for (const key of envKeys) { + const saved = savedEnv[key] + if (saved === undefined) { + delete process.env[key] + } else { + process.env[key] = saved + } + } + clearSettingsCache() + }) + + it('GET /catalog returns the descriptors when the flag is on', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/catalog')) + expect(res.status).toBe(200) + const body = await res.json() + expect(body.version).toBe('1') + expect(body.descriptors.map((d: AgentDescriptor) => d.id)).toEqual(['haystack']) + }) + + it('GET /catalog returns 404 when the agentDeploy flag is off', async () => { + delete process.env.AGENT_DEPLOY + clearSettingsCache() + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/catalog')) + expect(res.status).toBe(404) + }) + + it('GET /catalog returns 403 for anonymous users', async () => { + const app = buildApp(buildAuth({ id: 'anon-1', isAnonymous: true })) + const res = await app.handle(new Request('http://localhost/agents/catalog')) + expect(res.status).toBe(403) + }) + + it('POST /deploy dispatches a valid spec to the provider', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await post(app, '/agents/deploy', { descriptorId: 'haystack', schemaVersion: 3, spec: { name: 'Bot' } }) + expect(res.status).toBe(200) + const body = await res.json() + expect(body).toEqual({ deploymentId: 'haystack:tb-Bot', status: 'pending', connection: null }) + }) + + it('POST /deploy returns 409 on a stale schema version', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await post(app, '/agents/deploy', { descriptorId: 'haystack', schemaVersion: 2, spec: { name: 'Bot' } }) + expect(res.status).toBe(409) + const body = await res.json() + expect(body.code).toBe(schemaVersionMismatch) + }) + + it('POST /deploy returns 400 when the spec fails re-validation', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + // `name` is required by the descriptor but missing here. + const res = await post(app, '/agents/deploy', { descriptorId: 'haystack', schemaVersion: 3, spec: {} }) + expect(res.status).toBe(400) + }) + + it('POST /deploy returns 404 for an unknown descriptor', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await post(app, '/agents/deploy', { descriptorId: 'ghost', schemaVersion: 3, spec: { name: 'x' } }) + expect(res.status).toBe(404) + }) + + it('GET /deployments/:id returns live status from the provider', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/deployments/haystack:tb-bot')) + expect(res.status).toBe(200) + const body = await res.json() + expect(body).toEqual({ deploymentId: 'haystack:tb-bot', status: 'running', connection: null }) + }) + + it('GET /deployments/:id returns 400 for a malformed id', async () => { + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/deployments/nocolon')) + expect(res.status).toBe(400) + }) + + it('GET /deployments/:id returns 404 for an unknown provider', async () => { + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/deployments/ghost:ref')) + expect(res.status).toBe(404) + }) + + const del = (app: Elysia, path: string) => app.handle(new Request(`http://localhost${path}`, { method: 'DELETE' })) + + it('DELETE /deployments/:id dispatches undeploy to the provider', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await del(app, '/agents/deployments/haystack:tb-bot') + expect(res.status).toBe(200) + const body = await res.json() + expect(body).toEqual({ deploymentId: 'haystack:tb-bot', status: 'gone' }) + }) + + it('DELETE /deployments/:id returns 404 when the feature is off', async () => { + delete process.env.AGENT_DEPLOY + clearSettingsCache() + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await del(app, '/agents/deployments/haystack:tb-bot') + expect(res.status).toBe(404) + }) + + it('DELETE /deployments/:id returns 400 for a malformed id', async () => { + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await del(app, '/agents/deployments/nocolon') + expect(res.status).toBe(400) + }) + + it('DELETE /deployments/:id returns 404 when the provider has no undeploy', async () => { + registerAgentProvider({ id: 'haystack', list: () => [] }) // discovery-only, no undeploy + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await del(app, '/agents/deployments/haystack:tb-bot') + expect(res.status).toBe(404) + }) + + it('DELETE /deployments/:id returns 403 for anonymous users', async () => { + const app = buildApp(buildAuth({ id: 'anon-1', isAnonymous: true })) + const res = await del(app, '/agents/deployments/haystack:tb-bot') + expect(res.status).toBe(403) + }) +}) diff --git a/backend/src/agents/routes.ts b/backend/src/agents/routes.ts index 0747ac99f..c234eb796 100644 --- a/backend/src/agents/routes.ts +++ b/backend/src/agents/routes.ts @@ -7,11 +7,26 @@ import { getEnabledAgentsList, getSettings, type Settings } from '@/config/setti import { createStandaloneLogger } from '@/config/logger' import { safeErrorHandler } from '@/middleware/error-handling' import type { AgentDiscoveryResponse, RemoteAgentDescriptor } from '@shared/acp-types' +import { + deployRequestSchema, + schemaVersionMismatch, + specSchemaForDescriptor, + type AgentCatalogResponse, + type AgentDescriptor, + type AgentSpec, + type DeployResponse, + type DeploymentStatusResponse, + type UndeployResponse, +} from '@shared/agent-descriptors' import type { User } from '@shared/types/auth' import { Elysia } from 'elysia' -import { getRegisteredProviders } from './discovery' +import { decodeDeploymentId } from './deployment-id' +import { getProviderById, getRegisteredProviders, type ProviderContext } from './discovery' import type { AgentsErrorResponse } from './types' +/** Generic error body for the deploy endpoints (may carry an app-specific `code`). */ +type ErrorBody = { error: string; code?: string } + /** * Mounts `GET /agents`, the ACP discovery endpoint. * @@ -37,7 +52,33 @@ export const createAgentsRoutes = (auth: Auth) => const sessionUser = session?.user as User | undefined return { user: sessionUser ?? null } }) - .get('/', ({ request, set, user }): AgentDiscoveryResponse | AgentsErrorResponse | { error: string } => { + .get( + '/', + async ({ request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } + + const settings = getSettings() + const enabledIds = getEnabledAgentsList(settings) + const allowedById = (id: string) => enabledIds.length === 0 || enabledIds.includes(id) + + const agents = await collectAgents(request, settings) + const filtered = agents.filter((descriptor) => allowedById(descriptor.id)) + + return { + version: '1', + agents: filtered, + allowCustomAgents: settings.allowCustomAgents, + } + }, + ) + .get('/catalog', async ({ request, set, user }): Promise => { if (!user) { set.status = 401 return { error: 'Unauthorized' } @@ -46,19 +87,111 @@ export const createAgentsRoutes = (auth: Auth) => set.status = 403 return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } } - const settings = getSettings() - const enabledIds = getEnabledAgentsList(settings) - const allowedById = (id: string) => enabledIds.length === 0 || enabledIds.includes(id) - - const agents = collectAgents(request, settings) - const filtered = agents.filter((descriptor) => allowedById(descriptor.id)) - - return { - version: '1', - agents: filtered, - allowCustomAgents: settings.allowCustomAgents, + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + return { version: '1', descriptors: collectCatalog({ request, settings, userId: user.id }) } + }) + .post('/deploy', async ({ body, request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } + const settings = getSettings() + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + const parsed = deployRequestSchema.safeParse(body) + if (!parsed.success) { + set.status = 400 + return { error: 'Invalid deploy request' } + } + const ctx: ProviderContext = { request, settings, userId: user.id } + const descriptor = collectCatalog(ctx).find((d) => d.id === parsed.data.descriptorId) + if (!descriptor) { + set.status = 404 + return { error: 'Unknown agent' } + } + // Reject a spec built against a stale descriptor before we act on it. + if (descriptor.schemaVersion !== parsed.data.schemaVersion) { + set.status = 409 + return { error: 'Schema version mismatch', code: schemaVersionMismatch } + } + const provider = getProviderById(descriptor.provider) + if (!provider?.deploy) { + set.status = 404 + return { error: 'Agent is not deployable' } + } + // Re-validate the spec server-side — the client is never trusted. + const specResult = specSchemaForDescriptor(descriptor).safeParse(parsed.data.spec) + if (!specResult.success) { + set.status = 400 + return { error: 'Invalid spec' } + } + return provider.deploy(specResult.data as AgentSpec, ctx) + }) + .get('/deployments/:id', async ({ params, request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } + const settings = getSettings() + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + let decoded: { provider: string; ref: string } + try { + decoded = decodeDeploymentId(params.id) + } catch { + set.status = 400 + return { error: 'Invalid deployment id' } + } + const provider = getProviderById(decoded.provider) + if (!provider?.status) { + set.status = 404 + return { error: 'Unknown provider' } + } + return provider.status(decoded.ref, { request, settings, userId: user.id }) + }) + .delete('/deployments/:id', async ({ params, request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } } + const settings = getSettings() + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + let decoded: { provider: string; ref: string } + try { + decoded = decodeDeploymentId(params.id) + } catch { + set.status = 400 + return { error: 'Invalid deployment id' } + } + const provider = getProviderById(decoded.provider) + if (!provider?.undeploy) { + set.status = 404 + return { error: 'Unknown provider' } + } + return provider.undeploy(decoded.ref, { request, settings, userId: user.id }) }) /** @@ -66,15 +199,36 @@ export const createAgentsRoutes = (auth: Auth) => * results. A throwing provider is logged and skipped — one misbehaving plugin * never poisons the response for the others. */ -const collectAgents = (request: Request, settings: Settings): RemoteAgentDescriptor[] => { +const collectAgents = async (request: Request, settings: Settings): Promise => { const log = createStandaloneLogger(settings) const out: RemoteAgentDescriptor[] = [] for (const provider of getRegisteredProviders()) { try { - out.push(...provider.list(request, settings)) + out.push(...(await provider.list(request, settings))) } catch (error) { log.warn({ err: error, providerId: provider.id }, 'agent provider list() failed; skipping') } } return out } + +/** + * Ask every deployable provider for its creation descriptors. Like + * {@link collectAgents}, a throwing provider is logged and skipped so one bad + * plugin never poisons the catalog. + */ +const collectCatalog = (ctx: ProviderContext): AgentDescriptor[] => { + const log = createStandaloneLogger(ctx.settings) + const out: AgentDescriptor[] = [] + for (const provider of getRegisteredProviders()) { + if (!provider.catalog) { + continue + } + try { + out.push(...provider.catalog(ctx)) + } catch (error) { + log.warn({ err: error, providerId: provider.id }, 'agent provider catalog() failed; skipping') + } + } + return out +} diff --git a/backend/src/api/config.ts b/backend/src/api/config.ts index 8a05f6262..3cdedb2c9 100644 --- a/backend/src/api/config.ts +++ b/backend/src/api/config.ts @@ -24,6 +24,8 @@ export const createConfigRoutes = (settings: Settings) => // contract reads as a positive capability ("enabled"). builtInAgentEnabled: !settings.disableBuiltInAgent, allowCustomAgents: settings.allowCustomAgents, + // Opt-in flag (THU-743): exposes the descriptor-driven agent deploy flow. + agentDeploy: settings.agentDeploy, // Omit when unset so the frontend treats it as "no enforcement" without parsing an empty string as semver. minAppVersion: settings.minAppVersion || undefined, defaults: { diff --git a/backend/src/api/powersync.test.ts b/backend/src/api/powersync.test.ts index 0bda64d28..7a97a80ef 100644 --- a/backend/src/api/powersync.test.ts +++ b/backend/src/api/powersync.test.ts @@ -73,10 +73,14 @@ const powersyncSettings: Settings = { enabledAgents: '', allowCustomAgents: true, disableBuiltInAgent: false, + agentDeploy: false, haystackBaseUrl: '', haystackApiKey: '', haystackWorkspace: '', - haystackPipelines: '', + haystackTemplatePipeline: '', + e2bApiKey: '', + openclawModel: '', + openclawOpenrouterApiKey: '', minAppVersion: '', } diff --git a/backend/src/config/settings.ts b/backend/src/config/settings.ts index 3b61e3b61..04f4a4d85 100644 --- a/backend/src/config/settings.ts +++ b/backend/src/config/settings.ts @@ -142,14 +142,26 @@ const settingsSchema = z // list (not just disabled) — for deployments that ship only their own agents (e.g. Deepset). // Surfaced to the UI via GET /config as `builtInAgentEnabled`. disableBuiltInAgent: z.boolean().default(false), + // When true, exposes the descriptor-driven "Add agent" deploy flow (THU-743). + // Opt-in: absent/false hides it. Surfaced to the UI via GET /config as `agentDeploy`. + agentDeploy: z.boolean().default(false), // Haystack-specific config (consumed by the Haystack provider, defined here for centralized config). haystackBaseUrl: z.string().default(''), haystackApiKey: z.string().default(''), // Deepset workspace slug. URLs are `${baseUrl}/api/v1/workspaces/${workspace}/...`. haystackWorkspace: z.string().default(''), - // JSON array of pipeline descriptors: [{id, name, pipelineName, pipelineId, description?, icon?}]. - // `id` is the public slug; `pipelineName` is the Deepset URL slug; `pipelineId` is the Deepset UUID. - haystackPipelines: z.string().default(''), + // Name of an existing Deepset pipeline whose query_yaml is cloned as the template + // for descriptor-driven deploys (THU-743). Empty = Haystack is not deployable. + haystackTemplatePipeline: z.string().default(''), + // OpenClaw-specific config (consumed by the OpenClaw provider). OpenClaw is a + // sandbox-hosted managed-acp agent: we spin a per-user E2B microVM from a + // static, prebuilt template and relay ACP frames to it. Owner-managed / one-click: + // the owner fixes the sandbox + model here; the user just clicks Deploy. + e2bApiKey: z.string().default(''), + // OpenClaw "provider/model" string the deployed agent runs (e.g. openrouter/...). Empty = not deployable. + openclawModel: z.string().default(''), + // Shared OpenRouter key injected into the sandbox env for inference. Empty = not deployable. + openclawOpenrouterApiKey: z.string().default(''), }) .superRefine((data, ctx) => { if (data.powersyncUrl && data.powersyncJwtSecret.length < 32) { @@ -229,10 +241,14 @@ const parseSettings = (): Settings => { enabledAgents: process.env.ENABLED_AGENTS || '', allowCustomAgents: process.env.ALLOW_CUSTOM_AGENTS !== 'false', disableBuiltInAgent: process.env.DISABLE_BUILT_IN_AGENT === 'true', + agentDeploy: process.env.AGENT_DEPLOY === 'true', haystackBaseUrl: process.env.HAYSTACK_BASE_URL || '', haystackApiKey: process.env.HAYSTACK_API_KEY || '', haystackWorkspace: process.env.HAYSTACK_WORKSPACE || '', - haystackPipelines: process.env.HAYSTACK_PIPELINES || '', + haystackTemplatePipeline: process.env.HAYSTACK_TEMPLATE_PIPELINE || '', + e2bApiKey: process.env.E2B_API_KEY || '', + openclawModel: process.env.OPENCLAW_MODEL || '', + openclawOpenrouterApiKey: process.env.OPENCLAW_OPENROUTER_API_KEY || '', } return settingsSchema.parse(env) diff --git a/backend/src/haystack/acp-server.test.ts b/backend/src/haystack/acp-server.test.ts index 96e9cabc7..e556de35c 100644 --- a/backend/src/haystack/acp-server.test.ts +++ b/backend/src/haystack/acp-server.test.ts @@ -30,7 +30,6 @@ const buildSettings = (overrides: Partial = {}): Settings => haystackBaseUrl: 'https://haystack.test', haystackApiKey: 'test-key', haystackWorkspace: 'ws-test', - haystackPipelines: '', logLevel: 'INFO', ...overrides, }) as Settings diff --git a/backend/src/haystack/files.test.ts b/backend/src/haystack/files.test.ts index 273cbf4b1..c8f95c8fb 100644 --- a/backend/src/haystack/files.test.ts +++ b/backend/src/haystack/files.test.ts @@ -66,7 +66,6 @@ const haystackSettings = createTestSettings({ haystackBaseUrl: 'https://haystack.test', haystackApiKey: 'sekrit', haystackWorkspace: 'ws-test', - haystackPipelines: '', }) const buildApp = (auth: Auth, fetchFn: typeof fetch) => diff --git a/backend/src/haystack/index.ts b/backend/src/haystack/index.ts index fa7a213cb..6f1beefb4 100644 --- a/backend/src/haystack/index.ts +++ b/backend/src/haystack/index.ts @@ -3,14 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ export { createHaystackRoutes } from './routes' -export { createHaystackProvider, haystackProviderId, parsePipelinesEnv } from './provider' +export { createHaystackProvider, haystackProviderId, resolveHaystackPipeline } from './provider' export { HaystackAcpServer, type HaystackAcpDeps } from './acp-server' export { parseHaystackSseStream, HaystackSseParseError } from './sse-parser' -export { - haystackEventSchema, - haystackPipelineDescriptorSchema, - haystackPipelinesEnvSchema, - type HaystackEvent, - type HaystackPipelineDescriptor, - type HaystackSessionContext, -} from './types' +export { haystackEventSchema, type HaystackEvent, type HaystackSessionContext } from './types' diff --git a/backend/src/haystack/management-client.test.ts b/backend/src/haystack/management-client.test.ts new file mode 100644 index 000000000..de93997fd --- /dev/null +++ b/backend/src/haystack/management-client.test.ts @@ -0,0 +1,126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { DeepsetManagementClient, DeepsetManagementError, type HaystackManagementConfig } from './management-client' + +const config: HaystackManagementConfig = { + haystackBaseUrl: 'https://api.cloud.deepset.ai', + haystackApiKey: 'sk-test', + haystackWorkspace: 'tutorial', +} + +type FakeResult = { status?: number; body?: unknown; text?: string } +type Call = { url: string; init?: RequestInit } + +const makeFetch = (handler: (url: string, init?: RequestInit) => FakeResult) => { + const calls: Call[] = [] + const fetchFn = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + calls.push({ url, init }) + const result = handler(url, init) + const status = result.status ?? 200 + const bodyText = result.text ?? (result.body !== undefined ? JSON.stringify(result.body) : '') + return new Response(bodyText, { status, statusText: status >= 400 ? 'Error' : 'OK' }) + }) as typeof fetch + return { fetchFn, calls } +} + +const wsBase = 'https://api.cloud.deepset.ai/api/v1/workspaces/tutorial' + +const pipeline = { + name: 'tb-my-agent', + pipeline_id: '70d781d8-4d4c-4154-95c1-6422cdf2c6fb', + status: 'DEPLOYMENT_IN_PROGRESS', + desired_status: 'DEPLOYED', +} + +describe('DeepsetManagementClient', () => { + it('creates a pipeline with the v2 body and drains the 201', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'tb-my-agent' } })) + const client = new DeepsetManagementClient(config, { fetchFn }) + await client.createPipeline({ name: 'tb-my-agent', queryYaml: 'components: {}' }) + expect(calls[0].url).toBe(`${wsBase}/pipelines`) + expect(calls[0].init?.method).toBe('POST') + expect(JSON.parse(String(calls[0].init?.body))).toEqual({ + name: 'tb-my-agent', + query_yaml: 'components: {}', + deepset_cloud_version: 'v2', + }) + }) + + it('appends dry_run when requested', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'x' } })) + const client = new DeepsetManagementClient(config, { fetchFn }) + await client.createPipeline({ name: 'x', queryYaml: 'y', dryRun: true }) + expect(calls[0].url).toBe(`${wsBase}/pipelines?dry_run=true`) + }) + + it('sends the bearer token from config', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'x' } })) + await new DeepsetManagementClient(config, { fetchFn }).createPipeline({ name: 'x', queryYaml: 'y' }) + expect((calls[0].init?.headers as Record).authorization).toBe('Bearer sk-test') + }) + + it('omits the auth header when no api key is configured', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'x' } })) + await new DeepsetManagementClient({ ...config, haystackApiKey: '' }, { fetchFn }).createPipeline({ + name: 'x', + queryYaml: 'y', + }) + expect((calls[0].init?.headers as Record).authorization).toBeUndefined() + }) + + it('deploys and returns the parsed pipeline', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 200, body: pipeline })) + const client = new DeepsetManagementClient(config, { fetchFn }) + const result = await client.deployPipeline('tb-my-agent') + expect(calls[0].url).toBe(`${wsBase}/pipelines/tb-my-agent/deploy`) + expect(result.status).toBe('DEPLOYMENT_IN_PROGRESS') + expect(result.pipeline_id).toBe(pipeline.pipeline_id) + }) + + it('gets pipeline status', async () => { + const { fetchFn } = makeFetch(() => ({ status: 200, body: { ...pipeline, status: 'DEPLOYED' } })) + const result = await new DeepsetManagementClient(config, { fetchFn }).getPipeline('tb-my-agent') + expect(result.status).toBe('DEPLOYED') + }) + + it('lists pipelines from the data envelope', async () => { + const { fetchFn, calls } = makeFetch(() => ({ + status: 200, + body: { data: [pipeline, { ...pipeline, name: 'b' }] }, + })) + const result = await new DeepsetManagementClient(config, { fetchFn }).listPipelines() + expect(calls[0].url).toBe(`${wsBase}/pipelines?limit=100`) + expect(result.map((p) => p.name)).toEqual(['tb-my-agent', 'b']) + }) + + it('fetches the pipeline query_yaml', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 200, body: { query_yaml: 'components:\n agent: {}' } })) + const yaml = await new DeepsetManagementClient(config, { fetchFn }).getPipelineYaml('Rai-RAG-Research-Agent') + expect(calls[0].url).toBe(`${wsBase}/pipelines/Rai-RAG-Research-Agent/yaml`) + expect(yaml).toContain('components:') + }) + + it('undeploys and deletes with the right methods', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 200, text: '' })) + const client = new DeepsetManagementClient(config, { fetchFn }) + await client.undeployPipeline('tb-my-agent') + await client.deletePipeline('tb-my-agent') + expect(calls[0].url).toBe(`${wsBase}/pipelines/tb-my-agent/undeploy`) + expect(calls[0].init?.method).toBe('POST') + expect(calls[1].url).toBe(`${wsBase}/pipelines/tb-my-agent`) + expect(calls[1].init?.method).toBe('DELETE') + }) + + it('throws DeepsetManagementError on a non-2xx response', async () => { + const { fetchFn } = makeFetch(() => ({ status: 422, text: 'invalid yaml' })) + const client = new DeepsetManagementClient(config, { fetchFn }) + const error = await client.deployPipeline('bad').catch((e) => e) + expect(error).toBeInstanceOf(DeepsetManagementError) + expect(error.status).toBe(422) + expect(error.message).toContain('invalid yaml') + }) +}) diff --git a/backend/src/haystack/management-client.ts b/backend/src/haystack/management-client.ts new file mode 100644 index 000000000..f74be02a6 --- /dev/null +++ b/backend/src/haystack/management-client.ts @@ -0,0 +1,147 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Deepset Cloud pipeline management (create / deploy / status / list), the + * lifecycle counterpart to the runtime `HaystackAcpServer`. Deepset ships no + * JS SDK and no public OpenAPI, so this is raw `fetch` + zod against the + * spike-confirmed REST shapes (THU-743): + * + * create POST /pipelines {name, query_yaml, deepset_cloud_version:'v2'} → 201 {name} + * deploy POST /pipelines/:name/deploy → 200 pipeline + * status GET /pipelines/:name → 200 pipeline + * list GET /pipelines?limit=N → 200 {data:[…]} + * yaml GET /pipelines/:name/yaml → 200 {query_yaml} + * + * Deepset `PipelineStatus` values: `DEPLOYMENT_IN_PROGRESS`, `ACTIVATING` + * (transient), `DEPLOYED`, `IDLE` (deployed but auto-slept; wakes on query), + * `DEPLOYMENT_FAILED`, `UNDEPLOYED`, `UNDEPLOYMENT_IN_PROGRESS`. All URLs are + * workspace-scoped. + */ + +import type { Settings } from '@/config/settings' +import { z } from 'zod' + +/** The Deepset config this client needs — a subset of {@link Settings}. */ +export type HaystackManagementConfig = Pick + +export type HaystackManagementDeps = { + /** Injectable for tests; defaults to the global `fetch`. */ + fetchFn?: typeof fetch +} + +/** A non-2xx response from Deepset, carrying the status and body for diagnosis. */ +export class DeepsetManagementError extends Error { + constructor( + readonly status: number, + readonly statusText: string, + readonly body: string, + ) { + super(`deepset management ${status} ${statusText}: ${body}`) + this.name = 'DeepsetManagementError' + } +} + +/** A Deepset pipeline as returned by deploy / status / list. Extra fields are + * tolerated (stripped) so Deepset can evolve its payload without breaking us. */ +export const deepsetPipelineSchema = z.object({ + name: z.string(), + pipeline_id: z.string(), + status: z.string(), + desired_status: z.string().nullable().optional(), + supports_prompt: z.boolean().optional(), + output_type: z.string().nullable().optional(), +}) +export type DeepsetPipeline = z.infer + +const listResponseSchema = z.object({ data: z.array(deepsetPipelineSchema) }) +const yamlResponseSchema = z.object({ query_yaml: z.string() }) + +export class DeepsetManagementClient { + private readonly config: HaystackManagementConfig + private readonly fetchFn: typeof fetch + + constructor(config: HaystackManagementConfig, deps: HaystackManagementDeps = {}) { + this.config = config + this.fetchFn = deps.fetchFn ?? globalThis.fetch + } + + /** Create a pipeline. `dryRun` validates the YAML without persisting. Returns + * nothing — the 201 body is just `{name}`. */ + async createPipeline(input: { name: string; queryYaml: string; dryRun?: boolean }): Promise { + const query = input.dryRun ? '?dry_run=true' : '' + const res = await this.send(`${this.pipelinesUrl()}${query}`, { + method: 'POST', + headers: this.jsonHeaders(), + body: JSON.stringify({ name: input.name, query_yaml: input.queryYaml, deepset_cloud_version: 'v2' }), + }) + await res.text() + } + + /** Deploy (activate) a created pipeline. Returns the pipeline with its (in-progress) status. */ + async deployPipeline(name: string): Promise { + const res = await this.send(`${this.pipelineUrl(name)}/deploy`, { method: 'POST', headers: this.jsonHeaders() }) + return deepsetPipelineSchema.parse(await res.json()) + } + + /** Live status of a single pipeline. */ + async getPipeline(name: string): Promise { + const res = await this.send(this.pipelineUrl(name), { headers: this.jsonHeaders() }) + return deepsetPipelineSchema.parse(await res.json()) + } + + /** All pipelines in the workspace. */ + async listPipelines(limit = 100): Promise { + const res = await this.send(`${this.pipelinesUrl()}?limit=${limit}`, { headers: this.jsonHeaders() }) + return listResponseSchema.parse(await res.json()).data + } + + /** The `query_yaml` of an existing pipeline — used to clone a curated template. */ + async getPipelineYaml(name: string): Promise { + const res = await this.send(`${this.pipelineUrl(name)}/yaml`, { headers: this.jsonHeaders() }) + return yamlResponseSchema.parse(await res.json()).query_yaml + } + + /** Stop a running pipeline (teardown step before delete). */ + async undeployPipeline(name: string): Promise { + const res = await this.send(`${this.pipelineUrl(name)}/undeploy`, { method: 'POST', headers: this.jsonHeaders() }) + await res.text() + } + + /** Permanently remove a pipeline. */ + async deletePipeline(name: string): Promise { + const res = await this.send(this.pipelineUrl(name), { method: 'DELETE', headers: this.jsonHeaders() }) + await res.text() + } + + private async send(url: string, init: RequestInit): Promise { + const res = await this.fetchFn(url, init) + if (!res.ok) { + const body = await res.text().catch(() => '') + throw new DeepsetManagementError(res.status, res.statusText, body) + } + return res + } + + private workspaceBaseUrl(): string { + const base = this.config.haystackBaseUrl.replace(/\/$/, '') + return `${base}/api/v1/workspaces/${this.config.haystackWorkspace}` + } + + private pipelinesUrl(): string { + return `${this.workspaceBaseUrl()}/pipelines` + } + + private pipelineUrl(name: string): string { + return `${this.pipelinesUrl()}/${encodeURIComponent(name)}` + } + + private authHeaders(): Record { + return this.config.haystackApiKey ? { authorization: `Bearer ${this.config.haystackApiKey}` } : {} + } + + private jsonHeaders(): Record { + return { 'content-type': 'application/json', accept: 'application/json', ...this.authHeaders() } + } +} diff --git a/backend/src/haystack/provider.test.ts b/backend/src/haystack/provider.test.ts new file mode 100644 index 000000000..def0b36ca --- /dev/null +++ b/backend/src/haystack/provider.test.ts @@ -0,0 +1,215 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { createTestSettings } from '@/test-utils/settings' +import type { ProviderContext } from '@/agents' +import { createHaystackProvider, resolveHaystackPipeline } from './provider' + +const deployableSettings = () => + createTestSettings({ + haystackBaseUrl: 'https://api.cloud.deepset.ai', + haystackApiKey: 'sk-test', + haystackWorkspace: 'tutorial', + haystackTemplatePipeline: 'Template-Pipeline', + }) + +const makeContext = (settings = deployableSettings()): ProviderContext => ({ + request: new Request('http://localhost:8000/v1/agents/deploy'), + settings, + userId: 'user-1', +}) + +type Route = (url: string, init?: RequestInit) => { status?: number; body?: unknown } | undefined + +const routedFetch = (routes: Route) => { + const calls: { url: string; init?: RequestInit }[] = [] + const fetchFn = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + calls.push({ url, init }) + const result = routes(url, init) ?? { status: 404, body: { error: 'no route' } } + const status = result.status ?? 200 + return new Response(result.body !== undefined ? JSON.stringify(result.body) : '', { + status, + statusText: status >= 400 ? 'Error' : 'OK', + }) + }) as typeof fetch + return { fetchFn, calls } +} + +describe('haystack provider — catalog', () => { + it('offers the descriptor when deploy is configured', () => { + const provider = createHaystackProvider() + const catalog = provider.catalog!(makeContext()) + expect(catalog).toHaveLength(1) + expect(catalog[0].id).toBe('haystack') + expect(catalog[0].provider).toBe('haystack') + }) + + it('offers nothing when the template pipeline is unset', () => { + const provider = createHaystackProvider() + const catalog = provider.catalog!(makeContext(createTestSettings({ haystackTemplatePipeline: '' }))) + expect(catalog).toEqual([]) + }) +}) + +describe('haystack provider — deploy', () => { + it('clones the template yaml, creates under tb- namespace, and deploys', async () => { + const { fetchFn, calls } = routedFetch((url, init) => { + if (url.endsWith('/pipelines/Template-Pipeline/yaml')) { + return { status: 200, body: { query_yaml: 'components:\n agent: {}' } } + } + if (init?.method === 'POST' && url.endsWith('/pipelines')) { + return { status: 201, body: { name: 'created' } } + } + if (init?.method === 'POST' && url.endsWith('/deploy')) { + return { status: 200, body: { name: 'ref', pipeline_id: 'pid-1', status: 'DEPLOYMENT_IN_PROGRESS' } } + } + return undefined + }) + const provider = createHaystackProvider({ fetchFn }) + const result = await provider.deploy!({ name: 'My Agent' }, makeContext()) + + expect(result.deploymentId.startsWith('haystack:tb-my-agent-')).toBe(true) + expect(result.status).toBe('pending') + // Deploy returns the deterministic chat endpoint straight away. + expect(result.connection?.transport).toBe('websocket') + expect(result.connection?.url).toContain('haystack/ws?pipeline=tb-my-agent-') + // The create body carried the cloned template YAML. + const createCall = calls.find((c) => c.init?.method === 'POST' && c.url.endsWith('/pipelines')) + expect(JSON.parse(String(createCall?.init?.body)).query_yaml).toContain('agent: {}') + }) +}) + +describe('haystack provider — status', () => { + const statusProvider = (deepsetStatus: string) => { + const { fetchFn } = routedFetch(() => ({ + status: 200, + body: { name: 'tb-x', pipeline_id: 'pid-1', status: deepsetStatus }, + })) + return createHaystackProvider({ fetchFn }) + } + + it('maps DEPLOYED to running with a websocket connection', async () => { + const result = await statusProvider('DEPLOYED').status!('tb-my-agent-abc', makeContext()) + expect(result.status).toBe('running') + expect(result.connection?.transport).toBe('websocket') + expect(result.connection?.url).toContain('haystack/ws?pipeline=tb-my-agent-abc') + expect(result.deploymentId).toBe('haystack:tb-my-agent-abc') + }) + + it('maps an in-progress status to pending with no connection', async () => { + const result = await statusProvider('DEPLOYMENT_IN_PROGRESS').status!('tb-x', makeContext()) + expect(result.status).toBe('pending') + expect(result.connection).toBeNull() + }) + + it('maps ACTIVATING (waking from idle) to pending', async () => { + const result = await statusProvider('ACTIVATING').status!('tb-x', makeContext()) + expect(result.status).toBe('pending') + expect(result.connection).toBeNull() + }) + + it('maps IDLE (auto-slept but usable) to running with a connection', async () => { + const result = await statusProvider('IDLE').status!('tb-my-agent-abc', makeContext()) + expect(result.status).toBe('running') + expect(result.connection?.url).toContain('haystack/ws?pipeline=tb-my-agent-abc') + }) + + it('maps a failure status to failed', async () => { + const result = await statusProvider('DEPLOYMENT_FAILED').status!('tb-x', makeContext()) + expect(result.status).toBe('failed') + }) + + it('maps an undeployed pipeline to gone with no connection', async () => { + const result = await statusProvider('UNDEPLOYED').status!('tb-x', makeContext()) + expect(result.status).toBe('gone') + expect(result.connection).toBeNull() + }) + + it('reports gone (never throws) when the pipeline no longer exists', async () => { + const { fetchFn } = routedFetch(() => ({ status: 404, body: { error: 'not found' } })) + const result = await createHaystackProvider({ fetchFn }).status!('tb-x', makeContext()) + expect(result.status).toBe('gone') + expect(result.connection).toBeNull() + }) +}) + +describe('haystack provider — undeploy', () => { + it('deletes the cloned tb- pipeline and reports gone', async () => { + const { fetchFn, calls } = routedFetch((url, init) => + init?.method === 'DELETE' && url.endsWith('/pipelines/tb-my-agent-abc') ? { status: 204 } : undefined, + ) + const result = await createHaystackProvider({ fetchFn }).undeploy!('tb-my-agent-abc', makeContext()) + expect(result).toEqual({ deploymentId: 'haystack:tb-my-agent-abc', status: 'gone' }) + expect(calls.some((c) => c.init?.method === 'DELETE' && c.url.endsWith('/pipelines/tb-my-agent-abc'))).toBe(true) + }) + + it('treats an already-deleted (404) pipeline as success', async () => { + const { fetchFn } = routedFetch(() => ({ status: 404, body: { error: 'not found' } })) + const result = await createHaystackProvider({ fetchFn }).undeploy!('tb-x', makeContext()) + expect(result.status).toBe('gone') + }) + + it('never deletes a non-tb pipeline (template / shared) — no-op success', async () => { + const { fetchFn, calls } = routedFetch(() => ({ status: 200 })) + const result = await createHaystackProvider({ fetchFn }).undeploy!('Template-Pipeline', makeContext()) + expect(result.status).toBe('gone') + expect(calls).toEqual([]) + }) +}) + +describe('haystack provider — list', () => { + const request = new Request('http://localhost:8000/v1/agents') + const listBody = { + data: [ + { name: 'RAG', pipeline_id: 'p1', status: 'DEPLOYED', desired_status: 'DEPLOYED', supports_prompt: true }, + // Auto-idled but intended-deployed → still a usable agent (wakes on query). + { name: 'Napping', pipeline_id: 'p2', status: 'IDLE', desired_status: 'DEPLOYED', supports_prompt: true }, + // tb-* deploys are excluded — they live in the synced agents table, not /agents. + { name: 'tb-mine-x', pipeline_id: 'p3', status: 'DEPLOYED', desired_status: 'DEPLOYED', supports_prompt: true }, + { name: 'Undeployed', pipeline_id: 'p4', status: 'IDLE', desired_status: 'UNDEPLOYED', supports_prompt: true }, + { name: 'Indexer', pipeline_id: 'p5', status: 'DEPLOYED', desired_status: 'DEPLOYED', supports_prompt: false }, + ], + } + + it('returns [] when Haystack is not configured', async () => { + expect(await createHaystackProvider().list(request, createTestSettings())).toEqual([]) + }) + + it('maps intended-deployed, prompt-capable pipelines (including idle, excluding tb-)', async () => { + const { fetchFn, calls } = routedFetch(() => ({ status: 200, body: listBody })) + const list = await createHaystackProvider({ fetchFn }).list(request, deployableSettings()) + expect(list.map((a) => a.id)).toEqual(['RAG', 'Napping']) + expect(list[0]).toMatchObject({ name: 'RAG', type: 'managed-acp', transport: 'websocket', isSystem: 1 }) + expect(list[0].url).toContain('haystack/ws?pipeline=RAG') + expect(calls[0].url).toContain('/pipelines?limit=') + }) + + it('returns [] (never throws) when the host errors', async () => { + const { fetchFn } = routedFetch(() => ({ status: 500 })) + expect(await createHaystackProvider({ fetchFn }).list(request, deployableSettings())).toEqual([]) + }) +}) + +describe('resolveHaystackPipeline', () => { + it('resolves a pipeline by fetching its pipeline_id live', async () => { + const { fetchFn, calls } = routedFetch(() => ({ + status: 200, + body: { name: 'RAG', pipeline_id: 'pid-live', status: 'DEPLOYED' }, + })) + const resolved = await resolveHaystackPipeline('RAG', deployableSettings(), { fetchFn }) + expect(resolved).toEqual({ pipelineId: 'pid-live', pipelineName: 'RAG', supportsFiles: false }) + expect(calls[0].url).toContain('/pipelines/RAG') + }) + + it('returns null when Haystack is not configured', async () => { + expect(await resolveHaystackPipeline('RAG', createTestSettings())).toBeNull() + }) + + it('returns null (not throw) when the host lookup fails', async () => { + const { fetchFn } = routedFetch(() => ({ status: 404, body: { error: 'gone' } })) + expect(await resolveHaystackPipeline('missing', deployableSettings(), { fetchFn })).toBeNull() + }) +}) diff --git a/backend/src/haystack/provider.ts b/backend/src/haystack/provider.ts index a09735dab..a95247dda 100644 --- a/backend/src/haystack/provider.ts +++ b/backend/src/haystack/provider.ts @@ -2,12 +2,21 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { AgentProvider } from '@/agents' -import { buildWebSocketUrl } from '@/agents' +import type { AgentProvider, ProviderContext } from '@/agents' +import { buildWebSocketUrl, encodeDeploymentId } from '@/agents' import { createStandaloneLogger } from '@/config/logger' import type { Settings } from '@/config/settings' import type { RemoteAgentDescriptor } from '@shared/acp-types' -import { haystackPipelinesEnvSchema } from './types' +import type { + AgentConnection, + AgentDescriptor, + AgentSpec, + DeploymentStatusResponse, + DeployResponse, + DeployStatus, + UndeployResponse, +} from '@shared/agent-descriptors' +import { DeepsetManagementClient, DeepsetManagementError } from './management-client' /** * Provider id registered into the agent discovery registry. The string is @@ -16,66 +25,236 @@ import { haystackPipelinesEnvSchema } from './types' */ export const haystackProviderId = 'haystack' +/** Injectable dependencies for the provider (test seam for the management client's fetch). */ +export type HaystackProviderDeps = { + fetchFn?: typeof fetch +} + +/** Version of {@link haystackDescriptor}; bump when its fields change. */ +const haystackSchemaVersion = 1 + /** - * Build the Haystack provider. Reads `HAYSTACK_PIPELINES` (JSON array) from - * the injected `settings`. An empty / missing / malformed value yields an - * empty descriptor list — we log and skip rather than throw so a deployment - * with no Haystack config doesn't fail other providers. - * - * Each pipeline becomes a `managed-acp`, websocket-transport descriptor whose - * URL points at `/v1/haystack/ws?pipeline=`. The host is derived - * from the inbound `Request` via {@link buildWebSocketUrl} so dev (`ws://`) - * and prod (`wss://` behind a reverse proxy) both produce correct URLs - * without env-var pinning. + * The curated "Add agent" form for Haystack. Phase 1 is curated mode: the owner + * fixes the pipeline template (`HAYSTACK_TEMPLATE_PIPELINE`), so the user only + * names the agent. BYO fields (model/key/index) hang off this same descriptor later. */ -export const createHaystackProvider = (): AgentProvider => ({ +const haystackDescriptor: AgentDescriptor = { id: haystackProviderId, - list: (request: Request, settings: Settings): RemoteAgentDescriptor[] => { - const pipelines = parsePipelinesEnv(settings) - if (pipelines.length === 0) { - return [] - } - return pipelines.map((pipeline) => ({ - id: pipeline.id, - name: pipeline.name, - type: 'managed-acp', - transport: 'websocket', - // URL carries the public slug — the WS route resolves it back to the - // Deepset pipelineName / pipelineId from the same env-driven descriptor. - url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(pipeline.id)}`), - description: pipeline.description ?? null, - icon: pipeline.icon ?? null, - isSystem: 1, - })) - }, -}) + provider: haystackProviderId, + name: 'Haystack RAG agent', + description: 'Deploy a Haystack pipeline as a chat agent.', + icon: 'file-search', + schemaVersion: haystackSchemaVersion, + action: 'deploy', + steps: [ + { + id: 'basics', + title: 'Name your agent', + fields: [ + { + key: 'name', + label: 'Name', + widget: 'text', + required: true, + maxLength: 60, + placeholder: 'My research agent', + }, + ], + }, + ], +} + +/** Whether Haystack is configured to talk to Deepset at all (discovery + chat). */ +const isHaystackConfigured = (settings: Settings): boolean => + Boolean(settings.haystackBaseUrl && settings.haystackApiKey && settings.haystackWorkspace) + +/** Whether Haystack is also deployable (needs a template pipeline to clone). */ +const isDeployConfigured = (settings: Settings): boolean => + isHaystackConfigured(settings) && Boolean(settings.haystackTemplatePipeline) + +/** + * Map a Deepset pipeline status onto our normalized deploy lifecycle. Deepset + * `PipelineStatus`: DEPLOYMENT_IN_PROGRESS, ACTIVATING (transient → `pending`); + * DEPLOYED and IDLE (both usable — an auto-idled pipeline wakes on query → + * `running`); DEPLOYMENT_FAILED (`failed`); UNDEPLOYED, UNDEPLOYMENT_IN_PROGRESS + * (unusable → `gone`, alongside a not-found host lookup). + */ +const mapStatus = (deepsetStatus: string): DeployStatus => { + const status = deepsetStatus.toUpperCase() + if (status === 'DEPLOYMENT_IN_PROGRESS' || status === 'ACTIVATING') { + return 'pending' + } + if (status === 'DEPLOYED' || status === 'IDLE') { + return 'running' + } + if (status.includes('FAIL')) { + return 'failed' + } + return 'gone' +} + +/** Derive a Deepset-safe pipeline name from a user-chosen display name. */ +const toPipelineRef = (name: string): string => { + const slug = + name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 40) || 'agent' + return `tb-${slug}-${Date.now().toString(36)}` +} + +/** Construct a management client from settings (shared by the provider verbs and the WS resolver). */ +const makeManagementClient = (settings: Settings, deps: HaystackProviderDeps): DeepsetManagementClient => + new DeepsetManagementClient( + { + haystackBaseUrl: settings.haystackBaseUrl, + haystackApiKey: settings.haystackApiKey, + haystackWorkspace: settings.haystackWorkspace, + }, + { fetchFn: deps.fetchFn }, + ) + +/** A pipeline resolved for a live chat connection. */ +export type ResolvedPipeline = { pipelineId: string; pipelineName: string; supportsFiles: boolean } /** - * Parse `HAYSTACK_PIPELINES` from settings. The env var is a JSON-encoded - * array of {@link haystackPipelinesEnvSchema} entries. Empty / missing values - * return `[]`. A malformed value also returns `[]` but is logged at WARN — - * silent dropping would hide a deployment-side typo, but throwing would - * cascade into a `GET /agents` 500 for unrelated providers (the discovery - * route catches the throw, but the operator wouldn't get a structured signal). + * Resolve a `?pipeline=` slug to its Deepset identifiers for the WS route by + * looking it up live (the slug is the Deepset pipeline name). Returns null for + * unknown / unconfigured slugs so the caller can close the socket. Pipelines are + * treated as text-only for now (`supportsFiles: false`). */ -export const parsePipelinesEnv = (settings: Settings) => { - const raw = settings.haystackPipelines.trim() - if (raw.length === 0) { - return [] +export const resolveHaystackPipeline = async ( + slug: string, + settings: Settings, + deps: HaystackProviderDeps = {}, +): Promise => { + if (!isHaystackConfigured(settings)) { + return null } - let parsedJson: unknown try { - parsedJson = JSON.parse(raw) - } catch (err) { - const log = createStandaloneLogger(settings) - log.warn({ err }, 'HAYSTACK_PIPELINES is not valid JSON; ignoring') - return [] + const pipeline = await makeManagementClient(settings, deps).getPipeline(slug) + return { pipelineId: pipeline.pipeline_id, pipelineName: slug, supportsFiles: false } + } catch { + return null } - const result = haystackPipelinesEnvSchema.safeParse(parsedJson) - if (!result.success) { - const log = createStandaloneLogger(settings) - log.warn({ issues: result.error.issues }, 'HAYSTACK_PIPELINES schema mismatch; ignoring') - return [] +} + +/** + * Build the Haystack provider. `list()` fetches the workspace's pipelines live + * from Deepset — `DEPLOYED`, prompt-capable, and excluding Thunderbolt-deployed + * `tb-*` instances (those are user-owned and live in the synced `agents` table). + * On any host error it logs and returns `[]` so discovery never fails. + * + * Each pipeline becomes a `managed-acp`, websocket descriptor whose URL points at + * `/v1/haystack/ws?pipeline=`; the host is derived from the inbound request + * via {@link buildWebSocketUrl} so dev (`ws://`) and prod (`wss://`) both work. + */ +export const createHaystackProvider = (deps: HaystackProviderDeps = {}): AgentProvider => { + const managementClient = (settings: Settings) => makeManagementClient(settings, deps) + + return { + id: haystackProviderId, + list: async (request: Request, settings: Settings): Promise => { + if (!isHaystackConfigured(settings)) { + return [] + } + try { + const pipelines = await managementClient(settings).listPipelines() + return pipelines + .filter( + // Deepset auto-idles pipelines (status flips DEPLOYED→IDLE), but they + // wake on query — so key off the intended `desired_status`, not the + // transient runtime status. Non-prompt pipelines (indexes) aren't chat + // agents. Exclude our own `tb-*` deploys: those are user-owned and live + // in the synced agents table, so listing them here too would double them. + (p) => + (p.desired_status ?? p.status) === 'DEPLOYED' && p.supports_prompt !== false && !p.name.startsWith('tb-'), + ) + .map((p) => ({ + id: p.name, + name: p.name, + type: 'managed-acp', + transport: 'websocket', + url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(p.name)}`), + description: null, + icon: null, + isSystem: 1, + })) + } catch (err) { + createStandaloneLogger(settings).warn({ err }, 'haystack list pipelines failed; returning empty') + return [] + } + }, + catalog: ({ settings }: ProviderContext): AgentDescriptor[] => + isDeployConfigured(settings) ? [haystackDescriptor] : [], + deploy: async (spec: AgentSpec, { request, settings }: ProviderContext): Promise => { + const name = typeof spec.name === 'string' ? spec.name : '' + const ref = toPipelineRef(name) + const client = managementClient(settings) + // Clone the owner-curated template's YAML, create under our `tb-` namespace, then deploy. + const queryYaml = await client.getPipelineYaml(settings.haystackTemplatePipeline) + await client.createPipeline({ name: ref, queryYaml }) + const deployed = await client.deployPipeline(ref) + // The chat endpoint is deterministic from the ref, so return it now — the + // client persists the agent immediately without waiting for it to spin up. + const connection: AgentConnection = { + url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(ref)}`), + transport: 'websocket', + } + return { + deploymentId: encodeDeploymentId(haystackProviderId, ref), + status: mapStatus(deployed.status), + connection, + } + }, + status: async (ref: string, { request, settings }: ProviderContext): Promise => { + const deploymentId = encodeDeploymentId(haystackProviderId, ref) + // A not-found pipeline was deleted/undeployed on the host — report `gone` + // (a terminal state the client can warn on) rather than throwing. + const pipeline = await managementClient(settings) + .getPipeline(ref) + .catch((err) => { + if (err instanceof DeepsetManagementError && err.status === 404) { + return null + } + throw err + }) + if (!pipeline) { + return { deploymentId, status: 'gone', detail: 'not found', connection: null } + } + const status = mapStatus(pipeline.status) + // A usable (running, incl. auto-idled) pipeline carries the chat endpoint; + // pending/failed/gone don't. + const connection: AgentConnection | null = + status === 'running' + ? { + url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(ref)}`), + transport: 'websocket', + } + : null + return { deploymentId, status, detail: pipeline.status, connection } + }, + undeploy: async (ref: string, { settings }: ProviderContext): Promise => { + const deploymentId = encodeDeploymentId(haystackProviderId, ref) + // Only tear down our own `tb-*` deploy clones — never the owner-curated + // template or other workspace pipelines. A non-`tb-` ref is a no-op so the + // client can still drop its local row. + if (!ref.startsWith('tb-')) { + return { deploymentId, status: 'gone' } + } + // Delete removes the cloned pipeline entirely (the whole teardown for a + // user-deleted agent). A 404 means it's already gone — the goal state — so + // treat it as success rather than surfacing an error. + await managementClient(settings) + .deletePipeline(ref) + .catch((err) => { + if (err instanceof DeepsetManagementError && err.status === 404) { + return + } + throw err + }) + return { deploymentId, status: 'gone' } + }, } - return result.data } diff --git a/backend/src/haystack/routes.test.ts b/backend/src/haystack/routes.test.ts index 35f11a27e..9ad2e5d14 100644 --- a/backend/src/haystack/routes.test.ts +++ b/backend/src/haystack/routes.test.ts @@ -99,6 +99,34 @@ const bearerProtocols = (bearerToken: string): string[] => [ `thunderbolt.bearer.${encodeWsBearer(bearerToken)}`, ] +/** + * Fake Deepset for the WS resolver (`resolveHaystackPipeline` → `GET /pipelines/:name`): + * `rag` resolves to a deployed pipeline; everything else 404s → the route closes 4001. + * Injected as `createApp`'s `fetchFn`, so it only affects the live slug lookup. + */ +const haystackFetch = (async (input: RequestInfo | URL) => { + const url = String(input) + if (/\/pipelines\/rag$/.test(url)) { + return new Response(JSON.stringify({ name: 'rag', pipeline_id: 'pipe-uuid', status: 'DEPLOYED' }), { status: 200 }) + } + return new Response(JSON.stringify({ error: 'not found' }), { status: 404, statusText: 'Not Found' }) +}) as typeof fetch + +/** + * Same as {@link haystackFetch} but the slug lookup is deliberately slow, so the + * server-side async `open()` is still resolving when the client's first frame + * lands — the production condition (a real Deepset lookup takes tens of ms) that + * a synchronous fake hides. Exercises the early-frame buffering. + */ +const slowHaystackFetch = (async (input: RequestInfo | URL) => { + const url = String(input) + if (/\/pipelines\/rag$/.test(url)) { + await new Promise((resolve) => setTimeout(resolve, 150)) + return new Response(JSON.stringify({ name: 'rag', pipeline_id: 'pipe-uuid', status: 'DEPLOYED' }), { status: 200 }) + } + return new Response(JSON.stringify({ error: 'not found' }), { status: 404, statusText: 'Not Found' }) +}) as typeof fetch + describe('WS /v1/haystack/ws — auth gating', () => { const cleanups: Array<() => Promise> = [] // Isolated PGlite instance for the real-`.listen()` upgrades in this suite: @@ -106,12 +134,12 @@ describe('WS /v1/haystack/ws — auth gating', () => { // singleton (head-of-line blocking under CI starvation; the 5s `initialize` // timeout was the canary). Closed once in afterAll to avoid exit-99. let iso: IsolatedTestDb - // Real settings come from `getSettings()` which reads `process.env`. Inject a - // valid pipelines config so the route's slug→descriptor lookup resolves; - // otherwise even authenticated upgrades would close 4001 with "unknown - // pipeline" before initialize can run. - const originalPipelinesEnv = process.env.HAYSTACK_PIPELINES + // Real settings come from `getSettings()` which reads `process.env`. Point + // Haystack at a fake host so it's "configured"; the route's live slug→pipeline + // lookup is served by the injected `haystackFetch` (otherwise even + // authenticated upgrades would close 4001 with "unknown pipeline"). const originalBaseUrlEnv = process.env.HAYSTACK_BASE_URL + const originalApiKeyEnv = process.env.HAYSTACK_API_KEY const originalWorkspaceEnv = process.env.HAYSTACK_WORKSPACE beforeAll(async () => { @@ -120,10 +148,8 @@ describe('WS /v1/haystack/ws — auth gating', () => { beforeEach(() => { process.env.HAYSTACK_BASE_URL = 'https://haystack.test' + process.env.HAYSTACK_API_KEY = 'sk-test' process.env.HAYSTACK_WORKSPACE = 'ws-test' - process.env.HAYSTACK_PIPELINES = JSON.stringify([ - { id: 'rag', name: 'RAG', pipelineName: 'rag-pipeline', pipelineId: 'pipe-uuid' }, - ]) clearSettingsCache() }) @@ -131,16 +157,16 @@ describe('WS /v1/haystack/ws — auth gating', () => { for (const cleanup of cleanups.splice(0)) { await cleanup() } - if (originalPipelinesEnv === undefined) { - delete process.env.HAYSTACK_PIPELINES - } else { - process.env.HAYSTACK_PIPELINES = originalPipelinesEnv - } if (originalBaseUrlEnv === undefined) { delete process.env.HAYSTACK_BASE_URL } else { process.env.HAYSTACK_BASE_URL = originalBaseUrlEnv } + if (originalApiKeyEnv === undefined) { + delete process.env.HAYSTACK_API_KEY + } else { + process.env.HAYSTACK_API_KEY = originalApiKeyEnv + } if (originalWorkspaceEnv === undefined) { delete process.env.HAYSTACK_WORKSPACE } else { @@ -232,7 +258,7 @@ describe('WS /v1/haystack/ws — auth gating', () => { }) it('opens the socket and answers initialize when a valid bearer is offered', async () => { - const handle = await createTestApp({ database: iso.db }) + const handle = await createTestApp({ database: iso.db, fetchFn: haystackFetch }) const port = await startApp(handle.app as unknown as RunningApp) cleanups.push(async () => { await stopApp(handle.app as unknown as RunningApp) @@ -267,15 +293,49 @@ describe('WS /v1/haystack/ws — auth gating', () => { client.close() }, 15000) - it('closes 4001 when the ?pipeline= slug is not in HAYSTACK_PIPELINES', async () => { - const handle = await createTestApp({ database: iso.db }) + it('answers an initialize sent immediately, even while open() is still resolving', async () => { + // The production race: the FE sends `initialize` the instant the socket opens, + // while the server's async open() is still doing the auth + (slow) pipeline + // lookup. Without buffering that frame is dropped and the handshake hangs. + const handle = await createTestApp({ database: iso.db, fetchFn: slowHaystackFetch }) + const port = await startApp(handle.app as unknown as RunningApp) + cleanups.push(async () => { + await stopApp(handle.app as unknown as RunningApp) + await handle.cleanup() + }) + + const client = new WebSocket( + `ws://127.0.0.1:${port}/v1/haystack/ws?pipeline=rag`, + bearerProtocols(handle.bearerToken), + ) + // Bind the reply listener up front, then fire initialize the moment the + // socket opens — before the slow server-side open() can finish. + const replyPromise = new Promise((resolve, reject) => { + client.addEventListener('message', (event: MessageEvent) => + resolve(typeof event.data === 'string' ? event.data : ''), + ) + client.addEventListener('close', () => reject(new Error('closed before reply'))) + client.addEventListener('error', () => reject(new Error('errored before reply'))) + }) + client.addEventListener('open', () => { + client.send(JSON.stringify({ jsonrpc: '2.0', id: 7, method: 'initialize' })) + }) + + const reply = JSON.parse(await replyPromise) + expect(reply.id).toBe(7) + expect(reply.result.protocolVersion).toBeGreaterThan(0) + client.close() + }, 15000) + + it('closes 4001 when the ?pipeline= slug is not a known pipeline', async () => { + const handle = await createTestApp({ database: iso.db, fetchFn: haystackFetch }) const port = await startApp(handle.app as unknown as RunningApp) cleanups.push(async () => { await stopApp(handle.app as unknown as RunningApp) await handle.cleanup() }) - // `rag` is configured in beforeEach; `nope` is not — the route should reject + // `rag` resolves via the fake host; `nope` 404s — the route should reject // even with a valid bearer. const client = new WebSocket( `ws://127.0.0.1:${port}/v1/haystack/ws?pipeline=nope`, diff --git a/backend/src/haystack/routes.ts b/backend/src/haystack/routes.ts index 46debb381..7aeca3eb6 100644 --- a/backend/src/haystack/routes.ts +++ b/backend/src/haystack/routes.ts @@ -12,7 +12,19 @@ import type { User } from '@shared/types/auth' import { wsCarrierSubprotocol } from '@shared/ws-bearer' import { Elysia, t } from 'elysia' import { HaystackAcpServer, type HaystackAcpDeps } from './acp-server' -import { createHaystackProvider, parsePipelinesEnv } from './provider' +import { createHaystackProvider, resolveHaystackPipeline } from './provider' + +/** + * Per-connection state stashed on `ws.data`. `server` is created once the async + * `open()` finishes; `pending` buffers frames that arrive during that window so + * the FE's eager `initialize` isn't dropped (see `open`). `request` is Bun's + * original upgrade request. + */ +type HaystackWsSlot = { + request?: Request + server?: HaystackAcpServer + pending?: string[] +} /** * Mount the Haystack ACP adapter routes. @@ -121,32 +133,41 @@ export const createHaystackRoutes = (settings: Settings, auth: Auth, deps?: Hays }, async open(ws) { const log = createStandaloneLogger(settings) - const data = ws.data as unknown as { request?: Request } + const slot = ws.data as unknown as HaystackWsSlot + + // Bun delivers `message` frames WITHOUT awaiting this async handler, so + // the FE's first ACP frame (`initialize`, sent the instant the socket + // opens) can arrive while the auth + live pipeline lookup below is still + // in flight. Buffer anything that lands during that window and drain it + // once the server exists, so no frame is silently dropped (that stall was + // the whole bug: the ACP handshake never completed and chat hung). + slot.pending = [] // Validate the bearer exactly once per accepted socket. Doing this in // `beforeHandle` instead is unsafe because Bun's adapter can call it // more than once per upgrade. The bearer rides a subprotocol entry // (browsers can't set `Authorization` on `new WebSocket()`); it is // verified via the same Better Auth path REST uses (HMAC + DB lookup). - const subprotocolHeader = data.request?.headers.get('sec-websocket-protocol') ?? null + const subprotocolHeader = slot.request?.headers.get('sec-websocket-protocol') ?? null const user: User | null = await authorizeWsBearer(auth, subprotocolHeader) if (!user) { ws.close(wsCloseUnauthorized, 'unauthorized') return } - const url = new URL(data.request?.url ?? 'http://localhost/haystack/ws') + const url = new URL(slot.request?.url ?? 'http://localhost/haystack/ws') const pipelineSlug = url.searchParams.get('pipeline') if (!pipelineSlug) { ws.close(wsCloseUnauthorized, 'missing pipeline parameter') return } - // Resolve the public slug back to its Deepset identifiers. A missing - // descriptor here means a stale FE URL or a redeploy that dropped the - // pipeline — we close instead of opening to keep error surface tight. - const descriptor = parsePipelinesEnv(settings).find((p) => p.id === pipelineSlug) - if (!descriptor) { + // Resolve the slug to its Deepset identifiers: an env-declared descriptor, + // or a Thunderbolt-deployed `tb-*` pipeline looked up live. A null result + // means a stale FE URL or a dropped pipeline — close instead of opening to + // keep the error surface tight. + const resolved = await resolveHaystackPipeline(pipelineSlug, settings, { fetchFn: deps?.fetchFn }) + if (!resolved) { ws.close(wsCloseUnauthorized, 'unknown pipeline') return } @@ -155,29 +176,43 @@ export const createHaystackRoutes = (settings: Settings, auth: Auth, deps?: Hays send: (payload) => { ws.send(payload) }, - pipelineId: descriptor.pipelineId, - pipelineName: descriptor.pipelineName, - supportsFiles: descriptor.supportedContent.files, + pipelineId: resolved.pipelineId, + pipelineName: resolved.pipelineName, + supportsFiles: resolved.supportsFiles, settings, deps, }) - ;(ws.data as unknown as { __haystackServer?: HaystackAcpServer }).__haystackServer = server - log.debug({ pipelineSlug, pipelineName: descriptor.pipelineName, userId: user.id }, 'haystack ws opened') + slot.server = server + // Drain buffered frames in arrival order before going live. Frames that + // arrive mid-drain are appended to the same queue (see `message`), so the + // loop picks them up and ordering is preserved. + while (slot.pending.length > 0) { + await server.handleMessage(slot.pending.shift()!) + } + slot.pending = undefined + log.debug({ pipelineSlug, pipelineName: resolved.pipelineName, userId: user.id }, 'haystack ws opened') }, async message(ws, message) { - const server = (ws.data as unknown as { __haystackServer?: HaystackAcpServer }).__haystackServer - if (!server) { - // Auth already rejected this socket; the client may still race a - // first frame before the close lands. Drop quietly. + const slot = ws.data as unknown as HaystackWsSlot + const text = typeof message === 'string' ? message : JSON.stringify(message) + // Still inside (or draining after) an async `open()` — preserve order by + // queueing; `open()` drains this once the server is ready. + if (slot.pending) { + slot.pending.push(text) return } - const text = typeof message === 'string' ? message : JSON.stringify(message) - await server.handleMessage(text) + if (!slot.server) { + // Auth rejected this socket; the client may still race a first frame + // before the close lands. Drop quietly. + return + } + await slot.server.handleMessage(text) }, close(ws) { - const slot = ws.data as unknown as { __haystackServer?: HaystackAcpServer } - slot.__haystackServer?.dispose() - slot.__haystackServer = undefined + const slot = ws.data as unknown as HaystackWsSlot + slot.server?.dispose() + slot.server = undefined + slot.pending = undefined }, }) } diff --git a/backend/src/haystack/types.ts b/backend/src/haystack/types.ts index 50a1c053a..fb92efe32 100644 --- a/backend/src/haystack/types.ts +++ b/backend/src/haystack/types.ts @@ -4,57 +4,6 @@ import { z } from 'zod' -/** - * Haystack pipeline descriptor as configured via the `HAYSTACK_PIPELINES` - * environment variable. The variable is a JSON array of these objects. Each - * pipeline becomes one {@link RemoteAgentDescriptor} on `GET /agents`. - * - * Deepset Cloud surfaces pipelines under two identifiers: - * - `pipelineName` — URL slug used in `/pipelines/${pipelineName}/chat-stream`. - * - `pipelineId` — the workspace-scoped UUID, used as the `pipeline_id` - * body field when bootstrapping a `search_session`. - * - * We keep them as separate fields because Deepset can rename a pipeline (slug - * changes) without re-issuing its UUID, and vice versa. - * - * - `id` — public slug we surface to the FE (`rag-chat`, etc). The - * only identifier the FE sees. - * - `name` — human-readable label for the agent picker. - * - `pipelineName`— Deepset URL slug. - * - `pipelineId` — Deepset UUID. - * - `description` — optional one-line description shown in the picker. - * - `icon` — optional Phosphor icon name; defaults applied by the - * provider when omitted. - * - `supportedContent` — which prompt content the pipeline accepts. `files: - * true` is load-bearing: it both advertises ACP - * `promptCapabilities.embeddedContext` to the client AND - * switches the run path to `temporary_files` upload + - * `search-stream` (generative pipelines), instead of the - * `search_sessions` + `chat-stream` path used by chat - * pipelines. Absent → `{ text: true, files: false }`, so - * existing chat configs need no change. - */ -const supportedContentSchema = z - .object({ - text: z.boolean().default(true), - files: z.boolean().default(false), - }) - .default({ text: true, files: false }) - -export const haystackPipelineDescriptorSchema = z.object({ - id: z.string().min(1), - name: z.string().min(1), - pipelineName: z.string().min(1), - pipelineId: z.string().min(1), - description: z.string().optional(), - icon: z.string().optional(), - supportedContent: supportedContentSchema, -}) - -export type HaystackPipelineDescriptor = z.infer - -export const haystackPipelinesEnvSchema = z.array(haystackPipelineDescriptorSchema) - /** * SSE event types emitted by Deepset's `/chat-stream` endpoint. Mirrors the * shape exercised by Deepset Cloud production traffic and the upstream diff --git a/backend/src/index.ts b/backend/src/index.ts index 8ea408e1b..3c55dd630 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -27,6 +27,7 @@ import { createWaitlistRoutes } from '@/waitlist/routes' import { createAccountRoutes } from '@/api/account' import { createAgentsRoutes } from '@/agents' import { createHaystackRoutes } from '@/haystack' +import { createOpenclawRoutes } from '@/openclaw' import { createConfigRoutes } from '@/api/config' import { createEncryptionRoutes } from '@/api/encryption' import { createPowerSyncRoutes } from '@/api/powersync' @@ -157,6 +158,7 @@ export const createApp = async (deps?: AppDeps) => { .use(createAccountRoutes(auth, database)) .use(createAgentsRoutes(auth)) .use(createHaystackRoutes(settings, auth, { fetchFn })) + .use(createOpenclawRoutes(settings, auth)) ) } diff --git a/backend/src/openclaw/e2b.test.ts b/backend/src/openclaw/e2b.test.ts new file mode 100644 index 000000000..70bd3bce0 --- /dev/null +++ b/backend/src/openclaw/e2b.test.ts @@ -0,0 +1,179 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, test } from 'bun:test' +import { + deployOpenclawSandbox, + killOpenclawSandboxForUser, + openclawSandboxStatusForUser, + resolveOpenclawSandboxForUser, + type E2bClient, + type E2bSandbox, + type OpenclawE2bConfig, +} from './e2b' + +const config: OpenclawE2bConfig = { + apiKey: 'e2b-key', + model: 'openrouter/some-model', + openrouterApiKey: 'or-key', +} + +/** A fake E2B client that records calls and serves scripted metadata. */ +const fakeClient = (overrides: Partial & { metadataById?: Record> } = {}) => { + const created: { template: string; opts: Parameters[1] }[] = [] + const ran: string[] = [] + const killed: string[] = [] + const connected: string[] = [] + + const makeSandbox = (sandboxId: string): E2bSandbox => ({ + sandboxId, + getHost: (port) => `${port}-${sandboxId}.e2b.app`, + commands: { + run: async (cmd) => { + ran.push(cmd) + }, + }, + kill: async () => { + killed.push(sandboxId) + }, + }) + + const client: E2bClient = { + create: async (template, opts) => { + created.push({ template, opts }) + return makeSandbox('sbx-1') + }, + connect: async (sandboxId) => { + connected.push(sandboxId) + return makeSandbox(sandboxId) + }, + getMetadata: async (sandboxId) => overrides.metadataById?.[sandboxId] ?? null, + kill: async (sandboxId) => { + killed.push(sandboxId) + }, + ...overrides, + } + return { client, created, ran, killed, connected } +} + +describe('deployOpenclawSandbox', () => { + test('stamps owner metadata + envs, launches the entrypoint, returns the handle without waiting', async () => { + const fake = fakeClient() + + const result = await deployOpenclawSandbox('user-a', config, { client: fake.client }) + + expect(result).toEqual({ sandboxId: 'sbx-1', wsUrl: 'wss://8790-sbx-1.e2b.app' }) + expect(fake.created[0]?.template).toBe('thunderbolt-openclaw') + expect(fake.created[0]?.opts.metadata).toEqual({ userId: 'user-a', kind: 'openclaw' }) + expect(fake.created[0]?.opts.envs.OPENROUTER_API_KEY).toBe('or-key') + expect(fake.created[0]?.opts.envs.MODEL).toBe('openrouter/some-model') + expect(fake.created[0]?.opts.secure).toBe(false) + expect(fake.ran).toEqual([ + "sh -c 'setsid nohup bash /opt/docker-entrypoint.sh >/tmp/openclaw-boot.log 2>&1 { + let killedSelf = false + const client = { + ...fakeClient().client, + create: async (): Promise => ({ + sandboxId: 'sbx-1', + getHost: (port) => `${port}-sbx-1.e2b.app`, + commands: { + run: async () => { + throw new Error('launch failed') + }, + }, + kill: async () => { + killedSelf = true + }, + }), + } + + await expect(deployOpenclawSandbox('user-a', config, { client })).rejects.toThrow('launch failed') + expect(killedSelf).toBe(true) + }) +}) + +describe('resolveOpenclawSandboxForUser', () => { + test('returns the handle when the caller owns the sandbox', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'user-a', kind: 'openclaw' } } }) + const result = await resolveOpenclawSandboxForUser('sbx-9', 'user-a', config.apiKey, { client: fake.client }) + expect(result).toEqual({ sandboxId: 'sbx-9', wsUrl: 'wss://8790-sbx-9.e2b.app' }) + expect(fake.connected).toEqual(['sbx-9']) + }) + + test('returns null and never connects when another user owns the sandbox', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'victim', kind: 'openclaw' } } }) + const result = await resolveOpenclawSandboxForUser('sbx-9', 'attacker', config.apiKey, { client: fake.client }) + expect(result).toBeNull() + expect(fake.connected).toEqual([]) // cross-tenant relay never dials + }) + + test('returns null when the sandbox no longer exists', async () => { + const fake = fakeClient({ metadataById: {} }) + const result = await resolveOpenclawSandboxForUser('gone', 'user-a', config.apiKey, { client: fake.client }) + expect(result).toBeNull() + }) +}) + +describe('killOpenclawSandboxForUser', () => { + test('kills the sandbox when the caller owns it', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'user-a', kind: 'openclaw' } } }) + const result = await killOpenclawSandboxForUser('sbx-9', 'user-a', config.apiKey, { client: fake.client }) + expect(result).toBe(true) + expect(fake.killed).toEqual(['sbx-9']) + }) + + test('never kills another tenant sandbox', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'victim', kind: 'openclaw' } } }) + const result = await killOpenclawSandboxForUser('sbx-9', 'attacker', config.apiKey, { client: fake.client }) + expect(result).toBe(false) + expect(fake.killed).toEqual([]) + }) + + test('no-op for a missing / already-gone sandbox', async () => { + const fake = fakeClient({ metadataById: {} }) + const result = await killOpenclawSandboxForUser('gone', 'user-a', config.apiKey, { client: fake.client }) + expect(result).toBe(false) + expect(fake.killed).toEqual([]) + }) +}) + +describe('openclawSandboxStatusForUser', () => { + test('running once the owner sandbox answers ACP', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'user-a' } } }) + const status = await openclawSandboxStatusForUser('sbx-9', 'user-a', config.apiKey, { + client: fake.client, + isAcpReady: async () => true, + }) + expect(status).toBe('running') + }) + + test('pending while the owner sandbox is up but ACP is not answering yet', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'user-a' } } }) + const status = await openclawSandboxStatusForUser('sbx-9', 'user-a', config.apiKey, { + client: fake.client, + isAcpReady: async () => false, + }) + expect(status).toBe('pending') + }) + + test('gone for another user or a missing sandbox, without probing ACP', async () => { + const fake = fakeClient({ metadataById: { 'sbx-9': { userId: 'victim' } } }) + let probed = false + const isAcpReady = async () => { + probed = true + return true + } + expect( + await openclawSandboxStatusForUser('sbx-9', 'attacker', config.apiKey, { client: fake.client, isAcpReady }), + ).toBe('gone') + expect( + await openclawSandboxStatusForUser('missing', 'user-a', config.apiKey, { client: fake.client, isAcpReady }), + ).toBe('gone') + expect(probed).toBe(false) + }) +}) diff --git a/backend/src/openclaw/e2b.ts b/backend/src/openclaw/e2b.ts new file mode 100644 index 000000000..e48b78902 --- /dev/null +++ b/backend/src/openclaw/e2b.ts @@ -0,0 +1,232 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Direct E2B provisioning for OpenClaw — no `SandboxProvider` abstraction yet + * (that seam, and other providers, come later). Each deploy is a Firecracker + * microVM from one static, prebuilt E2B template that bakes the OpenClaw stack + * (onboard + gateway + ACP↔WS shim) plus its entrypoint script. + * + * Ownership lives in the sandbox's own metadata (E2B is the source of truth, the + * way Deepset is for Haystack): the deploy stamps `{ userId }`, and every relay / + * status lookup rejects unless that stamp matches the authenticated caller — so a + * forged `?instance=` can never reach another tenant's sandbox, even after a + * backend restart. No backend instance table. + */ + +import { Sandbox } from 'e2b' +import { probeAcpReady } from './readiness' + +/** Internal port the sandbox's ACP↔WS shim listens on (baked into the template entrypoint). */ +const acpPort = 8790 +/** + * Boots the OpenClaw stack. The template bakes this script but no start command + * (E2B validates start commands at build time, yet the entrypoint needs the + * runtime OpenRouter key), so we invoke it at deploy with the key in `envs`. + * + * A short-lived `sh -c` backgrounds a fully-detached (`setsid nohup … &`) daemon, + * then exits after a beat. This survives the deploy RPC ending: without `setsid` + * the entrypoint's final `exec bun shim` is the command's foreground process, + * which E2B tears down when the RPC completes — killing the shim (port 8790) + * while the backgrounded gateway lingers, so the relay could never reach the + * sandbox. The trailing `sleep` lets the background stream establish and close + * cleanly, avoiding an instant-exit "socket closed unexpectedly" from the SDK. + */ +const entrypointCommand = + "sh -c 'setsid nohup bash /opt/docker-entrypoint.sh >/tmp/openclaw-boot.log 2>&1 string + commands: { run: (cmd: string, opts: { background?: boolean; envs?: Record }) => Promise } + kill: () => Promise +} +export type E2bClient = { + create: ( + template: string, + opts: { + apiKey: string + timeoutMs: number + secure: boolean + metadata: Record + envs: Record + }, + ) => Promise + connect: (sandboxId: string, opts: { apiKey: string }) => Promise + /** The sandbox's saved metadata, or null if it no longer exists (killed/expired). */ + getMetadata: (sandboxId: string, opts: { apiKey: string }) => Promise | null> + kill: (sandboxId: string, opts: { apiKey: string }) => Promise +} + +const toSandbox = (sbx: Sandbox): E2bSandbox => ({ + sandboxId: sbx.sandboxId, + getHost: (port) => sbx.getHost(port), + commands: { run: (cmd, opts) => sbx.commands.run(cmd, opts) }, + kill: () => sbx.kill(), +}) + +/** Default client backed by the real e2b SDK. */ +export const defaultE2bClient: E2bClient = { + create: async (template, opts) => + toSandbox( + await Sandbox.create(template, { + apiKey: opts.apiKey, + timeoutMs: opts.timeoutMs, + secure: opts.secure, + metadata: opts.metadata, + envs: opts.envs, + }), + ), + connect: async (sandboxId, opts) => toSandbox(await Sandbox.connect(sandboxId, { apiKey: opts.apiKey })), + getMetadata: async (sandboxId, opts) => { + try { + const info = await Sandbox.getInfo(sandboxId, { apiKey: opts.apiKey }) + return info.metadata + } catch { + return null + } + }, + kill: async (sandboxId, opts) => { + await Sandbox.kill(sandboxId, { apiKey: opts.apiKey }).catch(() => {}) + }, +} + +export type OpenclawE2bDeps = { + client?: E2bClient + /** ACP readiness probe (test seam). Defaults to a real single-shot `initialize` round-trip. */ + isAcpReady?: (wsUrl: string) => Promise +} + +/** E2B's `getHost(port)` returns a host that already encodes the port, so the relay dials `wss://` (TLS/443). */ +const wsUrlFor = (sandbox: E2bSandbox): string => `wss://${sandbox.getHost(acpPort)}` + +/** + * Provision a fresh OpenClaw sandbox for `userId`. Stamps the owner into + * metadata and kicks off the stack, then returns immediately — it does NOT wait + * for ACP to come up (that ~15-30s boot is observed via {@link openclawSandboxStatusForUser}, + * so the client persists the agent up front and the status badge tracks readiness). + * Tears the sandbox down if the create/launch itself fails. + */ +export const deployOpenclawSandbox = async ( + userId: string, + config: OpenclawE2bConfig, + deps: OpenclawE2bDeps = {}, +): Promise => { + const client = deps.client ?? defaultE2bClient + const envs = { + OPENROUTER_API_KEY: config.openrouterApiKey, + MODEL: config.model, + HOST: '0.0.0.0', + PORT: String(acpPort), + } + const sandbox = await client.create(openclawTemplate, { + apiKey: config.apiKey, + timeoutMs: sandboxTimeoutMs, + // POC posture: the ACP port is publicly reachable (unguessable host) and the + // Thunderbolt bearer at the relay is the real gate. Hardening later: use the + // sandbox access token as the upstream credential instead of `secure: false`. + secure: false, + metadata: { [ownerMetadataKey]: userId, kind: 'openclaw' }, + envs, + }) + try { + await sandbox.commands.run(entrypointCommand, { background: true, envs }) + return { sandboxId: sandbox.sandboxId, wsUrl: wsUrlFor(sandbox) } + } catch (err) { + await sandbox.kill().catch(() => {}) + throw err + } +} + +/** + * Resolve a live, caller-owned sandbox for the relay to dial. Reads the owner + * from the sandbox's own metadata and returns null unless it matches `userId`, + * so a forged `?instance=` can never relay into another tenant's sandbox. Null + * also covers a killed/expired sandbox. Connects only after the ownership check + * passes. + */ +export const resolveOpenclawSandboxForUser = async ( + sandboxId: string, + userId: string, + apiKey: string, + deps: OpenclawE2bDeps = {}, +): Promise => { + const client = deps.client ?? defaultE2bClient + const metadata = await client.getMetadata(sandboxId, { apiKey }) + if (!metadata || metadata[ownerMetadataKey] !== userId) { + return null + } + const sandbox = await client.connect(sandboxId, { apiKey }) + return { sandboxId: sandbox.sandboxId, wsUrl: wsUrlFor(sandbox) } +} + +/** + * Live status of a caller-owned sandbox, for the deploy-status poller. Resolves + * the sandbox owner-gated, then probes ACP: `gone` if it's been killed/expired or + * isn't the caller's; `running` once ACP answers; `pending` while it's still + * booting. The badge poller keeps polling on `pending` and stops on the rest. + */ +export const openclawSandboxStatusForUser = async ( + sandboxId: string, + userId: string, + apiKey: string, + deps: OpenclawE2bDeps = {}, +): Promise => { + const sandbox = await resolveOpenclawSandboxForUser(sandboxId, userId, apiKey, deps) + if (!sandbox) { + return 'gone' + } + const isReady = deps.isAcpReady ?? probeAcpReady + return (await isReady(sandbox.wsUrl)) ? 'running' : 'pending' +} + +/** + * Trigger teardown of a caller-owned sandbox. Owner-gated exactly like the relay: + * reads the sandbox's own metadata and kills only when it matches `userId`, so a + * forged `?instance=` can never kill another tenant's sandbox. A foreign or + * already-gone sandbox is a no-op (`false`) — undeploy is idempotent, so the + * client can still drop its local row. Returns once E2B accepts the kill; it does + * not wait for the microVM to fully tear down. + */ +export const killOpenclawSandboxForUser = async ( + sandboxId: string, + userId: string, + apiKey: string, + deps: OpenclawE2bDeps = {}, +): Promise => { + const client = deps.client ?? defaultE2bClient + const metadata = await client.getMetadata(sandboxId, { apiKey }) + if (!metadata || metadata[ownerMetadataKey] !== userId) { + return false + } + await client.kill(sandboxId, { apiKey }) + return true +} diff --git a/backend/src/openclaw/index.ts b/backend/src/openclaw/index.ts new file mode 100644 index 000000000..9b2e14e1a --- /dev/null +++ b/backend/src/openclaw/index.ts @@ -0,0 +1,5 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +export { createOpenclawRoutes } from './routes' diff --git a/backend/src/openclaw/provider.test.ts b/backend/src/openclaw/provider.test.ts new file mode 100644 index 000000000..d916aad88 --- /dev/null +++ b/backend/src/openclaw/provider.test.ts @@ -0,0 +1,135 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { ProviderContext } from '@/agents' +import { createTestSettings } from '@/test-utils/settings' +import { describe, expect, test } from 'bun:test' +import type { E2bClient, E2bSandbox } from './e2b' +import { createOpenclawProvider, openclawProviderId } from './provider' + +const configuredSettings = () => + createTestSettings({ + e2bApiKey: 'e2b-key', + openclawModel: 'openrouter/some-model', + openclawOpenrouterApiKey: 'or-key', + }) + +const makeSandbox = (sandboxId: string): E2bSandbox => ({ + sandboxId, + getHost: (port) => `${port}-${sandboxId}.e2b.app`, + commands: { run: async () => {} }, + kill: async () => {}, +}) + +/** Fake E2B client driving both deploy and the owner-gated status check. */ +const fakeClient = (metadataById: Record> = {}): E2bClient => ({ + create: async () => makeSandbox('sbx-1'), + connect: async (sandboxId) => makeSandbox(sandboxId), + getMetadata: async (sandboxId) => metadataById[sandboxId] ?? null, + kill: async () => {}, +}) + +const ctx = (userId: string): ProviderContext => ({ + request: new Request('http://localhost:8000/v1/agents/deploy'), + settings: configuredSettings(), + userId, +}) + +describe('createOpenclawProvider', () => { + test('catalog is exposed only when configured', () => { + const provider = createOpenclawProvider() + expect(provider.catalog?.({ ...ctx('user-a'), settings: configuredSettings() })).toHaveLength(1) + expect(provider.catalog?.({ ...ctx('user-a'), settings: createTestSettings() })).toEqual([]) + }) + + test('list is empty — deployed instances live in the synced agents table', async () => { + const provider = createOpenclawProvider() + expect(await provider.list(new Request('http://localhost'), createTestSettings())).toEqual([]) + }) + + test('deploy provisions a sandbox and returns the relay connection as pending', async () => { + const provider = createOpenclawProvider({ client: fakeClient() }) + const result = await provider.deploy!({}, ctx('user-a')) + + expect(result.deploymentId).toBe(`${openclawProviderId}:e2b:sbx-1`) + expect(result.status).toBe('pending') + expect(result.connection?.transport).toBe('websocket') + expect(result.connection?.url).toBe('ws://localhost:8000/v1/openclaw/ws?instance=e2b%3Asbx-1') + }) + + test('status is running (with connection) once ACP answers, pending while it boots', async () => { + const owned = fakeClient({ 'sbx-1': { userId: 'user-a' } }) + const ready = createOpenclawProvider({ client: owned, isAcpReady: async () => true }) + const running = await ready.status!('e2b:sbx-1', ctx('user-a')) + expect(running.status).toBe('running') + expect(running.connection?.url).toContain('instance=e2b%3Asbx-1') + + const booting = createOpenclawProvider({ client: owned, isAcpReady: async () => false }) + const pending = await booting.status!('e2b:sbx-1', ctx('user-a')) + expect(pending.status).toBe('pending') + expect(pending.connection).toBeNull() + }) + + test('status is gone for another user, a missing sandbox, or a malformed ref', async () => { + const provider = createOpenclawProvider({ + client: fakeClient({ 'sbx-1': { userId: 'victim' } }), + isAcpReady: async () => true, + }) + + const otherUser = await provider.status!('e2b:sbx-1', ctx('attacker')) + expect(otherUser.status).toBe('gone') + expect(otherUser.connection).toBeNull() + + const missing = await provider.status!('e2b:ghost', ctx('user-a')) + expect(missing.status).toBe('gone') + + const malformed = await provider.status!('not-a-ref', ctx('user-a')) + expect(malformed.status).toBe('gone') + }) + + test('undeploy kills the owner sandbox and reports gone', async () => { + const killed: string[] = [] + const client: E2bClient = { + ...fakeClient({ 'sbx-1': { userId: 'user-a' } }), + kill: async (sandboxId) => { + killed.push(sandboxId) + }, + } + const provider = createOpenclawProvider({ client }) + + const result = await provider.undeploy!('e2b:sbx-1', ctx('user-a')) + expect(result).toEqual({ deploymentId: `${openclawProviderId}:e2b:sbx-1`, status: 'gone' }) + expect(killed).toEqual(['sbx-1']) + }) + + test('undeploy never kills another tenant sandbox but still reports gone (idempotent)', async () => { + const killed: string[] = [] + const client: E2bClient = { + ...fakeClient({ 'sbx-1': { userId: 'victim' } }), + kill: async (sandboxId) => { + killed.push(sandboxId) + }, + } + const provider = createOpenclawProvider({ client }) + + const result = await provider.undeploy!('e2b:sbx-1', ctx('attacker')) + expect(result.status).toBe('gone') + expect(killed).toEqual([]) + }) + + test('undeploy is a no-op for a malformed ref', async () => { + const killed: string[] = [] + const client: E2bClient = { + ...fakeClient(), + kill: async (sandboxId) => { + killed.push(sandboxId) + }, + } + const provider = createOpenclawProvider({ client }) + + const result = await provider.undeploy!('not-a-ref', ctx('user-a')) + expect(result.status).toBe('gone') + expect(killed).toEqual([]) + }) +}) diff --git a/backend/src/openclaw/provider.ts b/backend/src/openclaw/provider.ts new file mode 100644 index 000000000..a7e044b06 --- /dev/null +++ b/backend/src/openclaw/provider.ts @@ -0,0 +1,144 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import type { AgentProvider, ProviderContext } from '@/agents' +import { buildWebSocketUrl, encodeDeploymentId } from '@/agents' +import type { Settings } from '@/config/settings' +import type { RemoteAgentDescriptor } from '@shared/acp-types' +import type { + AgentConnection, + AgentDescriptor, + AgentSpec, + DeploymentStatusResponse, + DeployResponse, + UndeployResponse, +} from '@shared/agent-descriptors' +import { + deployOpenclawSandbox, + killOpenclawSandboxForUser, + openclawSandboxStatusForUser, + type OpenclawE2bConfig, + type OpenclawE2bDeps, +} from './e2b' + +/** + * Provider id registered into the agent discovery registry. Stable: the registry + * dedupes on it and deploy requests route back here via `AgentDescriptor.provider`. + */ +export const openclawProviderId = 'openclaw' + +/** The sandbox backend encoded in a deployment ref (`:`). Only E2B for now. */ +const e2bRefPrefix = 'e2b' + +/** Version of {@link openclawDescriptor}; bump when its fields change. */ +const openclawSchemaVersion = 1 + +/** + * Curated one-click "Add agent" form for OpenClaw. Owner-managed: the backend + * fixes the sandbox provider + model (env), so the only field is an optional + * display name — which keeps it one-click-eligible (no required visible fields). + */ +const openclawDescriptor: AgentDescriptor = { + id: openclawProviderId, + provider: openclawProviderId, + name: 'OpenClaw', + description: 'Deploy a sandboxed OpenClaw coding agent.', + icon: 'terminal', + schemaVersion: openclawSchemaVersion, + action: 'deploy', + steps: [ + { + id: 'basics', + title: 'Name your agent', + fields: [ + { + key: 'name', + label: 'Name', + widget: 'text', + required: false, + maxLength: 60, + placeholder: 'OpenClaw', + default: 'OpenClaw', + }, + ], + }, + ], +} + +/** Whether OpenClaw is fully configured to deploy (E2B creds + model + inference key). */ +const isOpenclawConfigured = (settings: Settings): boolean => + Boolean(settings.e2bApiKey && settings.openclawModel && settings.openclawOpenrouterApiKey) + +/** Map settings onto the settings-agnostic E2B deploy config. */ +const e2bConfig = (settings: Settings): OpenclawE2bConfig => ({ + apiKey: settings.e2bApiKey, + model: settings.openclawModel, + openrouterApiKey: settings.openclawOpenrouterApiKey, +}) + +/** Parse a deployment ref (`e2b:`) to its sandbox id, or null if it isn't an E2B ref. */ +export const parseSandboxRef = (ref: string): string | null => { + const sep = ref.indexOf(':') + if (sep <= 0) { + return null + } + const sandboxId = ref.slice(sep + 1) + return ref.slice(0, sep) === e2bRefPrefix && sandboxId ? sandboxId : null +} + +/** The relay endpoint for a given ref — deterministic, so deploy can hand it back up front. */ +const connectionFor = (request: Request, ref: string): AgentConnection => ({ + url: buildWebSocketUrl(request, `openclaw/ws?instance=${encodeURIComponent(ref)}`), + transport: 'websocket', +}) + +/** + * Build the OpenClaw provider. Unlike Haystack it's deploy-only: `list()` is + * empty because a deployed instance is persisted to the synced `agents` table by + * the client (so listing it here too would double it). `deploy` provisions a + * per-user E2B sandbox and returns the relay URL immediately; `status` polls the + * sandbox live (owner-gated) and never stores runtime state. `deps` injects a + * fake E2B client in tests. + */ +export const createOpenclawProvider = (deps: OpenclawE2bDeps = {}): AgentProvider => ({ + id: openclawProviderId, + list: (): RemoteAgentDescriptor[] => [], + catalog: ({ settings }: ProviderContext): AgentDescriptor[] => + isOpenclawConfigured(settings) ? [openclawDescriptor] : [], + deploy: async (_spec: AgentSpec, { request, settings, userId }: ProviderContext): Promise => { + const sandbox = await deployOpenclawSandbox(userId, e2bConfig(settings), deps) + const ref = `${e2bRefPrefix}:${sandbox.sandboxId}` + // Return immediately with `pending` and the (deterministic) relay endpoint: + // deploy no longer blocks on the ~15-30s boot, so the client persists the + // agent up front and the status badge polls until ACP answers. + return { + deploymentId: encodeDeploymentId(openclawProviderId, ref), + status: 'pending', + connection: connectionFor(request, ref), + } + }, + status: async (ref: string, { request, settings, userId }: ProviderContext): Promise => { + const deploymentId = encodeDeploymentId(openclawProviderId, ref) + const sandboxId = parseSandboxRef(ref) + // Malformed ref → treat as gone (a terminal state the client warns on). + if (!sandboxId) { + return { deploymentId, status: 'gone', connection: null } + } + const status = await openclawSandboxStatusForUser(sandboxId, userId, settings.e2bApiKey, deps) + // Only a usable (ACP-answering) sandbox carries the chat endpoint. + const connection = status === 'running' ? connectionFor(request, ref) : null + return { deploymentId, status, connection } + }, + undeploy: async (ref: string, { settings, userId }: ProviderContext): Promise => { + const deploymentId = encodeDeploymentId(openclawProviderId, ref) + const sandboxId = parseSandboxRef(ref) + // Malformed ref, or a foreign / already-gone sandbox → idempotent no-op; the + // owner-gated kill returns false without throwing, so the client still drops + // its local row. + if (sandboxId) { + await killOpenclawSandboxForUser(sandboxId, userId, settings.e2bApiKey, deps) + } + return { deploymentId, status: 'gone' } + }, +}) diff --git a/backend/src/openclaw/readiness.ts b/backend/src/openclaw/readiness.ts new file mode 100644 index 000000000..59c4d4675 --- /dev/null +++ b/backend/src/openclaw/readiness.ts @@ -0,0 +1,48 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * ACP readiness probe for a sandbox. A sandbox reports as running (E2B) well + * before the OpenClaw stack inside it answers ACP, so `status` must probe the + * actual protocol, not just liveness — otherwise the first chat races a + * not-yet-serving agent. + * + * A bare TCP/HTTP check isn't enough (E2B's host resolves before the shim + * listens). So we do the real ACP `initialize` round-trip — the exact handshake + * the client does first — and count ready only when a JSON-RPC reply comes back. + */ + +/** One probe: resolves true iff an `initialize` reply arrives before `timeoutMs`. */ +export const probeAcpReady = (wsUrl: string, timeoutMs = 4000): Promise => + new Promise((resolve) => { + let settled = false + const ws = new WebSocket(wsUrl) + const finish = (ready: boolean) => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + try { + ws.close() + } catch { + // already closing + } + resolve(ready) + } + const timer = setTimeout(() => finish(false), timeoutMs) + ws.addEventListener('open', () => { + ws.send( + JSON.stringify({ + jsonrpc: '2.0', + id: 0, + method: 'initialize', + params: { protocolVersion: 1, clientCapabilities: { fs: { readTextFile: false, writeTextFile: false } } }, + }), + ) + }) + ws.addEventListener('message', () => finish(true)) + ws.addEventListener('error', () => finish(false)) + ws.addEventListener('close', () => finish(false)) + }) diff --git a/backend/src/openclaw/relay.test.ts b/backend/src/openclaw/relay.test.ts new file mode 100644 index 000000000..b611a9052 --- /dev/null +++ b/backend/src/openclaw/relay.test.ts @@ -0,0 +1,85 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, test } from 'bun:test' +import { createSandboxRelay, type UpstreamSocket } from './relay' + +/** Fake upstream socket that lets tests drive open/message/close by hand. */ +const fakeUpstream = () => { + const listeners: Record void)[]> = {} + const sent: string[] = [] + let closed = false + const socket: UpstreamSocket = { + readyState: 0, + send: (data) => sent.push(data), + close: () => { + closed = true + }, + addEventListener: (type, listener) => { + ;(listeners[type] ??= []).push(listener) + }, + } + const emit = (type: string, event: { data?: unknown } = {}) => { + for (const l of listeners[type] ?? []) { + l(event) + } + } + return { socket, sent, emit, isClosed: () => closed } +} + +describe('createSandboxRelay', () => { + test('queues browser frames until upstream opens, then flushes in order', () => { + const up = fakeUpstream() + const relay = createSandboxRelay( + 'ws://x', + () => {}, + () => {}, + { connect: () => up.socket }, + ) + + relay.fromBrowser('a') + relay.fromBrowser('b') + expect(up.sent).toEqual([]) // not open yet + + up.socket.readyState = 1 + up.emit('open') + expect(up.sent).toEqual(['a', 'b']) + + relay.fromBrowser('c') + expect(up.sent).toEqual(['a', 'b', 'c']) // sent immediately once open + }) + + test('forwards upstream messages to the browser', () => { + const up = fakeUpstream() + const received: string[] = [] + createSandboxRelay( + 'ws://x', + (data) => received.push(data), + () => {}, + { connect: () => up.socket }, + ) + + up.emit('message', { data: '{"jsonrpc":"2.0"}' }) + expect(received).toEqual(['{"jsonrpc":"2.0"}']) + }) + + test('notifies on upstream close and close() tears down upstream', () => { + const up = fakeUpstream() + let upstreamClosed = false + const relay = createSandboxRelay( + 'ws://x', + () => {}, + () => { + upstreamClosed = true + }, + { connect: () => up.socket }, + ) + + up.emit('close') + expect(upstreamClosed).toBe(true) + + relay.close() + expect(up.isClosed()).toBe(true) + }) +}) diff --git a/backend/src/openclaw/relay.ts b/backend/src/openclaw/relay.ts new file mode 100644 index 000000000..e2e5a2682 --- /dev/null +++ b/backend/src/openclaw/relay.ts @@ -0,0 +1,79 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Backend ACP frame relay: pipes a browser WebSocket to the sandbox's ACP-WS + * endpoint. It never parses ACP — each message is one JSON-RPC object (the same + * one-object-per-frame contract the sandbox shim and Thunderbolt's transport + * both use), so the relay is a dumb, version-proof pipe. + * + * This is the managed-acp equivalent of Haystack's in-backend ACP server: there + * the backend *is* the agent; here the backend just forwards to the per-user + * sandbox that runs the real agent. + */ + +/** Minimal upstream socket surface — lets tests inject a fake without a real WS. */ +export type UpstreamSocket = { + readyState: number + send: (data: string) => void + close: () => void + addEventListener: (type: 'open' | 'message' | 'close', listener: (event: { data?: unknown }) => void) => void +} + +export type SandboxRelay = { + /** Forward a frame from the browser to the sandbox (queued until upstream opens). */ + fromBrowser: (data: string) => void + /** Tear down the upstream connection. */ + close: () => void +} + +const readyStateOpen = 1 + +/** + * Open a relay to `targetUrl` (the sandbox ACP-WS endpoint). `toBrowser` is + * called with each frame the sandbox emits; `onUpstreamClose` fires when the + * sandbox side drops so the caller can close the browser socket too. + */ +export const createSandboxRelay = ( + targetUrl: string, + toBrowser: (data: string) => void, + onUpstreamClose: () => void, + deps: { connect?: (url: string) => UpstreamSocket } = {}, +): SandboxRelay => { + const connect = deps.connect ?? ((url) => new WebSocket(url) as unknown as UpstreamSocket) + const upstream = connect(targetUrl) + const queue: string[] = [] + let open = false + + upstream.addEventListener('open', () => { + open = true + for (const msg of queue) { + upstream.send(msg) + } + queue.length = 0 + }) + upstream.addEventListener('message', (event) => { + toBrowser(typeof event.data === 'string' ? event.data : String(event.data)) + }) + upstream.addEventListener('close', () => { + onUpstreamClose() + }) + + return { + fromBrowser: (data) => { + if (open && upstream.readyState === readyStateOpen) { + upstream.send(data) + return + } + queue.push(data) + }, + close: () => { + try { + upstream.close() + } catch { + // already closed + } + }, + } +} diff --git a/backend/src/openclaw/routes.test.ts b/backend/src/openclaw/routes.test.ts new file mode 100644 index 000000000..2ea41dc24 --- /dev/null +++ b/backend/src/openclaw/routes.test.ts @@ -0,0 +1,36 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The WS relay's auth contract mirrors Haystack's (covered in haystack/routes.test.ts: + * bearer in `Sec-WebSocket-Protocol`, validated in `open()`), and the relay pipe + + * owner gate are unit-tested in relay.test.ts / e2b.test.ts. What's unique here is + * the wiring: mounting the routes must register the provider so `/v1/agents/*` + * (deploy/catalog/status) route to OpenClaw. + */ + +import { getProviderById } from '@/agents' +import { resetAgentProvidersForTesting } from '@/agents/discovery' +import type { Auth } from '@/auth/elysia-plugin' +import { createTestSettings } from '@/test-utils/settings' +import { beforeEach, describe, expect, test } from 'bun:test' +import { createOpenclawRoutes } from './routes' + +const mockAuth = {} as Auth + +describe('createOpenclawRoutes', () => { + beforeEach(() => resetAgentProvidersForTesting()) + + test('registers the openclaw provider with deploy/catalog/status verbs', () => { + expect(getProviderById('openclaw')).toBeUndefined() + + createOpenclawRoutes(createTestSettings(), mockAuth) + + const provider = getProviderById('openclaw') + expect(provider?.id).toBe('openclaw') + expect(provider?.catalog).toBeDefined() + expect(provider?.deploy).toBeDefined() + expect(provider?.status).toBeDefined() + }) +}) diff --git a/backend/src/openclaw/routes.ts b/backend/src/openclaw/routes.ts new file mode 100644 index 000000000..7862a606d --- /dev/null +++ b/backend/src/openclaw/routes.ts @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { registerAgentProvider } from '@/agents' +import type { Auth } from '@/auth/elysia-plugin' +import { authorizeWsBearer, wsCloseUnauthorized } from '@/auth/ws-bearer-auth' +import { createStandaloneLogger } from '@/config/logger' +import type { Settings } from '@/config/settings' +import { safeErrorHandler } from '@/middleware/error-handling' +import type { User } from '@shared/types/auth' +import { wsCarrierSubprotocol } from '@shared/ws-bearer' +import { Elysia } from 'elysia' +import { resolveOpenclawSandboxForUser, type OpenclawE2bDeps } from './e2b' +import { createOpenclawProvider, parseSandboxRef } from './provider' +import { createSandboxRelay, type SandboxRelay } from './relay' + +/** + * Per-connection state on `ws.data`. `relay` is created once the async `open()` + * resolves; `pending` buffers frames that arrive during that window so the FE's + * eager `initialize` isn't dropped (same fix as `haystack/routes.ts`). `request` + * is Bun's original upgrade request. + */ +type WsSlot = { request?: Request; relay?: SandboxRelay; pending?: string[] } + +/** + * Mount the OpenClaw ACP relay. + * + * - Registers the OpenClaw provider into the agent registry on construction so + * its deploy/catalog/status verbs flow through the shared `/v1/agents/*` + * endpoints (idempotent — `registerAgentProvider` dedupes on `id`). + * - Exposes `WS /v1/openclaw/ws?instance=` — the managed-acp wire. Unlike + * Haystack (which translates ACP↔SSE in-backend), this is a dumb pipe: the + * sandbox already speaks ACP, so we just relay frames both ways. + * + * Auth mirrors Haystack exactly: the signed bearer rides a `Sec-WebSocket-Protocol` + * entry and is validated once in `open()`. The `ref` (`e2b:`) is + * resolved owner-gated against the sandbox's own metadata — a forged/foreign + * instance closes the socket without ever dialing the sandbox. `deps` injects a + * fake E2B client in tests. + */ +export const createOpenclawRoutes = (settings: Settings, auth: Auth, deps: OpenclawE2bDeps = {}) => { + registerAgentProvider(createOpenclawProvider(deps)) + + return new Elysia({ name: 'openclaw-routes', prefix: '/openclaw' }).onError(safeErrorHandler).ws('/ws', { + upgrade({ request, set }) { + // Echo the carrier subprotocol (RFC 6455) so strict clients accept the + // upgrade; the auth-bearing bearer entry is intentionally NOT echoed. + const subprotocolHeader = request.headers.get('sec-websocket-protocol') + if (subprotocolHeader?.split(',').some((entry) => entry.trim() === wsCarrierSubprotocol)) { + set.headers['sec-websocket-protocol'] = wsCarrierSubprotocol + } + }, + async open(ws) { + const log = createStandaloneLogger(settings) + const slot = ws.data as unknown as WsSlot + // Bun delivers `message` frames without awaiting this async handler, so the + // FE's first ACP frame can arrive while auth + the live sandbox lookup are + // still in flight. Buffer anything that lands and drain it once the relay + // exists (that dropped `initialize` was the whole hang on Haystack). + slot.pending = [] + + const subprotocolHeader = slot.request?.headers.get('sec-websocket-protocol') ?? null + const user: User | null = await authorizeWsBearer(auth, subprotocolHeader) + if (!user) { + ws.close(wsCloseUnauthorized, 'unauthorized') + return + } + + const url = new URL(slot.request?.url ?? 'http://localhost/openclaw/ws') + const ref = url.searchParams.get('instance') + const sandboxId = ref ? parseSandboxRef(ref) : null + if (!sandboxId) { + ws.close(wsCloseUnauthorized, 'missing or invalid instance') + return + } + + // Owner-gated: reads the sandbox's own metadata (E2B is the source of truth). + // A forged or foreign `?instance=` resolves to null → close without dialing. + const sandbox = await resolveOpenclawSandboxForUser(sandboxId, user.id, settings.e2bApiKey, deps) + if (!sandbox) { + log.warn({ sandboxId, userId: user.id }, 'openclaw ws: instance not found or not owned') + ws.close(wsCloseUnauthorized, 'unknown instance') + return + } + + const relay = createSandboxRelay( + sandbox.wsUrl, + (payload) => ws.send(payload), + () => ws.close(), + ) + slot.relay = relay + // Drain buffered frames in arrival order; frames arriving mid-drain append + // to the same queue (see `message`), so ordering is preserved. + while (slot.pending.length > 0) { + relay.fromBrowser(slot.pending.shift()!) + } + slot.pending = undefined + log.debug({ sandboxId, userId: user.id }, 'openclaw ws opened') + }, + message(ws, message) { + const slot = ws.data as unknown as WsSlot + const frame = typeof message === 'string' ? message : JSON.stringify(message) + // Still inside (or draining after) open() — queue to preserve order. + if (slot.pending) { + slot.pending.push(frame) + return + } + slot.relay?.fromBrowser(frame) + }, + close(ws) { + const slot = ws.data as unknown as WsSlot + slot.relay?.close() + slot.relay = undefined + slot.pending = undefined + }, + }) +} diff --git a/backend/src/test-utils/settings.ts b/backend/src/test-utils/settings.ts index f06cf646e..83f7570a1 100644 --- a/backend/src/test-utils/settings.ts +++ b/backend/src/test-utils/settings.ts @@ -59,10 +59,14 @@ export const createTestSettings = (overrides: Partial = {}): Settings enabledAgents: '', allowCustomAgents: true, disableBuiltInAgent: false, + agentDeploy: false, haystackBaseUrl: '', haystackApiKey: '', haystackWorkspace: '', - haystackPipelines: '', + haystackTemplatePipeline: '', + e2bApiKey: '', + openclawModel: '', + openclawOpenrouterApiKey: '', minAppVersion: '', ...overrides, }) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 8eb7c5dec..601a5129a 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -157,4 +157,5 @@ The custom SharedWorker extends `SharedSyncImplementation`, an `@internal` class - [Multi-Device Sync](./multi-device-sync.md) — the sync pipeline in more depth. - [End-to-End Encryption](./e2e-encryption.md) — key hierarchy and device approval. +- [Agent Deployment](./agent-deployment.md) — descriptor-driven deploy flow, provider seam, adding new agents. - [Quick Start](../development/quick-start.md) and [Testing](../development/testing.md) — schema rules, tests, the things that bite. diff --git a/docs/architecture/agent-deployment.md b/docs/architecture/agent-deployment.md new file mode 100644 index 000000000..943218d30 --- /dev/null +++ b/docs/architecture/agent-deployment.md @@ -0,0 +1,598 @@ +# Agent Deployment (descriptor-driven) + +How Thunderbolt lets a user pick an agent kind from a catalog, fill in a short +form, and get a running, chat-ready agent — without the frontend knowing +anything provider-specific. Haystack (managed RAG pipelines) and OpenClaw +(sandboxed coding agent on E2B) are the two providers today. + +## Mental model + +Three moving parts, one wire contract: + +1. **Descriptor** — "an agent-creation form as data." The backend curates it + (steps → fields → widgets); the frontend renders it over a fixed widget set. + Defined once in `shared/agent-descriptors.ts` and imported by both ends. +2. **Provider** — the adapter for one agent "kind" (`haystack`, `openclaw`, …). + It supplies the descriptor and does the real work of `deploy` / `status` / + `undeploy`. Providers self-register into a module-level registry. +3. **Generic endpoints** — `/v1/agents/*` never mention a provider by name. They + collect descriptors from every provider, and route a deploy/status/undeploy + back to the owning provider via the `provider` field on the descriptor. + +The key consequence: **adding a new agent kind means writing one provider file +and registering it. No changes to the frontend or the shared endpoints.** + +There is also **no server-side deployment table.** A deployment is identified by +a self-describing `deploymentId = ":"`. Status is polled _live_ +from the host (Haystack / E2B) on demand; the client persists the agent row +itself (in the synced `agents` table). + +```mermaid +sequenceDiagram + participant FE as Frontend + participant BE as Backend /v1/agents/* + participant P as Provider + participant H as Host (Haystack / E2B) + + FE->>BE: GET /agents/catalog + BE->>P: catalog() (all providers) + P-->>BE: descriptor[] + BE-->>FE: descriptors + Note over FE: render form from descriptor + + FE->>BE: POST /agents/deploy { descriptorId, spec } + BE->>BE: re-validate spec, dispatch by provider + BE->>P: deploy(spec) + P->>H: create + deploy + P-->>BE: { deploymentId, status, connection } + BE-->>FE: deploy response + Note over FE: persist agent row (synced) + + loop until running + FE->>BE: GET /deployments/:id + BE->>P: status(ref) (id decoded → owning provider) + P->>H: live status lookup + P-->>BE: { status, connection } + BE-->>FE: status response + end + + FE->>H: open connection.url (WS, managed-acp) + Note over FE,H: ACP chat frames relayed both ways +``` + +## Endpoints + +All live under the `/v1` prefix. All require an authenticated, **non-anonymous** +user (bearer token). Anonymous sessions get `403 ANONYMOUS_DISCOVERY_FORBIDDEN`; +missing/invalid auth gets `401`. + +| Method | Path | Purpose | Gated by | +| -------- | ----------------------------- | -------------------------------------------------------------------------- | ------------------- | +| `GET` | `/agents` | Discovery — every provider's `list()`, flattened. Powers the agent picker. | always on | +| `GET` | `/agents/catalog` | Deployable descriptors (every provider's `catalog()`). | `AGENT_DEPLOY=true` | +| `POST` | `/agents/deploy` | Deploy an instance from a submitted spec. | `AGENT_DEPLOY=true` | +| `GET` | `/agents/deployments/:id` | Live status for a deployment id. | `AGENT_DEPLOY=true` | +| `DELETE` | `/agents/deployments/:id` | Trigger teardown (undeploy) of a deployment. | `AGENT_DEPLOY=true` | +| `WS` | `/haystack/ws?pipeline=` | Haystack chat wire (ACP ↔ Haystack SSE, translated in-backend). | Haystack configured | +| `WS` | `/openclaw/ws?instance=` | OpenClaw chat wire (dumb ACP relay to the sandbox). | OpenClaw configured | + +When `AGENT_DEPLOY` is off, the three deploy endpoints return `404` — the +frontend treats that identically to "nothing to deploy" and hides the flow. + +### WebSocket auth + +Browsers can't set `Authorization` headers on `new WebSocket()`, so the chat +wires authenticate via a **subprotocol**: the client offers +`["thunderbolt.v1", "thunderbolt.bearer."]`. The server echoes +only the carrier (`thunderbolt.v1`) and validates the bearer once on `open()`. +This is attached automatically **only for `managed-acp` agents** +(`src/acp/transports/index.ts`). A user-added custom agent (`remote-acp`) never +sends Thunderbolt credentials — so pasting a `/openclaw/ws` or `/haystack/ws` +URL into "Add custom agent" will fail auth by design. Deployed agents get typed +`managed-acp` and connect correctly. + +### Ownership model + +`open()` on both chat wires requires a non-anonymous bearer — but _which_ +deployments a caller may reach differs by provider, and a new provider must +choose deliberately: + +- **OpenClaw — per-user.** The sandbox is stamped with the deployer's `userId` + in E2B metadata (the source of truth; no backend table). `status`, `undeploy`, + and the WS relay all re-check it, so a forged or foreign `?instance=` resolves + to `gone` / a no-op / a closed socket without ever touching the sandbox. +- **Haystack — workspace-shared.** Pipelines live in one Deepset workspace; any + authenticated user in the deployment can reach a pipeline by name (no per-user + gate). Appropriate for shared RAG pipelines — but don't assume it when your + host resource is per-tenant. + +### Errors + +All error bodies are `{ error: string, code?: string }`. + +| Status | When | Body | +| ------ | ------------------------------------------ | ----------------------------------------------------------------------- | +| `401` | missing / invalid bearer | `{ error: 'Unauthorized' }` | +| `403` | anonymous session | `{ error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' }` | +| `404` | `AGENT_DEPLOY` off (catalog/deploy/status) | `{ error: 'Not found' }` | +| `400` | malformed deploy body | `{ error: 'Invalid deploy request' }` | +| `404` | `descriptorId` not in catalog | `{ error: 'Unknown agent' }` | +| `409` | stale `schemaVersion` | `{ error: 'Schema version mismatch', code: 'SCHEMA_VERSION_MISMATCH' }` | +| `404` | provider exposes no `deploy` | `{ error: 'Agent is not deployable' }` | +| `400` | spec fails server re-validation | `{ error: 'Invalid spec' }` | +| `400` | malformed deployment id | `{ error: 'Invalid deployment id' }` | +| `404` | unknown provider in the id | `{ error: 'Unknown provider' }` | + +## Contracts + +Deploy-flow types live in `shared/agent-descriptors.ts` (zod-validated on both +ends); the discovery shape is in `shared/acp-types.ts`. + +**`GET /agents`** → `AgentDiscoveryResponse` — the always-on picker feed +(built-in agent + every provider's `list()`). + +```ts +{ version: '1', agents: RemoteAgentDescriptor[], allowCustomAgents: boolean } +// RemoteAgentDescriptor: +{ id, name, type: 'remote-acp' | 'managed-acp', transport: 'websocket', url, description, icon, isSystem: 0 | 1 } +``` + +`ENABLED_AGENTS` (comma-separated ids) narrows the list; `allowCustomAgents` +mirrors `ALLOW_CUSTOM_AGENTS` so the UI can hide "+ Add Custom Agent". + +**`GET /agents/catalog`** → `AgentCatalogResponse` + +```ts +{ version: '1', descriptors: AgentDescriptor[] } +``` + +```jsonc +// real: both providers configured (descriptor.steps trimmed for brevity) +{ + "version": "1", + "descriptors": [ + { + "id": "haystack", + "provider": "haystack", + "name": "Haystack RAG agent", + "schemaVersion": 1, + "action": "deploy", + "steps": [ + /* … */ + ], + }, + { + "id": "openclaw", + "provider": "openclaw", + "name": "OpenClaw", + "schemaVersion": 1, + "action": "deploy", + "steps": [ + /* … */ + ], + }, + ], +} +``` + +**`POST /agents/deploy`** — body `DeployRequest`, response `DeployResponse` + +```ts +// request +{ descriptorId: string, schemaVersion: number, spec: Record } +// response +{ deploymentId: string, status: DeployStatus, connection: AgentConnection | null } +``` + +```jsonc +// real: Haystack — user typed a name; deploys fast, usually already running +// request +{ "descriptorId": "haystack", "schemaVersion": 1, "spec": { "name": "My research agent" } } +// response +{ "deploymentId": "haystack:tb-my-research-agent-lx9f2k", "status": "running", + "connection": { "url": "ws://localhost:8000/v1/haystack/ws?pipeline=tb-my-research-agent-lx9f2k", "transport": "websocket" } } +``` + +```jsonc +// real: OpenClaw — one-click (name defaulted); returns pending, boots async +// request +{ "descriptorId": "openclaw", "schemaVersion": 1, "spec": { "name": "OpenClaw" } } +// response +{ "deploymentId": "openclaw:e2b:iq8fy72dtcwkosdu0jpu", "status": "pending", + "connection": { "url": "ws://localhost:8000/v1/openclaw/ws?instance=e2b%3Aiq8fy72dtcwkosdu0jpu", "transport": "websocket" } } +``` + +**`GET /agents/deployments/:id`** → `DeploymentStatusResponse` + +```ts +{ deploymentId: string, status: DeployStatus, detail?: string | null, connection?: AgentConnection | null } +``` + +```jsonc +// real: OpenClaw sandbox now answering ACP (id URL-encoded in the path) +{ + "deploymentId": "openclaw:e2b:iq8fy72dtcwkosdu0jpu", + "status": "running", + "connection": { + "url": "ws://localhost:8000/v1/openclaw/ws?instance=e2b%3Aiq8fy72dtcwkosdu0jpu", + "transport": "websocket", + }, +} +``` + +**`DELETE /agents/deployments/:id`** → `UndeployResponse` — triggers teardown on +the host and returns once accepted (does not block on full unmount). + +```ts +{ deploymentId: string, status: DeployStatus } // status is `gone` once teardown is triggered +``` + +```jsonc +// real: undeploy an OpenClaw sandbox (id URL-encoded in the path) +{ "deploymentId": "openclaw:e2b:iq8fy72dtcwkosdu0jpu", "status": "gone" } +``` + +Teardown is **idempotent**: a foreign / already-gone / malformed deployment is a +no-op that still returns `gone`, so the client can always drop its local row. A +genuine trigger failure (host/network error) surfaces as a `5xx` — the client +keeps the row and shows an error rather than orphaning a live deployment. + +The full curl walkthroughs for these live in [Worked examples](#worked-examples). + +Shared value types: + +```ts +type DeployStatus = 'pending' | 'running' | 'failed' | 'gone' +type AgentConnection = { url: string; transport: 'websocket' } +``` + +- `pending` — spinning up. `running` — usable (includes a Haystack pipeline that + auto-idled; it wakes on query). `failed` — deploy failed. `gone` — deleted / + undeployed on the host; unusable, redeploy or remove. +- `connection` is the deterministic chat endpoint. It's returned **up front on + deploy** (even while `pending`) so the client can persist the agent without + waiting. `status` returns it only once `running`. + +**`deploymentId` codec** (`backend/src/agents/deployment-id.ts`): +`":"`, split on the **first** `:` so the ref may itself contain +colons. Examples: `haystack:tb-my-agent-lx9f2` and `openclaw:e2b:iq8fy72dt...` +(here provider=`openclaw`, ref=`e2b:iq8fy72dt...`). + +## Descriptors + +A descriptor is a small tree: `steps[] → fields[] → widget`. A spec is the flat +`{ fieldKey: value }` map the user fills in. + +```ts +type AgentDescriptor = { + id: string // unique catalog entry + provider: string // registry key → which provider handles deploy/status + name: string + description: string | null + icon: string | null + schemaVersion: number // bump on any field change; guards stale submissions + action: 'deploy' | 'connect' + steps: { id; title; description?; fields: AgentField[] }[] +} + +type AgentField = { + key: string + label: string + widget: 'text' | 'password' | 'textarea' | 'select' | 'option-cards' | 'gallery' | 'file-upload' + required?: boolean + placeholder?: string + helpText?: string + default?: string | string[] + visibleWhen?: { field: string; equals: string } // conditional field + source?: // options for select/cards/gallery + { kind: 'inline'; options: { value; label; description?; icon? }[] } | { kind: 'fetched'; sourceId: string } // resolved live by the FE (see note) + multiple?: boolean // collects a list (always true for gallery) + maxLength?: number +} +``` + +Notes: + +- The **widget set is closed.** The renderer only knows those seven; you can't + ship a new widget from the backend. Adding one is a frontend change. +- **`visibleWhen`** shows a field only when another field currently equals a + value — e.g. reveal "Custom model" only when `mode === 'byo'`. +- **`source: 'fetched'`** is defined in the contract but **not yet wired** — + there is no `/agents/:id/sources/:sourceId` route today. Use `'inline'` until + it lands. (It's the future home for live option lists like Haystack indexes.) +- **One-click:** if a descriptor's defaults already satisfy every visible + required field, the UI can deploy it in a single click (`isOneClickEligible`). + OpenClaw's only field is optional with a default → one-click. + +### The two live descriptors + +**Haystack** (`backend/src/haystack/provider.ts`) — curated: the owner fixes the +pipeline template via env, so the user only names the agent. + +```ts +{ + id: 'haystack', provider: 'haystack', + name: 'Haystack RAG agent', + description: 'Deploy a Haystack pipeline as a chat agent.', + icon: 'file-search', schemaVersion: 1, action: 'deploy', + steps: [{ id: 'basics', title: 'Name your agent', fields: [ + { key: 'name', label: 'Name', widget: 'text', required: true, maxLength: 60, placeholder: 'My research agent' }, + ]}], +} +``` + +**OpenClaw** (`backend/src/openclaw/provider.ts`) — owner-managed one-click: the +backend fixes the sandbox provider + model via env, so the single name field is +optional with a default. + +```ts +{ + id: 'openclaw', provider: 'openclaw', + name: 'OpenClaw', + description: 'Deploy a sandboxed OpenClaw coding agent.', + icon: 'terminal', schemaVersion: 1, action: 'deploy', + steps: [{ id: 'basics', title: 'Name your agent', fields: [ + { key: 'name', label: 'Name', widget: 'text', required: false, maxLength: 60, placeholder: 'OpenClaw', default: 'OpenClaw' }, + ]}], +} +``` + +## Worked examples + +Assume local dev (`http://localhost:8000`), `AGENT_DEPLOY=true`, and a valid +bearer token in `$TOKEN`. + +### List the catalog + +```bash +curl -s http://localhost:8000/v1/agents/catalog -H "Authorization: Bearer $TOKEN" +``` + +```json +{ + "version": "1", + "descriptors": [ + { + "id": "haystack", + "provider": "haystack", + "name": "Haystack RAG agent", + "schemaVersion": 1, + "action": "deploy", + "steps": [ + /* … */ + ] + }, + { + "id": "openclaw", + "provider": "openclaw", + "name": "OpenClaw", + "schemaVersion": 1, + "action": "deploy", + "steps": [ + /* … */ + ] + } + ] +} +``` + +A provider only appears when it's configured (Haystack needs +`HAYSTACK_TEMPLATE_PIPELINE`; OpenClaw needs `E2B_API_KEY` + `OPENCLAW_MODEL` + +`OPENCLAW_OPENROUTER_API_KEY`). + +### Deploy Haystack + +```bash +curl -s -X POST http://localhost:8000/v1/agents/deploy \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d '{ "descriptorId": "haystack", "schemaVersion": 1, "spec": { "name": "My research agent" } }' +``` + +```json +{ + "deploymentId": "haystack:tb-my-research-agent-lx9f2k", + "status": "running", + "connection": { + "url": "ws://localhost:8000/v1/haystack/ws?pipeline=tb-my-research-agent-lx9f2k", + "transport": "websocket" + } +} +``` + +The provider slugged the name into a `tb-`namespaced pipeline ref, cloned the +template YAML on Haystack, deployed it, and handed back the deterministic chat +URL. Haystack deploys are quick, so status is often `running` immediately. + +### Deploy OpenClaw + +```bash +curl -s -X POST http://localhost:8000/v1/agents/deploy \ + -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' \ + -d '{ "descriptorId": "openclaw", "schemaVersion": 1, "spec": { "name": "OpenClaw" } }' +``` + +```json +{ + "deploymentId": "openclaw:e2b:iq8fy72dtcwkosdu0jpu", + "status": "pending", + "connection": { + "url": "ws://localhost:8000/v1/openclaw/ws?instance=e2b%3Aiq8fy72dtcwkosdu0jpu", + "transport": "websocket" + } +} +``` + +Deploy returns **immediately** — it creates the E2B sandbox and launches the +agent daemon, but does _not_ block on the ~15–30s boot. Status starts `pending`; +the client polls until the sandbox's ACP endpoint answers. The sandbox is +stamped with the caller's `userId` in E2B metadata (the ownership source of +truth — no backend table). + +### Poll status + +```bash +# note the deploymentId is URL-encoded in the path +curl -s "http://localhost:8000/v1/agents/deployments/openclaw%3Ae2b%3Aiq8fy72dtcwkosdu0jpu" \ + -H "Authorization: Bearer $TOKEN" +``` + +```json +{ + "deploymentId": "openclaw:e2b:iq8fy72dtcwkosdu0jpu", + "status": "running", + "connection": { + "url": "ws://localhost:8000/v1/openclaw/ws?instance=e2b%3Aiq8fy72dtcwkosdu0jpu", + "transport": "websocket" + } +} +``` + +`status` re-derives the state live from the host each call: Haystack pipeline +status for Haystack; an ACP-readiness probe against the sandbox for OpenClaw. +For OpenClaw the lookup is **owner-gated** — a deployment id whose sandbox +metadata `userId` doesn't match the caller resolves to `gone`, never leaking +another user's sandbox. + +### Undeploy (delete) + +```bash +curl -s -X DELETE "http://localhost:8000/v1/agents/deployments/openclaw%3Ae2b%3Aiq8fy72dtcwkosdu0jpu" \ + -H "Authorization: Bearer $TOKEN" +``` + +```json +{ "deploymentId": "openclaw:e2b:iq8fy72dtcwkosdu0jpu", "status": "gone" } +``` + +The frontend calls this when the user deletes a deployed agent: it undeploys +first (Haystack deletes the `tb-*` pipeline clone; OpenClaw kills the sandbox, +owner-gated) and only soft-deletes the local agent row once the teardown is +accepted. A `5xx` keeps the row and surfaces the error (see the idempotency note +under [Contracts](#contracts)). + +## Adding a new provider + +Everything provider-specific lives in one module. Concretely (mirror +`backend/src/openclaw/`): + +1. **Implement `AgentProvider`** (`backend/src/agents/discovery.ts`): + + ```ts + export const createFooProvider = (deps: FooDeps = {}): AgentProvider => ({ + id: 'foo', // === descriptor.provider + list: () => [], // discovery descriptors, or [] if deploy-only + catalog: ({ settings }) => (isConfigured(settings) ? [fooDescriptor] : []), + deploy: async (spec, { request, settings, userId }) => { + const ref = await provisionOnHost(spec, settings, userId) + return { + deploymentId: encodeDeploymentId('foo', ref), + status: 'pending', + connection: { + url: buildWebSocketUrl(request, `foo/ws?instance=${encodeURIComponent(ref)}`), + transport: 'websocket', + }, + } + }, + status: async (ref, { request, settings, userId }) => { + /* live host lookup → DeploymentStatusResponse */ + }, + undeploy: async (ref, { settings, userId }) => { + await tearDownOnHost(ref, settings, userId) // owner-gate + idempotent (no-op if foreign/gone) + return { deploymentId: encodeDeploymentId('foo', ref), status: 'gone' } + }, + }) + ``` + + - `catalog`/`deploy`/`status`/`undeploy` are all optional. Omit `deploy`/`status`/ + `undeploy` for a discovery-only provider; omit `catalog` if it isn't deployable. + - Make `undeploy` **idempotent** — a foreign or already-gone ref should no-op and + still return `gone`, so the client can drop its local row. Reserve throwing for + a real host/network failure (the FE keeps the row and shows an error). + - Use `encodeDeploymentId(providerId, ref)` and `buildWebSocketUrl(request, …)` + (both from `@/agents`) so ids and URLs stay consistent and prod/dev + (`wss`/`ws`) both work. + - A `deploy` that returns `pending` + connection up front is the norm — it + keeps the endpoint fast and lets the status badge drive readiness. + +2. **Register + mount.** Register in your route module's factory + (`registerAgentProvider(createFooProvider(deps))`) — it's idempotent on `id`. + Then mount that route group in `backend/src/index.ts`: + + ```ts + .use(createFooRoutes(settings, auth)) + ``` + + The generic `/v1/agents/*` endpoints pick it up automatically — no edits + there. + +3. **Add its chat wire** if it needs one — a `WS /v1/foo/ws` route that + authenticates with `authorizeWsBearer` and pipes ACP frames to the host. Copy + `backend/src/openclaw/routes.ts` (dumb relay, sandbox already speaks ACP) or + `backend/src/haystack/routes.ts` (translates ACP ↔ the host's own protocol). + +4. **Add settings** in `backend/src/config/settings.ts` (schema + env mapping), + the test fixture in `backend/src/test-utils/settings.ts`, and document the env + vars in `backend/.env.example`. Gate `catalog()` on them so an unconfigured + provider stays invisible. + +That's it — the frontend renders your descriptor and drives deploy/status/undeploy +with zero provider knowledge. + +## Adding a field to a descriptor + +To collect one more input (say a BYO model for Haystack): + +1. **Add the field** to a step in the descriptor, using an existing widget: + + ```ts + { key: 'model', label: 'Model', widget: 'select', required: true, + source: { kind: 'inline', options: [ + { value: 'gpt-4o', label: 'GPT-4o' }, + { value: 'claude-sonnet', label: 'Claude Sonnet' }, + ]}} + ``` + +2. **Bump `schemaVersion`** on that descriptor. A deploy submitted against an + old version is rejected with `409 SCHEMA_VERSION_MISMATCH`, so stale clients + fail loudly instead of silently deploying with missing fields. + +3. **Read it in `deploy()`** from `spec.model` and pass it to the host. + +You do **not** write a validator: the backend rebuilds a zod schema from the +descriptor (`specSchemaForDescriptor`) and re-validates every submitted spec +before `deploy()` runs — required-ness is enforced only for fields that are +_visible_ under the submitted values (so a hidden `visibleWhen` field is never +demanded). The frontend uses the same rebuilt schema via `zodResolver`, so both +ends agree with no duplicated rules. The client is never trusted; the server +re-validation is the authority. + +For conditional inputs, add `visibleWhen`; for a list input, set `multiple: true` +(or use the `gallery` widget). To source options from a live host, you'll need +the `source: 'fetched'` endpoint, which isn't implemented yet (see the +descriptor note above). + +## Current limitations + +- **`source: 'fetched'` is contract-only** — no resolver endpoint yet; use + `'inline'` options. +- **`deploymentIdForAgent` (frontend) reconstructs the id by parsing the stored + URL** — `?pipeline=` → `haystack:`, `?instance=` → `openclaw:`. This + works for both providers today, but a new managed provider that uses a + different query param must be added here until the `provider` field lands on the + agents row and the id can be read off it directly (see the TODO in + `src/api/agent-deploy.ts`). + +## Source map + +| Concern | File | +| ------------------------------------- | ----------------------------------------------------------- | +| Wire contract + validator | `shared/agent-descriptors.ts` | +| Discovery response shape | `shared/acp-types.ts` | +| Generic endpoints | `backend/src/agents/routes.ts` | +| Provider seam + registry + URL helper | `backend/src/agents/discovery.ts` | +| Deployment id codec | `backend/src/agents/deployment-id.ts` | +| Haystack provider / chat wire | `backend/src/haystack/{provider,routes}.ts` | +| OpenClaw provider / relay / E2B | `backend/src/openclaw/{provider,routes,relay,e2b}.ts` | +| WS bearer auth | `backend/src/auth/ws-bearer-auth.ts`, `shared/ws-bearer.ts` | +| Frontend deploy client | `src/api/agent-deploy.ts` | +| Frontend transport routing | `src/acp/transports/index.ts` | diff --git a/shared/agent-descriptors.test.ts b/shared/agent-descriptors.test.ts new file mode 100644 index 000000000..7e8cdc347 --- /dev/null +++ b/shared/agent-descriptors.test.ts @@ -0,0 +1,198 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { + agentCatalogResponseSchema, + agentDescriptorSchema, + allFields, + defaultSpec, + deployRequestSchema, + deployResponseSchema, + deploymentStatusResponseSchema, + fieldSchema, + isFieldVisible, + isOneClickEligible, + optionSourceSchema, + schemaVersionMismatch, + specSchemaForDescriptor, + visibleFields, + widgetSchema, + type AgentDescriptor, +} from './agent-descriptors.ts' + +const makeDescriptor = (overrides: Partial = {}): AgentDescriptor => ({ + id: 'haystack', + provider: 'haystack', + name: 'Haystack', + description: 'Deploy a Deepset pipeline', + icon: 'file-search', + schemaVersion: 1, + action: 'deploy', + steps: [ + { + id: 'basics', + title: 'Basics', + fields: [ + { key: 'name', label: 'Name', widget: 'text', required: true, maxLength: 40 }, + { key: 'mode', label: 'Mode', widget: 'select', default: 'curated' }, + { + key: 'apiKey', + label: 'API key', + widget: 'password', + required: true, + visibleWhen: { field: 'mode', equals: 'byo' }, + }, + { key: 'files', label: 'Files', widget: 'gallery' }, + ], + }, + ], + ...overrides, +}) + +describe('widget + field schemas', () => { + it('accepts the closed widget set', () => { + expect(widgetSchema.parse('option-cards')).toBe('option-cards') + }) + + it('rejects an unknown widget', () => { + expect(() => widgetSchema.parse('slider')).toThrow() + }) + + it('parses a minimal field', () => { + expect(fieldSchema.parse({ key: 'name', label: 'Name', widget: 'text' }).key).toBe('name') + }) + + it('parses inline and fetched option sources', () => { + expect(optionSourceSchema.parse({ kind: 'inline', options: [{ value: 'a', label: 'A' }] }).kind).toBe('inline') + expect(optionSourceSchema.parse({ kind: 'fetched', sourceId: 'indexes' }).kind).toBe('fetched') + }) + + it('rejects an option source with an unknown kind', () => { + expect(() => optionSourceSchema.parse({ kind: 'remote' })).toThrow() + }) +}) + +describe('descriptor + response schemas', () => { + it('round-trips a valid descriptor', () => { + expect(() => agentDescriptorSchema.parse(makeDescriptor())).not.toThrow() + }) + + it('rejects a descriptor missing provider', () => { + const { provider: _provider, ...rest } = makeDescriptor() + expect(() => agentDescriptorSchema.parse(rest)).toThrow() + }) + + it('parses the catalog envelope', () => { + const parsed = agentCatalogResponseSchema.parse({ version: '1', descriptors: [makeDescriptor()] }) + expect(parsed.descriptors).toHaveLength(1) + }) + + it('parses deploy request/response and deployment status', () => { + expect( + deployRequestSchema.parse({ descriptorId: 'haystack', schemaVersion: 1, spec: { name: 'x' } }).spec.name, + ).toBe('x') + expect( + deployResponseSchema.parse({ deploymentId: 'haystack:tb-x', status: 'pending', connection: null }).status, + ).toBe('pending') + const status = deploymentStatusResponseSchema.parse({ + deploymentId: 'haystack:tb-x', + status: 'running', + connection: { url: 'wss://h/v1/haystack/ws?pipeline=tb-x', transport: 'websocket' }, + }) + expect(status.connection?.url).toContain('wss://') + }) + + it('rejects an invalid deploy status', () => { + expect(() => deployResponseSchema.parse({ deploymentId: 'x', status: 'deployed' })).toThrow() + }) + + it('exposes the schema-version-mismatch code', () => { + expect(schemaVersionMismatch).toBe('SCHEMA_VERSION_MISMATCH') + }) +}) + +describe('field helpers', () => { + it('flattens fields across steps', () => { + expect(allFields(makeDescriptor()).map((f) => f.key)).toEqual(['name', 'mode', 'apiKey', 'files']) + }) + + it('treats fields without visibleWhen as always visible', () => { + const [name] = allFields(makeDescriptor()) + expect(isFieldVisible(name, {})).toBe(true) + }) + + it('honors visibleWhen equality', () => { + const apiKey = allFields(makeDescriptor()).find((f) => f.key === 'apiKey')! + expect(isFieldVisible(apiKey, { mode: 'curated' })).toBe(false) + expect(isFieldVisible(apiKey, { mode: 'byo' })).toBe(true) + }) + + it('lists only visible fields for the current spec', () => { + expect(visibleFields(makeDescriptor(), { mode: 'curated' }).map((f) => f.key)).toEqual(['name', 'mode', 'files']) + expect(visibleFields(makeDescriptor(), { mode: 'byo' }).map((f) => f.key)).toEqual([ + 'name', + 'mode', + 'apiKey', + 'files', + ]) + }) +}) + +describe('defaultSpec + one-click eligibility', () => { + it('seeds visible defaults only', () => { + expect(defaultSpec(makeDescriptor())).toEqual({ mode: 'curated' }) + }) + + it('is not one-click when a visible required field lacks a default', () => { + expect(isOneClickEligible(makeDescriptor())).toBe(false) + }) + + it('is one-click when every visible required field has a default', () => { + const descriptor = makeDescriptor({ + steps: [ + { + id: 'basics', + title: 'Basics', + fields: [{ key: 'name', label: 'Name', widget: 'text', required: true, default: 'My agent' }], + }, + ], + }) + expect(isOneClickEligible(descriptor)).toBe(true) + }) +}) + +describe('specSchemaForDescriptor (backend re-validation)', () => { + it('enforces required visible fields', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + expect(schema.safeParse({ mode: 'curated' }).success).toBe(false) + expect(schema.safeParse({ name: 'My agent', mode: 'curated' }).success).toBe(true) + }) + + it('skips required fields that are hidden by visibleWhen', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + // apiKey is required but hidden while mode=curated + expect(schema.safeParse({ name: 'My agent', mode: 'curated' }).success).toBe(true) + // once mode=byo, apiKey becomes required + expect(schema.safeParse({ name: 'My agent', mode: 'byo' }).success).toBe(false) + expect(schema.safeParse({ name: 'My agent', mode: 'byo', apiKey: 'sk-1' }).success).toBe(true) + }) + + it('strips unknown keys', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + const parsed = schema.parse({ name: 'My agent', mode: 'curated', bogus: 'x' }) + expect('bogus' in parsed).toBe(false) + }) + + it('enforces maxLength', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + expect(schema.safeParse({ name: 'x'.repeat(41), mode: 'curated' }).success).toBe(false) + }) + + it('shapes gallery/multiple fields as arrays and rejects scalars', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + expect(schema.safeParse({ name: 'My agent', mode: 'curated', files: ['a', 'b'] }).success).toBe(true) + expect(schema.safeParse({ name: 'My agent', mode: 'curated', files: 'a' }).success).toBe(false) + }) +}) diff --git a/shared/agent-descriptors.ts b/shared/agent-descriptors.ts new file mode 100644 index 000000000..d1057930c --- /dev/null +++ b/shared/agent-descriptors.ts @@ -0,0 +1,255 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Wire contract for the descriptor-driven agent creation flow (THU-743). + * + * A **descriptor** is "an agent creation form as data": the backend curates it, + * the frontend renders it over a closed widget registry, and the backend rebuilds + * a zod validator from the same descriptor to re-validate the submitted spec (the + * client is never trusted). Both ends import this file — drift here is silent + * breakage, so the shapes and the validator live in one place. + * + * Descriptors are deliberately NOT JSON Schema: the model is steps → fields → + * widgets, with `visibleWhen` conditional fields and `source`-backed options. A + * spec is the flat map of field key → value the user fills in. + */ + +import { z } from 'zod' + +/** Returned to the client (409) when a deploy carries a stale `schemaVersion`. */ +export const schemaVersionMismatch = 'SCHEMA_VERSION_MISMATCH' + +/** A single spec value: a scalar (`text`/`select`/…) or a list (`gallery`/multi). */ +export const specValueSchema = z.union([z.string(), z.array(z.string())]) +export type SpecValue = z.infer + +/** A deploy spec: the flat map of field key → value collected from the form. */ +export const agentSpecSchema = z.record(z.string(), specValueSchema) +export type AgentSpec = z.infer + +/** The closed set of widgets the frontend renderer knows how to draw. */ +export const widgetSchema = z.enum(['text', 'password', 'textarea', 'select', 'option-cards', 'gallery', 'file-upload']) +export type Widget = z.infer + +/** One selectable option for `select` / `option-cards` / `gallery` widgets. */ +export const optionSchema = z.object({ + value: z.string(), + label: z.string(), + description: z.string().optional(), + icon: z.string().nullable().optional(), +}) +export type AgentFieldOption = z.infer + +/** + * Where a field's options come from. `inline` ships them in the descriptor; + * `fetched` names a `sourceId` the frontend resolves at + * `GET /agents/:descriptorId/sources/:sourceId` (e.g. live Deepset indexes). + */ +export const optionSourceSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('inline'), options: z.array(optionSchema) }), + z.object({ kind: z.literal('fetched'), sourceId: z.string() }), +]) +export type OptionSource = z.infer + +/** Conditional visibility: show this field only when `field` currently equals `equals`. */ +export const visibleWhenSchema = z.object({ + field: z.string(), + equals: z.string(), +}) +export type VisibleWhen = z.infer + +/** A single form field bound to one spec key. */ +export const fieldSchema = z.object({ + key: z.string(), + label: z.string(), + widget: widgetSchema, + required: z.boolean().optional(), + placeholder: z.string().optional(), + helpText: z.string().optional(), + default: specValueSchema.optional(), + visibleWhen: visibleWhenSchema.optional(), + source: optionSourceSchema.optional(), + /** Whether the field collects multiple values (always true for `gallery`). */ + multiple: z.boolean().optional(), + maxLength: z.number().int().positive().optional(), +}) +export type AgentField = z.infer + +/** A group of fields shown together. A single-field/single-option step may be skipped in the UI. */ +export const stepSchema = z.object({ + id: z.string(), + title: z.string(), + description: z.string().optional(), + fields: z.array(fieldSchema), +}) +export type AgentStep = z.infer + +/** What submitting the descriptor does. Phase 1 ships `deploy`; `connect` is the + * future home for the URL-based custom-agent flow. */ +export const agentActionSchema = z.enum(['deploy', 'connect']) +export type AgentAction = z.infer + +/** + * A curated agent creation form. `id` is the unique catalog entry; `provider` is + * the registry key (the "kind": haystack/openclaw/…) the backend dispatches the + * deploy to. `schemaVersion` guards against a client submitting against a stale + * descriptor. + */ +export const agentDescriptorSchema = z.object({ + id: z.string(), + provider: z.string(), + name: z.string(), + description: z.string().nullable(), + icon: z.string().nullable(), + schemaVersion: z.number().int().nonnegative(), + action: agentActionSchema, + steps: z.array(stepSchema), +}) +export type AgentDescriptor = z.infer + +/** Envelope for `GET /agents/catalog`. */ +export const agentCatalogResponseSchema = z.object({ + version: z.literal('1'), + descriptors: z.array(agentDescriptorSchema), +}) +export type AgentCatalogResponse = z.infer + +/** + * Normalized deploy lifecycle state (host statuses map onto these). `pending` is + * spinning up; `running` is usable (a deployed pipeline, including one Deepset + * has auto-idled — it wakes on query); `failed` is a failed deploy; `gone` is + * deleted or undeployed on the host (unusable — redeploy or remove it). + */ +export const deployStatusSchema = z.enum(['pending', 'running', 'failed', 'gone']) +export type DeployStatus = z.infer + +/** How the frontend connects to a deployed agent once it is running. */ +export const agentConnectionSchema = z.object({ + url: z.string(), + transport: z.literal('websocket'), +}) +export type AgentConnection = z.infer + +/** Body of `POST /agents/deploy`. */ +export const deployRequestSchema = z.object({ + descriptorId: z.string(), + schemaVersion: z.number().int().nonnegative(), + spec: agentSpecSchema, +}) +export type DeployRequest = z.infer + +/** Response of `POST /agents/deploy`. `deploymentId` encodes `provider:ref`. + * `connection` is the (deterministic) chat endpoint, returned up front so the + * client can persist the agent immediately without polling. Null when a provider + * can't resolve it at deploy time. */ +export const deployResponseSchema = z.object({ + deploymentId: z.string(), + status: deployStatusSchema, + connection: agentConnectionSchema.nullable(), +}) +export type DeployResponse = z.infer + +/** Response of `GET /agents/deployments/:id` — status polled live from the host. */ +export const deploymentStatusResponseSchema = z.object({ + deploymentId: z.string(), + status: deployStatusSchema, + detail: z.string().nullable().optional(), + connection: agentConnectionSchema.nullable().optional(), +}) +export type DeploymentStatusResponse = z.infer + +/** Response of `DELETE /agents/deployments/:id`. `status` is `gone` once teardown + * is triggered on the host — the endpoint does not block on the full unmount. */ +export const undeployResponseSchema = z.object({ + deploymentId: z.string(), + status: deployStatusSchema, +}) +export type UndeployResponse = z.infer + +/** True when a spec value carries a usable (non-empty) value. */ +const hasValue = (value: SpecValue | undefined): boolean => + Array.isArray(value) ? value.length > 0 : typeof value === 'string' && value.trim().length > 0 + +/** Whether a field collects a list rather than a scalar. */ +const isMultiValue = (field: AgentField): boolean => field.multiple === true || field.widget === 'gallery' + +/** Every field across every step, flattened in order. */ +export const allFields = (descriptor: AgentDescriptor): AgentField[] => descriptor.steps.flatMap((step) => step.fields) + +/** A field is visible when it has no `visibleWhen`, or the guarded field's current + * value equals the guard. */ +export const isFieldVisible = (field: AgentField, spec: AgentSpec): boolean => { + if (!field.visibleWhen) { + return true + } + return spec[field.visibleWhen.field] === field.visibleWhen.equals +} + +/** The fields currently visible given `spec`. */ +export const visibleFields = (descriptor: AgentDescriptor, spec: AgentSpec): AgentField[] => + allFields(descriptor).filter((field) => isFieldVisible(field, spec)) + +/** + * Build the initial spec from field defaults. Visibility is evaluated against the + * full set of defaults, then hidden fields are dropped so we never seed a value + * the user can't see (which the backend re-validation would reject). + */ +export const defaultSpec = (descriptor: AgentDescriptor): AgentSpec => { + const withAllDefaults: AgentSpec = {} + for (const field of allFields(descriptor)) { + if (field.default !== undefined) { + withAllDefaults[field.key] = field.default + } + } + const result: AgentSpec = {} + for (const field of allFields(descriptor)) { + if (field.default !== undefined && isFieldVisible(field, withAllDefaults)) { + result[field.key] = field.default + } + } + return result +} + +/** + * True when the descriptor's default spec already satisfies every visible required + * field — i.e. it can be deployed in one click with no user input. + */ +export const isOneClickEligible = (descriptor: AgentDescriptor): boolean => { + const spec = defaultSpec(descriptor) + return visibleFields(descriptor, spec) + .filter((field) => field.required) + .every((field) => hasValue(spec[field.key])) +} + +/** + * Rebuild a zod validator from a descriptor to re-validate a submitted spec. The + * frontend uses it via `zodResolver`; the backend re-runs it as the authority. + * Unknown keys are stripped, values are shaped per widget (scalar vs list), and + * `required` is enforced only for fields that are visible under the given values. + */ +export const specSchemaForDescriptor = (descriptor: AgentDescriptor) => { + const fields = allFields(descriptor) + const shape: Record = {} + for (const field of fields) { + const scalar = field.maxLength ? z.string().max(field.maxLength) : z.string() + const value = isMultiValue(field) ? z.array(z.string()) : scalar + shape[field.key] = value.optional() + } + return z.object(shape).superRefine((spec, ctx) => { + const typed = spec as AgentSpec + for (const field of fields) { + if (!field.required || !isFieldVisible(field, typed)) { + continue + } + if (!hasValue(typed[field.key])) { + ctx.addIssue({ + code: 'custom', + path: [field.key], + message: `${field.label} is required`, + }) + } + } + }) +} diff --git a/src/acp/connection-test.test.ts b/src/acp/connection-test.test.ts index 8143cafe7..9ca4f285d 100644 --- a/src/acp/connection-test.test.ts +++ b/src/acp/connection-test.test.ts @@ -14,9 +14,15 @@ import { act } from '@testing-library/react' import { describe, expect, it, mock } from 'bun:test' import { getClock } from '@/testing-library' import type { Agent as AcpSdkAgent, Client, InitializeRequest, InitializeResponse } from '@agentclientprotocol/sdk' +import { encodeWsBearer } from '@shared/ws-bearer' import { testAcpConnection, type TestAcpConnectionResult } from './connection-test' import type { AcpTransport } from './types' +/** Read the `webSocketFactory` a mocked `openTransport` was invoked with. */ +const factoryPassedTo = (openTransport: unknown): ((url: string) => unknown) | undefined => + (openTransport as { mock: { calls: Array<[{ webSocketFactory?: (url: string) => unknown }]> } }).mock.calls[0][0] + .webSocketFactory + type FakeAgentCapabilities = NonNullable const buildFakeDeps = (opts: { @@ -177,4 +183,41 @@ describe('testAcpConnection', () => { expect(resolved).toEqual({ success: false, error: 'Connection timed out' }) expect(close).toHaveBeenCalledTimes(1) }) + + it('attaches the carrier + bearer subprotocol for managed-acp agents', async () => { + const { openTransport, FakeConnection } = buildFakeDeps({ capabilities: {} }) + const created: Array<{ url: string; protocols?: string | string[] }> = [] + const originalWs = globalThis.WebSocket + class FakeWs { + constructor(url: string, protocols?: string | string[]) { + created.push({ url, protocols }) + } + } + globalThis.WebSocket = FakeWs as unknown as typeof WebSocket + try { + await testAcpConnection({ + url: 'wss://relay.test/v1/openclaw/ws?instance=e2b%3Asbx-1', + agentType: 'managed-acp', + getAuthToken: () => 'tok-123', + openTransport: openTransport as never, + ClientSideConnection: FakeConnection as never, + }) + factoryPassedTo(openTransport)?.('wss://relay.test/ws') + expect(created[0]?.protocols).toEqual(['thunderbolt.v1', `thunderbolt.bearer.${encodeWsBearer('tok-123')}`]) + } finally { + globalThis.WebSocket = originalWs + } + }) + + it('connects natively (no bearer factory) for remote-acp agents', async () => { + const { openTransport, FakeConnection } = buildFakeDeps({ capabilities: {} }) + await testAcpConnection({ + url: 'wss://custom.test/ws', + agentType: 'remote-acp', + openTransport: openTransport as never, + ClientSideConnection: FakeConnection as never, + }) + // No explicit factory → openWebSocketTransport builds its native default. + expect(factoryPassedTo(openTransport)).toBeUndefined() + }) }) diff --git a/src/acp/connection-test.ts b/src/acp/connection-test.ts index c9066c3ee..d5a628dbf 100644 --- a/src/acp/connection-test.ts +++ b/src/acp/connection-test.ts @@ -8,18 +8,28 @@ * `initialize` handshake, and report success (with the agent's capabilities) or * a user-facing error. * - * Unlike {@link connectAcpAdapter}, this opens the endpoint via - * `openWebSocketTransport` *directly* — skipping the managed-ACP bearer / - * subprotocol routing in `openTransport`. Remote custom agents carry their own - * auth (or none), so the probe never needs a backend credential. The transport - * is always torn down in a `finally`, even on timeout, so no socket leaks. + * Opens the endpoint via `openWebSocketTransport` *directly* — not the full + * `openTransport` proxy routing. For `managed-acp` agents (deployed catalog + * agents whose endpoint is the backend relay) it attaches the same + * `thunderbolt.bearer` subprotocol the live transport uses, so the relay + * authenticates the probe; without it the relay closes the socket as + * unauthorized. Remote custom agents carry their own auth (or none), so they + * connect natively with no backend credential. The transport is always torn + * down in a `finally`, even on timeout, so no socket leaks. */ import type { Agent as AcpSdkAgent, ClientSideConnection, Client } from '@agentclientprotocol/sdk' import { ClientSideConnection as ClientSideConnectionImpl } from '@agentclientprotocol/sdk' +import { getAuthToken } from '@/lib/auth-token' import type { AgentCapabilities } from '@/types/acp' +import type { AgentType } from '@shared/acp-types' import { adaptCapabilities } from './acp-adapter' -import { openWebSocketTransport, type WebSocketFactory } from './transports/websocket' +import { + managedAcpSubprotocols, + openWebSocketTransport, + type WebSocketFactory, + type WebSocketLike, +} from './transports/websocket' import type { AcpTransport } from './types' const protocolVersion = 1 @@ -44,10 +54,15 @@ type ClientSideConnectionCtor = new ( export type TestAcpConnectionOptions = { url: string + /** The agent's type. `managed-acp` endpoints are the backend relay and need the + * `thunderbolt.bearer` subprotocol; omitted / `remote-acp` probes connect natively. */ + agentType?: AgentType /** Caller-owned signal — aborting tears the probe's transport down. */ signal?: AbortSignal - /** Test seam — production omits and the factory builds a native WebSocket. */ + /** Test seam — production omits and the factory is derived from `agentType`. */ webSocketFactory?: WebSocketFactory + /** Test seam — production reads the in-memory `getAuthToken()`. */ + getAuthToken?: () => string | null /** Override the handshake timeout. Defaults to 10s, matching the model flow. */ timeoutMs?: number /** Test seam — DI the transport opener. */ @@ -56,6 +71,22 @@ export type TestAcpConnectionOptions = { ClientSideConnection?: ClientSideConnectionCtor } +/** + * Pick the WebSocket factory for the probe. An explicit `webSocketFactory` + * (tests) wins; otherwise `managed-acp` targets get a bearer-carrying socket and + * everything else falls through to `openWebSocketTransport`'s native default. + */ +const resolveProbeFactory = (opts: TestAcpConnectionOptions): WebSocketFactory | undefined => { + if (opts.webSocketFactory) { + return opts.webSocketFactory + } + if (opts.agentType === 'managed-acp') { + const protocols = managedAcpSubprotocols((opts.getAuthToken ?? getAuthToken)()) + return (url) => new WebSocket(url, protocols) as unknown as WebSocketLike + } + return undefined +} + export type TestAcpConnectionResult = | { success: true; capabilities: AgentCapabilities } | { success: false; error: string } @@ -92,7 +123,7 @@ export const testAcpConnection = async (opts: TestAcpConnectionOptions): Promise transport = await openTransport({ url: opts.url, signal, - webSocketFactory: opts.webSocketFactory, + webSocketFactory: resolveProbeFactory(opts), }) const connection = new ConnectionCtor(() => probeClient, transport.stream) diff --git a/src/acp/transports/index.ts b/src/acp/transports/index.ts index 2be55c252..c93870c57 100644 --- a/src/acp/transports/index.ts +++ b/src/acp/transports/index.ts @@ -34,10 +34,9 @@ import { isTauri } from '@/lib/platform' import { computeEffectiveProxyEnabled, createProxyWebSocket } from '@/lib/proxy-fetch' import { useLocalSettingsStore } from '@/stores/local-settings-store' import type { AgentType } from '@shared/acp-types' -import { encodeWsBearer, wsBearerSubprotocolPrefix, wsCarrierSubprotocol } from '@shared/ws-bearer' import { openIrohTransport } from '../iroh/iroh-transport' import type { AcpTransport } from '../types' -import { openWebSocketTransport, type WebSocketFactory, type WebSocketLike } from './websocket' +import { managedAcpSubprotocols, openWebSocketTransport, type WebSocketFactory, type WebSocketLike } from './websocket' export type OpenTransportInputs = { url: string @@ -134,10 +133,7 @@ const resolveManagedAcpFactory = (inputs: OpenTransportInputs): WebSocketFactory if (!inputs.httpClient) { return nativeWebSocketFactory } - const token = (inputs.getAuthToken ?? getAuthToken)() - const protocols = token - ? [wsCarrierSubprotocol, `${wsBearerSubprotocolPrefix}${encodeWsBearer(token)}`] - : [wsCarrierSubprotocol] + const protocols = managedAcpSubprotocols((inputs.getAuthToken ?? getAuthToken)()) return (url) => new WebSocket(url, protocols) as unknown as WebSocketLike } diff --git a/src/acp/transports/websocket.ts b/src/acp/transports/websocket.ts index db44b7d06..a206763f2 100644 --- a/src/acp/transports/websocket.ts +++ b/src/acp/transports/websocket.ts @@ -23,6 +23,7 @@ import type { AnyMessage, Stream } from '@agentclientprotocol/sdk' import { getPlatform } from '@/lib/platform' +import { encodeWsBearer, wsBearerSubprotocolPrefix, wsCarrierSubprotocol } from '@shared/ws-bearer' import type { AcpTransport } from '../types' export const normalCloseCode = 1000 @@ -77,6 +78,16 @@ export type WebSocketEventMap = { export type WebSocketFactory = (url: string) => WebSocketLike +/** + * Subprotocol list a managed-ACP WebSocket offers on the upgrade: the carrier + * (`thunderbolt.v1`, echoed by the server) plus, when a bearer token is present, + * the auth entry the backend validates in `open()`. Shared by the live transport + * ({@link openTransport}) and the connection probe ({@link testAcpConnection}) so + * both authenticate identically against the managed relay. + */ +export const managedAcpSubprotocols = (token: string | null): string[] => + token ? [wsCarrierSubprotocol, `${wsBearerSubprotocolPrefix}${encodeWsBearer(token)}`] : [wsCarrierSubprotocol] + export type IsTauriIosFn = () => boolean const defaultBackoffMs = (attempt: number): number => 1000 * Math.pow(2, attempt - 1) diff --git a/src/api/agent-deploy.test.ts b/src/api/agent-deploy.test.ts new file mode 100644 index 000000000..f17187183 --- /dev/null +++ b/src/api/agent-deploy.test.ts @@ -0,0 +1,142 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { HttpError, type HttpClient } from '@/lib/http' +import type { AgentCatalogResponse, DeployRequest, DeployResponse } from '@shared/agent-descriptors' +import { + deployAgent, + deploymentIdForAgent, + fetchAgentCatalog, + getDeploymentStatus, + undeployAgent, +} from './agent-deploy' + +type Call = { url: string; method: string; options?: { json?: unknown } } + +/** Build a fake HttpClient whose get/post/delete resolve to `body`, recording calls. */ +const makeClient = (body: unknown, opts: { throwStatus?: number } = {}) => { + const calls: Call[] = [] + const respond = (method: string) => (url: string, options?: { json?: unknown }) => { + calls.push({ url, method, options }) + return { + json: async () => { + if (opts.throwStatus) { + throw new HttpError(new Response(null, { status: opts.throwStatus })) + } + return body + }, + } + } + const client = { + get: respond('GET'), + post: respond('POST'), + delete: respond('DELETE'), + } as unknown as HttpClient + return { client, calls } +} + +const cloudUrl = 'https://api.test/v1' + +const catalogBody: AgentCatalogResponse = { + version: '1', + descriptors: [ + { + id: 'haystack', + provider: 'haystack', + name: 'Haystack', + description: null, + icon: null, + schemaVersion: 1, + action: 'deploy', + steps: [{ id: 'basics', title: 'Basics', fields: [{ key: 'name', label: 'Name', widget: 'text' }] }], + }, + ], +} + +describe('fetchAgentCatalog', () => { + it('returns the descriptors from the catalog envelope', async () => { + const { client, calls } = makeClient(catalogBody) + const descriptors = await fetchAgentCatalog(cloudUrl, client) + expect(calls[0].url).toBe('https://api.test/v1/agents/catalog') + expect(descriptors.map((d) => d.id)).toEqual(['haystack']) + }) + + it('returns [] when the feature is disabled (404)', async () => { + const { client } = makeClient(null, { throwStatus: 404 }) + expect(await fetchAgentCatalog(cloudUrl, client)).toEqual([]) + }) + + it('rethrows non-404 errors', async () => { + const { client } = makeClient(null, { throwStatus: 500 }) + await expect(fetchAgentCatalog(cloudUrl, client)).rejects.toThrow() + }) +}) + +describe('deployAgent', () => { + it('POSTs the request and parses the response', async () => { + const body: DeployResponse = { + deploymentId: 'haystack:tb-x', + status: 'pending', + connection: { url: 'wss://h/v1/haystack/ws?pipeline=tb-x', transport: 'websocket' }, + } + const { client, calls } = makeClient(body) + const request: DeployRequest = { descriptorId: 'haystack', schemaVersion: 1, spec: { name: 'Bot' } } + const result = await deployAgent(cloudUrl, client, request) + expect(calls[0].url).toBe('https://api.test/v1/agents/deploy') + expect(calls[0].options?.json).toEqual(request) + expect(result).toEqual(body) + }) +}) + +describe('deploymentIdForAgent', () => { + it('derives : from a managed agent url', () => { + expect(deploymentIdForAgent({ type: 'managed-acp', url: 'wss://h/v1/haystack/ws?pipeline=tb-x' })).toBe( + 'haystack:tb-x', + ) + }) + + it('returns null for non-managed agents', () => { + expect(deploymentIdForAgent({ type: 'remote-acp', url: 'wss://h/v1/haystack/ws?pipeline=tb-x' })).toBeNull() + }) + + it('derives openclaw: from an instance url', () => { + expect(deploymentIdForAgent({ type: 'managed-acp', url: 'wss://h/v1/openclaw/ws?instance=e2b%3Asbx-1' })).toBe( + 'openclaw:e2b:sbx-1', + ) + }) + + it('returns null when the url carries no pipeline or instance ref', () => { + expect(deploymentIdForAgent({ type: 'managed-acp', url: 'wss://h/v1/haystack/ws' })).toBeNull() + }) + + it('returns null for a missing or malformed url', () => { + expect(deploymentIdForAgent({ type: 'managed-acp', url: null })).toBeNull() + expect(deploymentIdForAgent({ type: 'managed-acp', url: 'not a url' })).toBeNull() + }) +}) + +describe('undeployAgent', () => { + it('DELETEs the encoded deployment id and parses the response', async () => { + const { client, calls } = makeClient({ deploymentId: 'openclaw:e2b:sbx-1', status: 'gone' }) + const result = await undeployAgent(cloudUrl, client, 'openclaw:e2b:sbx-1') + expect(calls[0].method).toBe('DELETE') + expect(calls[0].url).toBe('https://api.test/v1/agents/deployments/openclaw%3Ae2b%3Asbx-1') + expect(result).toEqual({ deploymentId: 'openclaw:e2b:sbx-1', status: 'gone' }) + }) + + it('throws when the undeploy trigger fails', async () => { + const { client } = makeClient(null, { throwStatus: 500 }) + await expect(undeployAgent(cloudUrl, client, 'openclaw:e2b:sbx-1')).rejects.toThrow() + }) +}) + +describe('getDeploymentStatus', () => { + it('encodes the deployment id and parses the status', async () => { + const { client, calls } = makeClient({ deploymentId: 'haystack:tb-x', status: 'running', connection: null }) + const result = await getDeploymentStatus(cloudUrl, client, 'haystack:tb-x') + expect(calls[0].url).toBe('https://api.test/v1/agents/deployments/haystack%3Atb-x') + expect(result.status).toBe('running') + }) +}) diff --git a/src/api/agent-deploy.ts b/src/api/agent-deploy.ts new file mode 100644 index 000000000..2f56d9fd5 --- /dev/null +++ b/src/api/agent-deploy.ts @@ -0,0 +1,108 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Frontend client for the descriptor-driven agent deploy endpoints (THU-743). + * Uses the app's authenticated {@link HttpClient} and validates every response + * against the shared zod contract so a drifting backend surfaces loudly. + */ + +import { HttpError, type HttpClient } from '@/lib/http' +import type { Agent } from '@/types/acp' +import { + agentCatalogResponseSchema, + deployResponseSchema, + deploymentStatusResponseSchema, + undeployResponseSchema, + type AgentDescriptor, + type DeployRequest, + type DeployResponse, + type DeploymentStatusResponse, + type UndeployResponse, +} from '@shared/agent-descriptors' + +/** + * Fetch the deployable-agent catalog. Returns `[]` when the deploy feature is + * disabled server-side (the endpoint 404s) so callers can treat "off" and + * "nothing to deploy" identically. + */ +export const fetchAgentCatalog = async (cloudUrl: string, httpClient: HttpClient): Promise => { + try { + const data = await httpClient.get(`${cloudUrl}/agents/catalog`).json() + return agentCatalogResponseSchema.parse(data).descriptors + } catch (err) { + if (err instanceof HttpError && err.response.status === 404) { + return [] + } + throw err + } +} + +/** Start a deploy from a validated spec. Returns the deployment id + initial status. */ +export const deployAgent = async ( + cloudUrl: string, + httpClient: HttpClient, + request: DeployRequest, +): Promise => { + const data = await httpClient.post(`${cloudUrl}/agents/deploy`, { json: request }).json() + return deployResponseSchema.parse(data) +} + +/** + * Reconstruct a persisted agent's deployment id so its status can be polled or + * its deployment torn down after a reload — nothing device-local is stored. + * Managed agents encode their host ref in the connection url; the id is + * `:` (see the backend `deployment-id` codec): + * - Haystack → `…/haystack/ws?pipeline=` ⇒ `haystack:` + * - OpenClaw → `…/openclaw/ws?instance=` ⇒ `openclaw:` + * When the `provider` field lands on the agents row it will be read off the row + * instead of parsed from the url. Returns null for agents that carry no deployment. + */ +export const deploymentIdForAgent = (agent: Pick): string | null => { + if (agent.type !== 'managed-acp' || !agent.url) { + return null + } + try { + const params = new URL(agent.url).searchParams + const pipeline = params.get('pipeline') + if (pipeline) { + return `haystack:${pipeline}` + } + const instance = params.get('instance') + if (instance) { + return `openclaw:${instance}` + } + return null + } catch { + return null + } +} + +/** Poll a deployment's live status (fetched by the backend from the host). */ +export const getDeploymentStatus = async ( + cloudUrl: string, + httpClient: HttpClient, + deploymentId: string, +): Promise => { + const data = await httpClient + .get(`${cloudUrl}/agents/deployments/${encodeURIComponent(deploymentId)}`) + .json() + return deploymentStatusResponseSchema.parse(data) +} + +/** + * Trigger teardown of a deployment on its host. Resolves once the undeploy is + * accepted (not the full unmount); throws if the trigger fails so the caller + * keeps the local agent row and surfaces the error to the user. + */ +export const undeployAgent = async ( + cloudUrl: string, + httpClient: HttpClient, + deploymentId: string, +): Promise => { + const data = await httpClient + .delete(`${cloudUrl}/agents/deployments/${encodeURIComponent(deploymentId)}`) + .json() + return undeployResponseSchema.parse(data) +} diff --git a/src/api/config-store.ts b/src/api/config-store.ts index e29ce1e0b..008b71cf5 100644 --- a/src/api/config-store.ts +++ b/src/api/config-store.ts @@ -13,6 +13,9 @@ export type AppConfig = { * built-in agent shown, custom agents allowed. */ builtInAgentEnabled?: boolean allowCustomAgents?: boolean + /** Whether the descriptor-driven agent deploy flow (THU-743) is exposed. Opt-in: + * absent/false = hidden, unlike the sibling flags which default to enabled. */ + agentDeploy?: boolean /** Minimum semver string the server allows. Clients below this are hard-blocked * until they upgrade. Absent/empty = no enforcement. */ minAppVersion?: string @@ -51,3 +54,7 @@ export const selectBuiltInAgentEnabled = (config: AppConfig): boolean => config. /** Whether the UI offers adding custom agents. Absent config defaults to allowed. */ export const selectAllowCustomAgents = (config: AppConfig): boolean => config.allowCustomAgents !== false + +/** Whether the descriptor-driven agent deploy flow is exposed. Opt-in — absent + * config (offline/standalone) reads as OFF, unlike the sibling capability flags. */ +export const selectAgentDeploy = (config: AppConfig): boolean => config.agentDeploy === true diff --git a/src/components/agents/add-agent-body.test.tsx b/src/components/agents/add-agent-body.test.tsx new file mode 100644 index 000000000..2911477dd --- /dev/null +++ b/src/components/agents/add-agent-body.test.tsx @@ -0,0 +1,80 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { useConfigStore, type AppConfig } from '@/api/config-store' +import { resetTestDatabase, setupTestDatabase, teardownTestDatabase } from '@/dal/test-utils' +import { createMockAuthClient } from '@/test-utils/auth-client' +import { waitForElement } from '@/test-utils/powersync-reactivity-test' +import { createTestProvider } from '@/test-utils/test-provider' +import '@testing-library/jest-dom' +import { act, cleanup, render, screen } from '@testing-library/react' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'bun:test' +import type { ReactNode } from 'react' +import { AddAgentBody } from './add-agent-body' + +const authedSession = { user: { id: 'user-1', email: 'a@b.com', name: 'Alice', isAnonymous: false } } + +const catalogWith = (descriptors: unknown[]) => ({ version: '1', descriptors }) + +const haystackDescriptor = { + id: 'haystack', + provider: 'haystack', + name: 'Haystack RAG agent', + description: null, + icon: null, + schemaVersion: 1, + action: 'deploy', + steps: [{ id: 'basics', title: 'Basics', fields: [{ key: 'name', label: 'Name', widget: 'text' }] }], +} + +const renderBody = (mockResponse?: unknown) => { + const TestProvider = createTestProvider({ + authClient: createMockAuthClient({ session: authedSession }), + mockResponse, + }) + return render( {}} />, { + wrapper: ({ children }: { children: ReactNode }) => {children}, + }) +} + +describe('AddAgentBody', () => { + beforeAll(async () => { + await setupTestDatabase() + }) + + afterAll(async () => { + await teardownTestDatabase() + }) + + beforeEach(async () => { + await resetTestDatabase() + localStorage.clear() + }) + + afterEach(() => { + cleanup() + localStorage.clear() + useConfigStore.getState().updateConfig({} as AppConfig) + }) + + const setConfig = (config: AppConfig) => { + act(() => useConfigStore.getState().updateConfig(config)) + } + + it('renders the connect-only form when managed deploy is disabled', () => { + setConfig({ allowCustomAgents: true }) + renderBody() + + expect(screen.getByPlaceholderText('My Agent')).toBeInTheDocument() + expect(screen.queryByText('Connect custom agent')).not.toBeInTheDocument() + }) + + it('renders the catalog + connect list when managed deploy is enabled', async () => { + setConfig({ agentDeploy: true, allowCustomAgents: true }) + renderBody(catalogWith([haystackDescriptor])) + + expect(await waitForElement(() => screen.queryByText('Connect custom agent'))).toBeInTheDocument() + expect(screen.getByText('Haystack RAG agent')).toBeInTheDocument() + }) +}) diff --git a/src/components/agents/add-agent-body.tsx b/src/components/agents/add-agent-body.tsx new file mode 100644 index 000000000..49d7bb9f7 --- /dev/null +++ b/src/components/agents/add-agent-body.tsx @@ -0,0 +1,42 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The config-aware "Add agent" panel body, shared by every surface that offers + * adding an agent (the settings route and the chat picker's quick-create). It + * picks the panel by deployment capability so both surfaces stay in lockstep: + * when the managed-deploy flow is on it renders {@link AddAgentPanel} (catalog + + * inline connect), otherwise the connect-only {@link CreateAgentDetailPanel}. + * + * Renders its own `DetailPanel` header, so host it inside a `DetailPanelSurface`. + */ + +import { selectAgentDeploy, selectAllowCustomAgents, useConfigStore } from '@/api/config-store' +import { CreateAgentDetailPanel } from '@/components/settings/agents/create-agent-detail-panel' +import { AddAgentPanel } from './add-agent-panel' + +type AddAgentBodyProps = { + onClose: () => void + /** Test/DI override for reading this app's iroh NodeId (connect path). */ + loadAppNodeId?: () => Promise + /** Test/DI override for app NodeId self-enrollment (connect path). */ + enrollIroh?: () => Promise +} + +export const AddAgentBody = ({ onClose, loadAppNodeId, enrollIroh }: AddAgentBodyProps) => { + const agentDeploy = useConfigStore((state) => selectAgentDeploy(state.config)) + const allowCustomAgents = useConfigStore((state) => selectAllowCustomAgents(state.config)) + + if (agentDeploy) { + return ( + + ) + } + return +} diff --git a/src/components/agents/add-agent-panel.tsx b/src/components/agents/add-agent-panel.tsx new file mode 100644 index 000000000..ea90cf472 --- /dev/null +++ b/src/components/agents/add-agent-panel.tsx @@ -0,0 +1,173 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * The "Add agent" panel shown when the managed-deploy flow is enabled. It lists + * the deployable-agent catalog with "Connect custom agent" as the first entry, + * so both paths live in one list. Picking a catalog entry renders its descriptor + * form and deploys; picking Connect renders the custom-endpoint form inline. + * + * When the catalog is empty there is nothing to choose between, so the connect + * form is shown directly (assuming custom agents are allowed). + * + * A deploy returns the (deterministic) chat endpoint immediately, so the agent + * row is persisted right away and shows up in the list — the pipeline may still + * be spinning up on the host; live status badging is handled separately. + */ + +import { useQuery, useQueryClient } from '@tanstack/react-query' +import { Plug, Server } from 'lucide-react' +import { useState } from 'react' +import { v7 as uuidv7 } from 'uuid' + +import { testAcpConnection } from '@/acp' +import { deployAgent, fetchAgentCatalog } from '@/api/agent-deploy' +import { DetailPanel } from '@/components/detail-panel' +import { AddCustomAgentForm } from '@/components/settings/agents/add-custom-agent-form' +import { useAddCustomAgent } from '@/components/settings/agents/use-add-custom-agent' +import { SelectableCard } from '@/components/ui/selectable-card' +import { useAuth, useDatabase, useHttpClient } from '@/contexts' +import { createAgent } from '@/dal' +import { useLocalSettingsStore } from '@/stores/local-settings-store' +import type { AgentDescriptor, AgentSpec } from '@shared/agent-descriptors' +import { DescriptorForm } from './descriptor-form/descriptor-form' + +type AddAgentPanelProps = { + onClose: () => void + /** Whether the "Connect custom agent" path is offered alongside the catalog. */ + allowConnect: boolean + /** Test/DI override for reading this app's iroh NodeId (connect path). */ + loadAppNodeId?: () => Promise + /** Test/DI override for app NodeId self-enrollment (connect path). */ + enrollIroh?: () => Promise +} + +export const AddAgentPanel = ({ onClose, allowConnect, loadAppNodeId, enrollIroh }: AddAgentPanelProps) => { + const db = useDatabase() + const httpClient = useHttpClient() + const cloudUrl = useLocalSettingsStore((state) => state.cloudUrl) + const authClient = useAuth() + const queryClient = useQueryClient() + const { data: session } = authClient.useSession() + const userId = session?.user?.id ?? null + + const { data: catalog = [], isLoading } = useQuery({ + queryKey: ['agent-catalog'], + queryFn: () => fetchAgentCatalog(cloudUrl ?? '', httpClient), + enabled: Boolean(cloudUrl), + }) + + const [selected, setSelected] = useState(null) + const [showConnect, setShowConnect] = useState(false) + const [isDeploying, setIsDeploying] = useState(false) + const [error, setError] = useState(null) + + const addCustomAgent = useAddCustomAgent({ loadAppNodeId, enrollIroh }) + + const handleDeploy = async (spec: AgentSpec) => { + if (!cloudUrl || !userId || !selected) { + setError('Cannot deploy without an authenticated session.') + return + } + setIsDeploying(true) + setError(null) + try { + const result = await deployAgent(cloudUrl, httpClient, { + descriptorId: selected.id, + schemaVersion: selected.schemaVersion, + spec, + }) + if (!result.connection) { + setError('Deploy did not return a connection endpoint.') + setIsDeploying(false) + return + } + const name = typeof spec.name === 'string' && spec.name.trim().length > 0 ? spec.name.trim() : selected.name + await createAgent(db, { + id: uuidv7(), + name, + type: 'managed-acp', + transport: 'websocket', + url: result.connection.url, + enabled: 1, + userId, + }) + await queryClient.invalidateQueries({ queryKey: ['agents'] }) + onClose() + } catch (err) { + setError(err instanceof Error ? err.message : 'Deploy failed.') + setIsDeploying(false) + } + } + + // Nothing to choose between an empty catalog and Connect — go straight to the + // connect form (when custom agents are allowed). + const connectDirectly = !isLoading && catalog.length === 0 && allowConnect + + // Sub-steps reached from the list get a back affordance; a direct connect form + // (empty catalog) has no list to return to, so it stays close-only. + const onBack = selected ? () => setSelected(null) : showConnect ? () => setShowConnect(false) : undefined + + const renderBody = () => { + if (selected) { + return ( + + ) + } + if (showConnect || connectDirectly) { + return ( + + ) + } + if (isLoading) { + return

Loading…

+ } + if (catalog.length === 0) { + return ( +

No deployable agents are available.

+ ) + } + return ( +
+ {allowConnect && ( + setShowConnect(true)} + icon={} + title="Connect custom agent" + description="Connect your own agent by its ACP endpoint URL." + /> + )} + {catalog.map((descriptor) => ( + setSelected(descriptor)} + icon={} + title={descriptor.name} + description={descriptor.description ?? ''} + /> + ))} +
+ ) + } + + return ( + + {renderBody()} + + ) +} diff --git a/src/components/agents/descriptor-form/descriptor-form.test.tsx b/src/components/agents/descriptor-form/descriptor-form.test.tsx new file mode 100644 index 000000000..70891f331 --- /dev/null +++ b/src/components/agents/descriptor-form/descriptor-form.test.tsx @@ -0,0 +1,71 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import '@testing-library/jest-dom' +import { cleanup, fireEvent, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it, mock } from 'bun:test' +import type { AgentDescriptor } from '@shared/agent-descriptors' +import { DescriptorForm } from './descriptor-form' + +afterEach(cleanup) + +const conditionalDescriptor: AgentDescriptor = { + id: 'x', + provider: 'x', + name: 'X', + description: 'Deploy X', + icon: null, + schemaVersion: 1, + action: 'deploy', + steps: [ + { + id: 's', + title: 'S', + fields: [ + { key: 'name', label: 'Name', widget: 'text', required: true, placeholder: 'name' }, + { key: 'mode', label: 'Mode', widget: 'text', default: 'curated' }, + { key: 'apiKey', label: 'API key', widget: 'password', visibleWhen: { field: 'mode', equals: 'byo' } }, + ], + }, + ], +} + +const nameOnly: AgentDescriptor = { + ...conditionalDescriptor, + steps: [ + { + id: 's', + title: 'S', + fields: [{ key: 'name', label: 'Name', widget: 'text', required: true, placeholder: 'name' }], + }, + ], +} + +describe('DescriptorForm', () => { + it('renders visible fields and hides ones whose visibleWhen guard is unmet', () => { + render( {}} onCancel={() => {}} />) + expect(screen.getByText('Name')).toBeInTheDocument() + expect(screen.getByText('Mode')).toBeInTheDocument() + // mode defaults to 'curated', so the byo-only field is hidden. + expect(screen.queryByText('API key')).not.toBeInTheDocument() + }) + + it('shows the submit label and the descriptor description', () => { + render( {}} onCancel={() => {}} submitLabel="Deploy agent" />) + expect(screen.getByRole('button', { name: 'Deploy agent' })).toBeInTheDocument() + expect(screen.getByText('Deploy X')).toBeInTheDocument() + }) + + it('surfaces a submit-time error next to the buttons', () => { + render( {}} onCancel={() => {}} error="Deploy failed" />) + expect(screen.getByText('Deploy failed')).toBeInTheDocument() + }) + + it('fires onCancel from the Cancel button', () => { + const onCancel = mock(() => {}) + render( {}} onCancel={onCancel} />) + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })) + expect(onCancel).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/components/agents/descriptor-form/descriptor-form.tsx b/src/components/agents/descriptor-form/descriptor-form.tsx new file mode 100644 index 000000000..6cbc41b7c --- /dev/null +++ b/src/components/agents/descriptor-form/descriptor-form.tsx @@ -0,0 +1,137 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Renders an {@link AgentDescriptor} as a form and produces a validated spec. + * The descriptor owns the structure (steps → fields → widgets, `visibleWhen`); + * react-hook-form owns state, with `zodResolver(specSchemaForDescriptor)` — the + * same validator the backend re-runs — as the single source of validation truth. + */ + +import { useForm, type ControllerRenderProps, type Resolver } from 'react-hook-form' +import { zodResolver } from '@hookform/resolvers/zod' +import { + defaultSpec, + specSchemaForDescriptor, + visibleFields, + type AgentDescriptor, + type AgentField, + type AgentSpec, +} from '@shared/agent-descriptors' +import { Button } from '@/components/ui/button' +import { FormFooter } from '@/components/ui/form-footer' +import { Input } from '@/components/ui/input' +import { ResponsiveModalCancel } from '@/components/ui/responsive-modal' +import { Textarea } from '@/components/ui/textarea' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form' +import { submittableSpec } from './submittable-spec' + +type DescriptorFormProps = { + descriptor: AgentDescriptor + onSubmit: (spec: AgentSpec) => void | Promise + /** Dismiss the form (Cancel button); mirrors the connect form's onClose. */ + onCancel: () => void + submitLabel?: string + isSubmitting?: boolean + /** A submit-time error (e.g. a failed deploy) shown next to the buttons. */ + error?: string | null +} + +const inlineOptions = (field: AgentField) => (field.source?.kind === 'inline' ? field.source.options : []) + +/** Render the control for a single field's widget. Unknown widgets degrade to a note. */ +const FieldControl = ({ field, rhf }: { field: AgentField; rhf: ControllerRenderProps }) => { + const value = typeof rhf.value === 'string' ? rhf.value : '' + switch (field.widget) { + case 'textarea': + return