From 6621ada8baa5c9c4ecf03fbbf4772434fa9860cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Tue, 28 Jul 2026 18:14:36 -0300 Subject: [PATCH 01/28] feat(agents): add descriptor-driven agent contract --- shared/agent-descriptors.test.ts | 196 +++++++++++++++++++++++++ shared/agent-descriptors.ts | 238 +++++++++++++++++++++++++++++++ 2 files changed, 434 insertions(+) create mode 100644 shared/agent-descriptors.test.ts create mode 100644 shared/agent-descriptors.ts diff --git a/shared/agent-descriptors.test.ts b/shared/agent-descriptors.test.ts new file mode 100644 index 000000000..1bd022e0d --- /dev/null +++ b/shared/agent-descriptors.test.ts @@ -0,0 +1,196 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { + agentCatalogResponseSchema, + agentDescriptorSchema, + allFields, + defaultSpec, + deployRequestSchema, + deployResponseSchema, + deploymentStatusResponseSchema, + fieldSchema, + isFieldVisible, + isOneClickEligible, + optionSourceSchema, + schemaVersionMismatch, + specSchemaForDescriptor, + visibleFields, + widgetSchema, + type AgentDescriptor, +} from './agent-descriptors.ts' + +const makeDescriptor = (overrides: Partial = {}): AgentDescriptor => ({ + id: 'haystack', + provider: 'haystack', + name: 'Haystack', + description: 'Deploy a Deepset pipeline', + icon: 'file-search', + schemaVersion: 1, + action: 'deploy', + steps: [ + { + id: 'basics', + title: 'Basics', + fields: [ + { key: 'name', label: 'Name', widget: 'text', required: true, maxLength: 40 }, + { key: 'mode', label: 'Mode', widget: 'select', default: 'curated' }, + { + key: 'apiKey', + label: 'API key', + widget: 'password', + required: true, + visibleWhen: { field: 'mode', equals: 'byo' }, + }, + { key: 'files', label: 'Files', widget: 'gallery' }, + ], + }, + ], + ...overrides, +}) + +describe('widget + field schemas', () => { + it('accepts the closed widget set', () => { + expect(widgetSchema.parse('option-cards')).toBe('option-cards') + }) + + it('rejects an unknown widget', () => { + expect(() => widgetSchema.parse('slider')).toThrow() + }) + + it('parses a minimal field', () => { + expect(fieldSchema.parse({ key: 'name', label: 'Name', widget: 'text' }).key).toBe('name') + }) + + it('parses inline and fetched option sources', () => { + expect(optionSourceSchema.parse({ kind: 'inline', options: [{ value: 'a', label: 'A' }] }).kind).toBe('inline') + expect(optionSourceSchema.parse({ kind: 'fetched', sourceId: 'indexes' }).kind).toBe('fetched') + }) + + it('rejects an option source with an unknown kind', () => { + expect(() => optionSourceSchema.parse({ kind: 'remote' })).toThrow() + }) +}) + +describe('descriptor + response schemas', () => { + it('round-trips a valid descriptor', () => { + expect(() => agentDescriptorSchema.parse(makeDescriptor())).not.toThrow() + }) + + it('rejects a descriptor missing provider', () => { + const { provider: _provider, ...rest } = makeDescriptor() + expect(() => agentDescriptorSchema.parse(rest)).toThrow() + }) + + it('parses the catalog envelope', () => { + const parsed = agentCatalogResponseSchema.parse({ version: '1', descriptors: [makeDescriptor()] }) + expect(parsed.descriptors).toHaveLength(1) + }) + + it('parses deploy request/response and deployment status', () => { + expect( + deployRequestSchema.parse({ descriptorId: 'haystack', schemaVersion: 1, spec: { name: 'x' } }).spec.name, + ).toBe('x') + expect(deployResponseSchema.parse({ deploymentId: 'haystack:tb-x', status: 'pending' }).status).toBe('pending') + const status = deploymentStatusResponseSchema.parse({ + deploymentId: 'haystack:tb-x', + status: 'running', + connection: { url: 'wss://h/v1/haystack/ws?pipeline=tb-x', transport: 'websocket' }, + }) + expect(status.connection?.url).toContain('wss://') + }) + + it('rejects an invalid deploy status', () => { + expect(() => deployResponseSchema.parse({ deploymentId: 'x', status: 'deployed' })).toThrow() + }) + + it('exposes the schema-version-mismatch code', () => { + expect(schemaVersionMismatch).toBe('SCHEMA_VERSION_MISMATCH') + }) +}) + +describe('field helpers', () => { + it('flattens fields across steps', () => { + expect(allFields(makeDescriptor()).map((f) => f.key)).toEqual(['name', 'mode', 'apiKey', 'files']) + }) + + it('treats fields without visibleWhen as always visible', () => { + const [name] = allFields(makeDescriptor()) + expect(isFieldVisible(name, {})).toBe(true) + }) + + it('honors visibleWhen equality', () => { + const apiKey = allFields(makeDescriptor()).find((f) => f.key === 'apiKey')! + expect(isFieldVisible(apiKey, { mode: 'curated' })).toBe(false) + expect(isFieldVisible(apiKey, { mode: 'byo' })).toBe(true) + }) + + it('lists only visible fields for the current spec', () => { + expect(visibleFields(makeDescriptor(), { mode: 'curated' }).map((f) => f.key)).toEqual(['name', 'mode', 'files']) + expect(visibleFields(makeDescriptor(), { mode: 'byo' }).map((f) => f.key)).toEqual([ + 'name', + 'mode', + 'apiKey', + 'files', + ]) + }) +}) + +describe('defaultSpec + one-click eligibility', () => { + it('seeds visible defaults only', () => { + expect(defaultSpec(makeDescriptor())).toEqual({ mode: 'curated' }) + }) + + it('is not one-click when a visible required field lacks a default', () => { + expect(isOneClickEligible(makeDescriptor())).toBe(false) + }) + + it('is one-click when every visible required field has a default', () => { + const descriptor = makeDescriptor({ + steps: [ + { + id: 'basics', + title: 'Basics', + fields: [{ key: 'name', label: 'Name', widget: 'text', required: true, default: 'My agent' }], + }, + ], + }) + expect(isOneClickEligible(descriptor)).toBe(true) + }) +}) + +describe('specSchemaForDescriptor (backend re-validation)', () => { + it('enforces required visible fields', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + expect(schema.safeParse({ mode: 'curated' }).success).toBe(false) + expect(schema.safeParse({ name: 'My agent', mode: 'curated' }).success).toBe(true) + }) + + it('skips required fields that are hidden by visibleWhen', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + // apiKey is required but hidden while mode=curated + expect(schema.safeParse({ name: 'My agent', mode: 'curated' }).success).toBe(true) + // once mode=byo, apiKey becomes required + expect(schema.safeParse({ name: 'My agent', mode: 'byo' }).success).toBe(false) + expect(schema.safeParse({ name: 'My agent', mode: 'byo', apiKey: 'sk-1' }).success).toBe(true) + }) + + it('strips unknown keys', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + const parsed = schema.parse({ name: 'My agent', mode: 'curated', bogus: 'x' }) + expect('bogus' in parsed).toBe(false) + }) + + it('enforces maxLength', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + expect(schema.safeParse({ name: 'x'.repeat(41), mode: 'curated' }).success).toBe(false) + }) + + it('shapes gallery/multiple fields as arrays and rejects scalars', () => { + const schema = specSchemaForDescriptor(makeDescriptor()) + expect(schema.safeParse({ name: 'My agent', mode: 'curated', files: ['a', 'b'] }).success).toBe(true) + expect(schema.safeParse({ name: 'My agent', mode: 'curated', files: 'a' }).success).toBe(false) + }) +}) diff --git a/shared/agent-descriptors.ts b/shared/agent-descriptors.ts new file mode 100644 index 000000000..2f7c28069 --- /dev/null +++ b/shared/agent-descriptors.ts @@ -0,0 +1,238 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Wire contract for the descriptor-driven agent creation flow (THU-743). + * + * A **descriptor** is "an agent creation form as data": the backend curates it, + * the frontend renders it over a closed widget registry, and the backend rebuilds + * a zod validator from the same descriptor to re-validate the submitted spec (the + * client is never trusted). Both ends import this file — drift here is silent + * breakage, so the shapes and the validator live in one place. + * + * Descriptors are deliberately NOT JSON Schema: the model is steps → fields → + * widgets, with `visibleWhen` conditional fields and `source`-backed options. A + * spec is the flat map of field key → value the user fills in. + */ + +import { z } from 'zod' + +/** Returned to the client (409) when a deploy carries a stale `schemaVersion`. */ +export const schemaVersionMismatch = 'SCHEMA_VERSION_MISMATCH' + +/** A single spec value: a scalar (`text`/`select`/…) or a list (`gallery`/multi). */ +export const specValueSchema = z.union([z.string(), z.array(z.string())]) +export type SpecValue = z.infer + +/** A deploy spec: the flat map of field key → value collected from the form. */ +export const agentSpecSchema = z.record(z.string(), specValueSchema) +export type AgentSpec = z.infer + +/** The closed set of widgets the frontend renderer knows how to draw. */ +export const widgetSchema = z.enum(['text', 'password', 'textarea', 'select', 'option-cards', 'gallery', 'file-upload']) +export type Widget = z.infer + +/** One selectable option for `select` / `option-cards` / `gallery` widgets. */ +export const optionSchema = z.object({ + value: z.string(), + label: z.string(), + description: z.string().optional(), + icon: z.string().nullable().optional(), +}) +export type AgentFieldOption = z.infer + +/** + * Where a field's options come from. `inline` ships them in the descriptor; + * `fetched` names a `sourceId` the frontend resolves at + * `GET /agents/:descriptorId/sources/:sourceId` (e.g. live Deepset indexes). + */ +export const optionSourceSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('inline'), options: z.array(optionSchema) }), + z.object({ kind: z.literal('fetched'), sourceId: z.string() }), +]) +export type OptionSource = z.infer + +/** Conditional visibility: show this field only when `field` currently equals `equals`. */ +export const visibleWhenSchema = z.object({ + field: z.string(), + equals: z.string(), +}) +export type VisibleWhen = z.infer + +/** A single form field bound to one spec key. */ +export const fieldSchema = z.object({ + key: z.string(), + label: z.string(), + widget: widgetSchema, + required: z.boolean().optional(), + placeholder: z.string().optional(), + helpText: z.string().optional(), + default: specValueSchema.optional(), + visibleWhen: visibleWhenSchema.optional(), + source: optionSourceSchema.optional(), + /** Whether the field collects multiple values (always true for `gallery`). */ + multiple: z.boolean().optional(), + maxLength: z.number().int().positive().optional(), +}) +export type AgentField = z.infer + +/** A group of fields shown together. A single-field/single-option step may be skipped in the UI. */ +export const stepSchema = z.object({ + id: z.string(), + title: z.string(), + description: z.string().optional(), + fields: z.array(fieldSchema), +}) +export type AgentStep = z.infer + +/** What submitting the descriptor does. Phase 1 ships `deploy`; `connect` is the + * future home for the URL-based custom-agent flow. */ +export const agentActionSchema = z.enum(['deploy', 'connect']) +export type AgentAction = z.infer + +/** + * A curated agent creation form. `id` is the unique catalog entry; `provider` is + * the registry key (the "kind": haystack/openclaw/…) the backend dispatches the + * deploy to. `schemaVersion` guards against a client submitting against a stale + * descriptor. + */ +export const agentDescriptorSchema = z.object({ + id: z.string(), + provider: z.string(), + name: z.string(), + description: z.string().nullable(), + icon: z.string().nullable(), + schemaVersion: z.number().int().nonnegative(), + action: agentActionSchema, + steps: z.array(stepSchema), +}) +export type AgentDescriptor = z.infer + +/** Envelope for `GET /agents/catalog`. */ +export const agentCatalogResponseSchema = z.object({ + version: z.literal('1'), + descriptors: z.array(agentDescriptorSchema), +}) +export type AgentCatalogResponse = z.infer + +/** Normalized deploy lifecycle state (host statuses map onto these three). */ +export const deployStatusSchema = z.enum(['pending', 'running', 'failed']) +export type DeployStatus = z.infer + +/** How the frontend connects to a deployed agent once it is running. */ +export const agentConnectionSchema = z.object({ + url: z.string(), + transport: z.literal('websocket'), +}) +export type AgentConnection = z.infer + +/** Body of `POST /agents/deploy`. */ +export const deployRequestSchema = z.object({ + descriptorId: z.string(), + schemaVersion: z.number().int().nonnegative(), + spec: agentSpecSchema, +}) +export type DeployRequest = z.infer + +/** Response of `POST /agents/deploy`. `deploymentId` encodes `provider:ref`. */ +export const deployResponseSchema = z.object({ + deploymentId: z.string(), + status: deployStatusSchema, +}) +export type DeployResponse = z.infer + +/** Response of `GET /agents/deployments/:id` — status polled live from the host. */ +export const deploymentStatusResponseSchema = z.object({ + deploymentId: z.string(), + status: deployStatusSchema, + detail: z.string().nullable().optional(), + connection: agentConnectionSchema.nullable().optional(), +}) +export type DeploymentStatusResponse = z.infer + +/** True when a spec value carries a usable (non-empty) value. */ +const hasValue = (value: SpecValue | undefined): boolean => + Array.isArray(value) ? value.length > 0 : typeof value === 'string' && value.trim().length > 0 + +/** Whether a field collects a list rather than a scalar. */ +const isMultiValue = (field: AgentField): boolean => field.multiple === true || field.widget === 'gallery' + +/** Every field across every step, flattened in order. */ +export const allFields = (descriptor: AgentDescriptor): AgentField[] => descriptor.steps.flatMap((step) => step.fields) + +/** A field is visible when it has no `visibleWhen`, or the guarded field's current + * value equals the guard. */ +export const isFieldVisible = (field: AgentField, spec: AgentSpec): boolean => { + if (!field.visibleWhen) { + return true + } + return spec[field.visibleWhen.field] === field.visibleWhen.equals +} + +/** The fields currently visible given `spec`. */ +export const visibleFields = (descriptor: AgentDescriptor, spec: AgentSpec): AgentField[] => + allFields(descriptor).filter((field) => isFieldVisible(field, spec)) + +/** + * Build the initial spec from field defaults. Visibility is evaluated against the + * full set of defaults, then hidden fields are dropped so we never seed a value + * the user can't see (which the backend re-validation would reject). + */ +export const defaultSpec = (descriptor: AgentDescriptor): AgentSpec => { + const withAllDefaults: AgentSpec = {} + for (const field of allFields(descriptor)) { + if (field.default !== undefined) { + withAllDefaults[field.key] = field.default + } + } + const result: AgentSpec = {} + for (const field of allFields(descriptor)) { + if (field.default !== undefined && isFieldVisible(field, withAllDefaults)) { + result[field.key] = field.default + } + } + return result +} + +/** + * True when the descriptor's default spec already satisfies every visible required + * field — i.e. it can be deployed in one click with no user input. + */ +export const isOneClickEligible = (descriptor: AgentDescriptor): boolean => { + const spec = defaultSpec(descriptor) + return visibleFields(descriptor, spec) + .filter((field) => field.required) + .every((field) => hasValue(spec[field.key])) +} + +/** + * Rebuild a zod validator from a descriptor to re-validate a submitted spec. The + * frontend uses it via `zodResolver`; the backend re-runs it as the authority. + * Unknown keys are stripped, values are shaped per widget (scalar vs list), and + * `required` is enforced only for fields that are visible under the given values. + */ +export const specSchemaForDescriptor = (descriptor: AgentDescriptor) => { + const fields = allFields(descriptor) + const shape: Record = {} + for (const field of fields) { + const scalar = field.maxLength ? z.string().max(field.maxLength) : z.string() + const value = isMultiValue(field) ? z.array(z.string()) : scalar + shape[field.key] = value.optional() + } + return z.object(shape).superRefine((spec, ctx) => { + const typed = spec as AgentSpec + for (const field of fields) { + if (!field.required || !isFieldVisible(field, typed)) { + continue + } + if (!hasValue(typed[field.key])) { + ctx.addIssue({ + code: 'custom', + path: [field.key], + message: `${field.label} is required`, + }) + } + } + }) +} From d2e594800e03232b13c9f906060fe5eb6c2c9c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Tue, 28 Jul 2026 18:32:39 -0300 Subject: [PATCH 02/28] feat(config): add agentDeploy feature flag --- backend/.env.example | 5 +++++ backend/src/api/config.ts | 2 ++ backend/src/api/powersync.test.ts | 1 + backend/src/config/settings.ts | 4 ++++ backend/src/test-utils/settings.ts | 1 + src/api/config-store.ts | 7 +++++++ 6 files changed, 20 insertions(+) diff --git a/backend/.env.example b/backend/.env.example index 5b342cc17..63062fba4 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -112,6 +112,11 @@ OTEL_EXPORTER_OTLP_TOKEN= # Web search via Exa. EXA_API_KEY= +# === Agents — optional === +# AGENT_DEPLOY: expose the descriptor-driven "Add agent" deploy flow (THU-743). +# Opt-in; defaults off. Haystack is the first deployable provider. +AGENT_DEPLOY=false + # === Haystack / Deepset Cloud — optional === # Configure to expose Deepset RAG pipelines as managed ACP agents. # HAYSTACK_BASE_URL: Deepset Cloud root (e.g. https://api.cloud.deepset.ai). diff --git a/backend/src/api/config.ts b/backend/src/api/config.ts index 8a05f6262..3cdedb2c9 100644 --- a/backend/src/api/config.ts +++ b/backend/src/api/config.ts @@ -24,6 +24,8 @@ export const createConfigRoutes = (settings: Settings) => // contract reads as a positive capability ("enabled"). builtInAgentEnabled: !settings.disableBuiltInAgent, allowCustomAgents: settings.allowCustomAgents, + // Opt-in flag (THU-743): exposes the descriptor-driven agent deploy flow. + agentDeploy: settings.agentDeploy, // Omit when unset so the frontend treats it as "no enforcement" without parsing an empty string as semver. minAppVersion: settings.minAppVersion || undefined, defaults: { diff --git a/backend/src/api/powersync.test.ts b/backend/src/api/powersync.test.ts index 0bda64d28..1400fc971 100644 --- a/backend/src/api/powersync.test.ts +++ b/backend/src/api/powersync.test.ts @@ -73,6 +73,7 @@ const powersyncSettings: Settings = { enabledAgents: '', allowCustomAgents: true, disableBuiltInAgent: false, + agentDeploy: false, haystackBaseUrl: '', haystackApiKey: '', haystackWorkspace: '', diff --git a/backend/src/config/settings.ts b/backend/src/config/settings.ts index 3b61e3b61..5d57c9b0d 100644 --- a/backend/src/config/settings.ts +++ b/backend/src/config/settings.ts @@ -142,6 +142,9 @@ const settingsSchema = z // list (not just disabled) — for deployments that ship only their own agents (e.g. Deepset). // Surfaced to the UI via GET /config as `builtInAgentEnabled`. disableBuiltInAgent: z.boolean().default(false), + // When true, exposes the descriptor-driven "Add agent" deploy flow (THU-743). + // Opt-in: absent/false hides it. Surfaced to the UI via GET /config as `agentDeploy`. + agentDeploy: z.boolean().default(false), // Haystack-specific config (consumed by the Haystack provider, defined here for centralized config). haystackBaseUrl: z.string().default(''), haystackApiKey: z.string().default(''), @@ -229,6 +232,7 @@ const parseSettings = (): Settings => { enabledAgents: process.env.ENABLED_AGENTS || '', allowCustomAgents: process.env.ALLOW_CUSTOM_AGENTS !== 'false', disableBuiltInAgent: process.env.DISABLE_BUILT_IN_AGENT === 'true', + agentDeploy: process.env.AGENT_DEPLOY === 'true', haystackBaseUrl: process.env.HAYSTACK_BASE_URL || '', haystackApiKey: process.env.HAYSTACK_API_KEY || '', haystackWorkspace: process.env.HAYSTACK_WORKSPACE || '', diff --git a/backend/src/test-utils/settings.ts b/backend/src/test-utils/settings.ts index f06cf646e..82c6bdc31 100644 --- a/backend/src/test-utils/settings.ts +++ b/backend/src/test-utils/settings.ts @@ -59,6 +59,7 @@ export const createTestSettings = (overrides: Partial = {}): Settings enabledAgents: '', allowCustomAgents: true, disableBuiltInAgent: false, + agentDeploy: false, haystackBaseUrl: '', haystackApiKey: '', haystackWorkspace: '', diff --git a/src/api/config-store.ts b/src/api/config-store.ts index e29ce1e0b..008b71cf5 100644 --- a/src/api/config-store.ts +++ b/src/api/config-store.ts @@ -13,6 +13,9 @@ export type AppConfig = { * built-in agent shown, custom agents allowed. */ builtInAgentEnabled?: boolean allowCustomAgents?: boolean + /** Whether the descriptor-driven agent deploy flow (THU-743) is exposed. Opt-in: + * absent/false = hidden, unlike the sibling flags which default to enabled. */ + agentDeploy?: boolean /** Minimum semver string the server allows. Clients below this are hard-blocked * until they upgrade. Absent/empty = no enforcement. */ minAppVersion?: string @@ -51,3 +54,7 @@ export const selectBuiltInAgentEnabled = (config: AppConfig): boolean => config. /** Whether the UI offers adding custom agents. Absent config defaults to allowed. */ export const selectAllowCustomAgents = (config: AppConfig): boolean => config.allowCustomAgents !== false + +/** Whether the descriptor-driven agent deploy flow is exposed. Opt-in — absent + * config (offline/standalone) reads as OFF, unlike the sibling capability flags. */ +export const selectAgentDeploy = (config: AppConfig): boolean => config.agentDeploy === true From 8c6669785623d9aa50635ae4868e451345c1005a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Tue, 28 Jul 2026 18:36:00 -0300 Subject: [PATCH 03/28] feat(agents): extend provider seam with catalog/deploy/status --- backend/src/agents/deployment-id.test.ts | 33 ++++++++++++++++++++++ backend/src/agents/deployment-id.ts | 33 ++++++++++++++++++++++ backend/src/agents/discovery.ts | 35 +++++++++++++++++++++--- backend/src/agents/index.ts | 10 ++++++- 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 backend/src/agents/deployment-id.test.ts create mode 100644 backend/src/agents/deployment-id.ts diff --git a/backend/src/agents/deployment-id.test.ts b/backend/src/agents/deployment-id.test.ts new file mode 100644 index 000000000..c08ab1752 --- /dev/null +++ b/backend/src/agents/deployment-id.test.ts @@ -0,0 +1,33 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { decodeDeploymentId, encodeDeploymentId } from './deployment-id' + +describe('deployment id codec', () => { + it('round-trips provider + ref', () => { + const id = encodeDeploymentId('haystack', 'tb-my-agent') + expect(id).toBe('haystack:tb-my-agent') + expect(decodeDeploymentId(id)).toEqual({ provider: 'haystack', ref: 'tb-my-agent' }) + }) + + it('splits on the first separator so refs may contain colons', () => { + expect(decodeDeploymentId('haystack:a:b:c')).toEqual({ provider: 'haystack', ref: 'a:b:c' }) + }) + + it('rejects a provider containing the separator', () => { + expect(() => encodeDeploymentId('hay:stack', 'ref')).toThrow() + }) + + it('rejects an empty provider or ref', () => { + expect(() => encodeDeploymentId('', 'ref')).toThrow() + expect(() => encodeDeploymentId('haystack', '')).toThrow() + }) + + it('rejects malformed ids', () => { + expect(() => decodeDeploymentId('haystack')).toThrow() + expect(() => decodeDeploymentId(':ref')).toThrow() + expect(() => decodeDeploymentId('haystack:')).toThrow() + }) +}) diff --git a/backend/src/agents/deployment-id.ts b/backend/src/agents/deployment-id.ts new file mode 100644 index 000000000..d954313c1 --- /dev/null +++ b/backend/src/agents/deployment-id.ts @@ -0,0 +1,33 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * A deployment id is `:` — self-describing so the deployment-status + * endpoint can resolve the owning provider and poll the host directly, with no + * server-side deployment table (THU-743). `provider` is the registry key (the + * agent kind); `ref` is the host-scoped identifier (e.g. a Deepset pipeline name). + */ + +const separator = ':' + +/** Build a deployment id from a provider id and a host reference. */ +export const encodeDeploymentId = (provider: string, ref: string): string => { + if (!provider || provider.includes(separator)) { + throw new Error(`invalid provider for deployment id: ${JSON.stringify(provider)}`) + } + if (!ref) { + throw new Error('deployment id requires a non-empty ref') + } + return `${provider}${separator}${ref}` +} + +/** Parse a deployment id back into its provider and ref. Splits on the first + * separator so a ref may itself contain `:`. */ +export const decodeDeploymentId = (deploymentId: string): { provider: string; ref: string } => { + const index = deploymentId.indexOf(separator) + if (index <= 0 || index === deploymentId.length - 1) { + throw new Error(`malformed deployment id: ${JSON.stringify(deploymentId)}`) + } + return { provider: deploymentId.slice(0, index), ref: deploymentId.slice(index + 1) } +} diff --git a/backend/src/agents/discovery.ts b/backend/src/agents/discovery.ts index dc8c1163d..0b3627fcd 100644 --- a/backend/src/agents/discovery.ts +++ b/backend/src/agents/discovery.ts @@ -4,19 +4,42 @@ import type { Settings } from '@/config/settings' import type { RemoteAgentDescriptor } from '@shared/acp-types' +import type { AgentDescriptor, AgentSpec, DeployResponse, DeploymentStatusResponse } from '@shared/agent-descriptors' /** - * An agent provider contributes one or more {@link RemoteAgentDescriptor} entries - * to the `GET /agents` response. The Haystack module calls {@link registerAgentProvider} - * at startup with its provider; future managed agents follow the same shape. + * Everything a provider needs to service a catalog/deploy/status request: the + * incoming request (WS URL host derivation), resolved settings, and the calling + * user's id (deployment namespacing). + */ +export type ProviderContext = { + request: Request + settings: Settings + userId: string +} + +/** + * An agent provider is the adapter for one agent "kind" (haystack/openclaw/…), + * keyed by {@link AgentProvider.id}. It always contributes discovery descriptors + * (`list`), and may optionally be *deployable* — exposing a creation `catalog` + * plus `deploy`/`status` lifecycle verbs (THU-743). The Haystack module calls + * {@link registerAgentProvider} at startup; future managed agents follow the same shape. */ export type AgentProvider = { - /** Stable identifier for the provider. Re-registering the same id is a no-op. */ + /** Stable identifier for the provider. Re-registering the same id is a no-op. + * Matches `AgentDescriptor.provider`, so deploy requests route back here. */ id: string /** Returns descriptors visible to the caller. May read settings or the request * (e.g. for WS URL host derivation). Throwing here is isolated per-provider — * the discovery route swallows the failure and continues. */ list: (request: Request, settings: Settings) => RemoteAgentDescriptor[] + /** Curated creation descriptors (static). Absent → the provider is discovery-only + * and not deployable. */ + catalog?: (ctx: ProviderContext) => AgentDescriptor[] + /** Create + deploy an instance from an already-validated spec. The returned + * `deploymentId` encodes `provider:ref` (see `deployment-id.ts`). */ + deploy?: (spec: AgentSpec, ctx: ProviderContext) => Promise + /** Live deployment status for `ref`, fetched from the host — never stored. */ + status?: (ref: string, ctx: ProviderContext) => Promise } /** @@ -38,6 +61,10 @@ export const registerAgentProvider = (provider: AgentProvider): void => { /** Return the current set of registered providers in registration order. */ export const getRegisteredProviders = (): AgentProvider[] => [...providers] +/** Look up a provider by id (its "kind"). Used to route deploy/status/catalog + * requests back to the owning provider. */ +export const getProviderById = (id: string): AgentProvider | undefined => providers.find((p) => p.id === id) + /** Test helper — clears all providers. Not exported from the module index. */ export const resetAgentProvidersForTesting = (): void => { providers.length = 0 diff --git a/backend/src/agents/index.ts b/backend/src/agents/index.ts index 4a23f3d00..189d4cc7a 100644 --- a/backend/src/agents/index.ts +++ b/backend/src/agents/index.ts @@ -3,5 +3,13 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ export { createAgentsRoutes } from './routes' -export { registerAgentProvider, getRegisteredProviders, buildWebSocketUrl, type AgentProvider } from './discovery' +export { + registerAgentProvider, + getRegisteredProviders, + getProviderById, + buildWebSocketUrl, + type AgentProvider, + type ProviderContext, +} from './discovery' +export { encodeDeploymentId, decodeDeploymentId } from './deployment-id' export type { AgentsErrorCode, AgentsErrorResponse } from './types' From 6c18f615100d82512ee285a2a577f4a810b83211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Tue, 28 Jul 2026 19:05:38 -0300 Subject: [PATCH 04/28] feat(haystack): add Deepset management client --- .../src/haystack/management-client.test.ts | 126 +++++++++++++++ backend/src/haystack/management-client.ts | 143 ++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 backend/src/haystack/management-client.test.ts create mode 100644 backend/src/haystack/management-client.ts diff --git a/backend/src/haystack/management-client.test.ts b/backend/src/haystack/management-client.test.ts new file mode 100644 index 000000000..de93997fd --- /dev/null +++ b/backend/src/haystack/management-client.test.ts @@ -0,0 +1,126 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { DeepsetManagementClient, DeepsetManagementError, type HaystackManagementConfig } from './management-client' + +const config: HaystackManagementConfig = { + haystackBaseUrl: 'https://api.cloud.deepset.ai', + haystackApiKey: 'sk-test', + haystackWorkspace: 'tutorial', +} + +type FakeResult = { status?: number; body?: unknown; text?: string } +type Call = { url: string; init?: RequestInit } + +const makeFetch = (handler: (url: string, init?: RequestInit) => FakeResult) => { + const calls: Call[] = [] + const fetchFn = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + calls.push({ url, init }) + const result = handler(url, init) + const status = result.status ?? 200 + const bodyText = result.text ?? (result.body !== undefined ? JSON.stringify(result.body) : '') + return new Response(bodyText, { status, statusText: status >= 400 ? 'Error' : 'OK' }) + }) as typeof fetch + return { fetchFn, calls } +} + +const wsBase = 'https://api.cloud.deepset.ai/api/v1/workspaces/tutorial' + +const pipeline = { + name: 'tb-my-agent', + pipeline_id: '70d781d8-4d4c-4154-95c1-6422cdf2c6fb', + status: 'DEPLOYMENT_IN_PROGRESS', + desired_status: 'DEPLOYED', +} + +describe('DeepsetManagementClient', () => { + it('creates a pipeline with the v2 body and drains the 201', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'tb-my-agent' } })) + const client = new DeepsetManagementClient(config, { fetchFn }) + await client.createPipeline({ name: 'tb-my-agent', queryYaml: 'components: {}' }) + expect(calls[0].url).toBe(`${wsBase}/pipelines`) + expect(calls[0].init?.method).toBe('POST') + expect(JSON.parse(String(calls[0].init?.body))).toEqual({ + name: 'tb-my-agent', + query_yaml: 'components: {}', + deepset_cloud_version: 'v2', + }) + }) + + it('appends dry_run when requested', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'x' } })) + const client = new DeepsetManagementClient(config, { fetchFn }) + await client.createPipeline({ name: 'x', queryYaml: 'y', dryRun: true }) + expect(calls[0].url).toBe(`${wsBase}/pipelines?dry_run=true`) + }) + + it('sends the bearer token from config', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'x' } })) + await new DeepsetManagementClient(config, { fetchFn }).createPipeline({ name: 'x', queryYaml: 'y' }) + expect((calls[0].init?.headers as Record).authorization).toBe('Bearer sk-test') + }) + + it('omits the auth header when no api key is configured', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 201, body: { name: 'x' } })) + await new DeepsetManagementClient({ ...config, haystackApiKey: '' }, { fetchFn }).createPipeline({ + name: 'x', + queryYaml: 'y', + }) + expect((calls[0].init?.headers as Record).authorization).toBeUndefined() + }) + + it('deploys and returns the parsed pipeline', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 200, body: pipeline })) + const client = new DeepsetManagementClient(config, { fetchFn }) + const result = await client.deployPipeline('tb-my-agent') + expect(calls[0].url).toBe(`${wsBase}/pipelines/tb-my-agent/deploy`) + expect(result.status).toBe('DEPLOYMENT_IN_PROGRESS') + expect(result.pipeline_id).toBe(pipeline.pipeline_id) + }) + + it('gets pipeline status', async () => { + const { fetchFn } = makeFetch(() => ({ status: 200, body: { ...pipeline, status: 'DEPLOYED' } })) + const result = await new DeepsetManagementClient(config, { fetchFn }).getPipeline('tb-my-agent') + expect(result.status).toBe('DEPLOYED') + }) + + it('lists pipelines from the data envelope', async () => { + const { fetchFn, calls } = makeFetch(() => ({ + status: 200, + body: { data: [pipeline, { ...pipeline, name: 'b' }] }, + })) + const result = await new DeepsetManagementClient(config, { fetchFn }).listPipelines() + expect(calls[0].url).toBe(`${wsBase}/pipelines?limit=100`) + expect(result.map((p) => p.name)).toEqual(['tb-my-agent', 'b']) + }) + + it('fetches the pipeline query_yaml', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 200, body: { query_yaml: 'components:\n agent: {}' } })) + const yaml = await new DeepsetManagementClient(config, { fetchFn }).getPipelineYaml('Rai-RAG-Research-Agent') + expect(calls[0].url).toBe(`${wsBase}/pipelines/Rai-RAG-Research-Agent/yaml`) + expect(yaml).toContain('components:') + }) + + it('undeploys and deletes with the right methods', async () => { + const { fetchFn, calls } = makeFetch(() => ({ status: 200, text: '' })) + const client = new DeepsetManagementClient(config, { fetchFn }) + await client.undeployPipeline('tb-my-agent') + await client.deletePipeline('tb-my-agent') + expect(calls[0].url).toBe(`${wsBase}/pipelines/tb-my-agent/undeploy`) + expect(calls[0].init?.method).toBe('POST') + expect(calls[1].url).toBe(`${wsBase}/pipelines/tb-my-agent`) + expect(calls[1].init?.method).toBe('DELETE') + }) + + it('throws DeepsetManagementError on a non-2xx response', async () => { + const { fetchFn } = makeFetch(() => ({ status: 422, text: 'invalid yaml' })) + const client = new DeepsetManagementClient(config, { fetchFn }) + const error = await client.deployPipeline('bad').catch((e) => e) + expect(error).toBeInstanceOf(DeepsetManagementError) + expect(error.status).toBe(422) + expect(error.message).toContain('invalid yaml') + }) +}) diff --git a/backend/src/haystack/management-client.ts b/backend/src/haystack/management-client.ts new file mode 100644 index 000000000..295c24b18 --- /dev/null +++ b/backend/src/haystack/management-client.ts @@ -0,0 +1,143 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Deepset Cloud pipeline management (create / deploy / status / list), the + * lifecycle counterpart to the runtime `HaystackAcpServer`. Deepset ships no + * JS SDK and no public OpenAPI, so this is raw `fetch` + zod against the + * spike-confirmed REST shapes (THU-743): + * + * create POST /pipelines {name, query_yaml, deepset_cloud_version:'v2'} → 201 {name} + * deploy POST /pipelines/:name/deploy → 200 pipeline + * status GET /pipelines/:name → 200 pipeline + * list GET /pipelines?limit=N → 200 {data:[…]} + * yaml GET /pipelines/:name/yaml → 200 {query_yaml} + * + * Statuses seen: `DEPLOYMENT_IN_PROGRESS` (transient), `DEPLOYED` (running), + * `FAILED`/`DEPLOYMENT_FAILED`. All URLs are workspace-scoped. + */ + +import type { Settings } from '@/config/settings' +import { z } from 'zod' + +/** The Deepset config this client needs — a subset of {@link Settings}. */ +export type HaystackManagementConfig = Pick + +export type HaystackManagementDeps = { + /** Injectable for tests; defaults to the global `fetch`. */ + fetchFn?: typeof fetch +} + +/** A non-2xx response from Deepset, carrying the status and body for diagnosis. */ +export class DeepsetManagementError extends Error { + constructor( + readonly status: number, + readonly statusText: string, + readonly body: string, + ) { + super(`deepset management ${status} ${statusText}: ${body}`) + this.name = 'DeepsetManagementError' + } +} + +/** A Deepset pipeline as returned by deploy / status / list. Extra fields are + * tolerated (stripped) so Deepset can evolve its payload without breaking us. */ +export const deepsetPipelineSchema = z.object({ + name: z.string(), + pipeline_id: z.string(), + status: z.string(), + desired_status: z.string().nullable().optional(), +}) +export type DeepsetPipeline = z.infer + +const listResponseSchema = z.object({ data: z.array(deepsetPipelineSchema) }) +const yamlResponseSchema = z.object({ query_yaml: z.string() }) + +export class DeepsetManagementClient { + private readonly config: HaystackManagementConfig + private readonly fetchFn: typeof fetch + + constructor(config: HaystackManagementConfig, deps: HaystackManagementDeps = {}) { + this.config = config + this.fetchFn = deps.fetchFn ?? globalThis.fetch + } + + /** Create a pipeline. `dryRun` validates the YAML without persisting. Returns + * nothing — the 201 body is just `{name}`. */ + async createPipeline(input: { name: string; queryYaml: string; dryRun?: boolean }): Promise { + const query = input.dryRun ? '?dry_run=true' : '' + const res = await this.send(`${this.pipelinesUrl()}${query}`, { + method: 'POST', + headers: this.jsonHeaders(), + body: JSON.stringify({ name: input.name, query_yaml: input.queryYaml, deepset_cloud_version: 'v2' }), + }) + await res.text() + } + + /** Deploy (activate) a created pipeline. Returns the pipeline with its (in-progress) status. */ + async deployPipeline(name: string): Promise { + const res = await this.send(`${this.pipelineUrl(name)}/deploy`, { method: 'POST', headers: this.jsonHeaders() }) + return deepsetPipelineSchema.parse(await res.json()) + } + + /** Live status of a single pipeline. */ + async getPipeline(name: string): Promise { + const res = await this.send(this.pipelineUrl(name), { headers: this.jsonHeaders() }) + return deepsetPipelineSchema.parse(await res.json()) + } + + /** All pipelines in the workspace. */ + async listPipelines(limit = 100): Promise { + const res = await this.send(`${this.pipelinesUrl()}?limit=${limit}`, { headers: this.jsonHeaders() }) + return listResponseSchema.parse(await res.json()).data + } + + /** The `query_yaml` of an existing pipeline — used to clone a curated template. */ + async getPipelineYaml(name: string): Promise { + const res = await this.send(`${this.pipelineUrl(name)}/yaml`, { headers: this.jsonHeaders() }) + return yamlResponseSchema.parse(await res.json()).query_yaml + } + + /** Stop a running pipeline (teardown step before delete). */ + async undeployPipeline(name: string): Promise { + const res = await this.send(`${this.pipelineUrl(name)}/undeploy`, { method: 'POST', headers: this.jsonHeaders() }) + await res.text() + } + + /** Permanently remove a pipeline. */ + async deletePipeline(name: string): Promise { + const res = await this.send(this.pipelineUrl(name), { method: 'DELETE', headers: this.jsonHeaders() }) + await res.text() + } + + private async send(url: string, init: RequestInit): Promise { + const res = await this.fetchFn(url, init) + if (!res.ok) { + const body = await res.text().catch(() => '') + throw new DeepsetManagementError(res.status, res.statusText, body) + } + return res + } + + private workspaceBaseUrl(): string { + const base = this.config.haystackBaseUrl.replace(/\/$/, '') + return `${base}/api/v1/workspaces/${this.config.haystackWorkspace}` + } + + private pipelinesUrl(): string { + return `${this.workspaceBaseUrl()}/pipelines` + } + + private pipelineUrl(name: string): string { + return `${this.pipelinesUrl()}/${encodeURIComponent(name)}` + } + + private authHeaders(): Record { + return this.config.haystackApiKey ? { authorization: `Bearer ${this.config.haystackApiKey}` } : {} + } + + private jsonHeaders(): Record { + return { 'content-type': 'application/json', accept: 'application/json', ...this.authHeaders() } + } +} From 218a0b1d76d0922201439ec81494e606353586ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Tue, 28 Jul 2026 19:31:36 -0300 Subject: [PATCH 05/28] feat(haystack): implement catalog/deploy/status --- backend/.env.example | 3 + backend/src/api/powersync.test.ts | 1 + backend/src/config/settings.ts | 4 + backend/src/haystack/provider.test.ts | 109 +++++++++++++++++ backend/src/haystack/provider.ts | 162 ++++++++++++++++++++++---- backend/src/test-utils/settings.ts | 1 + 6 files changed, 257 insertions(+), 23 deletions(-) create mode 100644 backend/src/haystack/provider.test.ts diff --git a/backend/.env.example b/backend/.env.example index 63062fba4..a16845c84 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -134,6 +134,9 @@ HAYSTACK_BASE_URL= HAYSTACK_API_KEY= HAYSTACK_WORKSPACE= HAYSTACK_PIPELINES='[{"id":"rag-chat","name":"RAG Chat","pipelineName":"rag-chat-pipeline","pipelineId":"15cf8b39-0000-0000-0000-000000000000","icon":"book"}]' +# HAYSTACK_TEMPLATE_PIPELINE: name of an existing pipeline whose YAML is cloned when a +# user deploys a Haystack agent (requires AGENT_DEPLOY=true). Empty = not deployable. +HAYSTACK_TEMPLATE_PIPELINE= # === Trusted proxy === # Controls which proxy headers are trusted for IP extraction in rate limiting. diff --git a/backend/src/api/powersync.test.ts b/backend/src/api/powersync.test.ts index 1400fc971..7e2695d4e 100644 --- a/backend/src/api/powersync.test.ts +++ b/backend/src/api/powersync.test.ts @@ -78,6 +78,7 @@ const powersyncSettings: Settings = { haystackApiKey: '', haystackWorkspace: '', haystackPipelines: '', + haystackTemplatePipeline: '', minAppVersion: '', } diff --git a/backend/src/config/settings.ts b/backend/src/config/settings.ts index 5d57c9b0d..19c28eae7 100644 --- a/backend/src/config/settings.ts +++ b/backend/src/config/settings.ts @@ -153,6 +153,9 @@ const settingsSchema = z // JSON array of pipeline descriptors: [{id, name, pipelineName, pipelineId, description?, icon?}]. // `id` is the public slug; `pipelineName` is the Deepset URL slug; `pipelineId` is the Deepset UUID. haystackPipelines: z.string().default(''), + // Name of an existing Deepset pipeline whose query_yaml is cloned as the template + // for descriptor-driven deploys (THU-743). Empty = Haystack is not deployable. + haystackTemplatePipeline: z.string().default(''), }) .superRefine((data, ctx) => { if (data.powersyncUrl && data.powersyncJwtSecret.length < 32) { @@ -237,6 +240,7 @@ const parseSettings = (): Settings => { haystackApiKey: process.env.HAYSTACK_API_KEY || '', haystackWorkspace: process.env.HAYSTACK_WORKSPACE || '', haystackPipelines: process.env.HAYSTACK_PIPELINES || '', + haystackTemplatePipeline: process.env.HAYSTACK_TEMPLATE_PIPELINE || '', } return settingsSchema.parse(env) diff --git a/backend/src/haystack/provider.test.ts b/backend/src/haystack/provider.test.ts new file mode 100644 index 000000000..78d0ff7ad --- /dev/null +++ b/backend/src/haystack/provider.test.ts @@ -0,0 +1,109 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { createTestSettings } from '@/test-utils/settings' +import type { ProviderContext } from '@/agents' +import { createHaystackProvider } from './provider' + +const deployableSettings = () => + createTestSettings({ + haystackBaseUrl: 'https://api.cloud.deepset.ai', + haystackApiKey: 'sk-test', + haystackWorkspace: 'tutorial', + haystackTemplatePipeline: 'Template-Pipeline', + }) + +const makeContext = (settings = deployableSettings()): ProviderContext => ({ + request: new Request('http://localhost:8000/v1/agents/deploy'), + settings, + userId: 'user-1', +}) + +type Route = (url: string, init?: RequestInit) => { status?: number; body?: unknown } | undefined + +const routedFetch = (routes: Route) => { + const calls: { url: string; init?: RequestInit }[] = [] + const fetchFn = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + calls.push({ url, init }) + const result = routes(url, init) ?? { status: 404, body: { error: 'no route' } } + const status = result.status ?? 200 + return new Response(result.body !== undefined ? JSON.stringify(result.body) : '', { + status, + statusText: status >= 400 ? 'Error' : 'OK', + }) + }) as typeof fetch + return { fetchFn, calls } +} + +describe('haystack provider — catalog', () => { + it('offers the descriptor when deploy is configured', () => { + const provider = createHaystackProvider() + const catalog = provider.catalog!(makeContext()) + expect(catalog).toHaveLength(1) + expect(catalog[0].id).toBe('haystack') + expect(catalog[0].provider).toBe('haystack') + }) + + it('offers nothing when the template pipeline is unset', () => { + const provider = createHaystackProvider() + const catalog = provider.catalog!(makeContext(createTestSettings({ haystackTemplatePipeline: '' }))) + expect(catalog).toEqual([]) + }) +}) + +describe('haystack provider — deploy', () => { + it('clones the template yaml, creates under tb- namespace, and deploys', async () => { + const { fetchFn, calls } = routedFetch((url, init) => { + if (url.endsWith('/pipelines/Template-Pipeline/yaml')) { + return { status: 200, body: { query_yaml: 'components:\n agent: {}' } } + } + if (init?.method === 'POST' && url.endsWith('/pipelines')) { + return { status: 201, body: { name: 'created' } } + } + if (init?.method === 'POST' && url.endsWith('/deploy')) { + return { status: 200, body: { name: 'ref', pipeline_id: 'pid-1', status: 'DEPLOYMENT_IN_PROGRESS' } } + } + return undefined + }) + const provider = createHaystackProvider({ fetchFn }) + const result = await provider.deploy!({ name: 'My Agent' }, makeContext()) + + expect(result.deploymentId.startsWith('haystack:tb-my-agent-')).toBe(true) + expect(result.status).toBe('pending') + // The create body carried the cloned template YAML. + const createCall = calls.find((c) => c.init?.method === 'POST' && c.url.endsWith('/pipelines')) + expect(JSON.parse(String(createCall?.init?.body)).query_yaml).toContain('agent: {}') + }) +}) + +describe('haystack provider — status', () => { + const statusProvider = (deepsetStatus: string) => { + const { fetchFn } = routedFetch(() => ({ + status: 200, + body: { name: 'tb-x', pipeline_id: 'pid-1', status: deepsetStatus }, + })) + return createHaystackProvider({ fetchFn }) + } + + it('maps DEPLOYED to running with a websocket connection', async () => { + const result = await statusProvider('DEPLOYED').status!('tb-my-agent-abc', makeContext()) + expect(result.status).toBe('running') + expect(result.connection?.transport).toBe('websocket') + expect(result.connection?.url).toContain('haystack/ws?pipeline=tb-my-agent-abc') + expect(result.deploymentId).toBe('haystack:tb-my-agent-abc') + }) + + it('maps an in-progress status to pending with no connection', async () => { + const result = await statusProvider('DEPLOYMENT_IN_PROGRESS').status!('tb-x', makeContext()) + expect(result.status).toBe('pending') + expect(result.connection).toBeNull() + }) + + it('maps a failure status to failed', async () => { + const result = await statusProvider('DEPLOYMENT_FAILED').status!('tb-x', makeContext()) + expect(result.status).toBe('failed') + }) +}) diff --git a/backend/src/haystack/provider.ts b/backend/src/haystack/provider.ts index a09735dab..4901c14fd 100644 --- a/backend/src/haystack/provider.ts +++ b/backend/src/haystack/provider.ts @@ -2,11 +2,20 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { AgentProvider } from '@/agents' -import { buildWebSocketUrl } from '@/agents' +import type { AgentProvider, ProviderContext } from '@/agents' +import { buildWebSocketUrl, encodeDeploymentId } from '@/agents' import { createStandaloneLogger } from '@/config/logger' import type { Settings } from '@/config/settings' import type { RemoteAgentDescriptor } from '@shared/acp-types' +import type { + AgentConnection, + AgentDescriptor, + AgentSpec, + DeploymentStatusResponse, + DeployResponse, + DeployStatus, +} from '@shared/agent-descriptors' +import { DeepsetManagementClient } from './management-client' import { haystackPipelinesEnvSchema } from './types' /** @@ -16,6 +25,77 @@ import { haystackPipelinesEnvSchema } from './types' */ export const haystackProviderId = 'haystack' +/** Injectable dependencies for the provider (test seam for the management client's fetch). */ +export type HaystackProviderDeps = { + fetchFn?: typeof fetch +} + +/** Version of {@link haystackDescriptor}; bump when its fields change. */ +const haystackSchemaVersion = 1 + +/** + * The curated "Add agent" form for Haystack. Phase 1 is curated mode: the owner + * fixes the pipeline template (`HAYSTACK_TEMPLATE_PIPELINE`), so the user only + * names the agent. BYO fields (model/key/index) hang off this same descriptor later. + */ +const haystackDescriptor: AgentDescriptor = { + id: haystackProviderId, + provider: haystackProviderId, + name: 'Haystack RAG agent', + description: 'Deploy a Deepset Cloud pipeline as a chat agent.', + icon: 'file-search', + schemaVersion: haystackSchemaVersion, + action: 'deploy', + steps: [ + { + id: 'basics', + title: 'Name your agent', + fields: [ + { + key: 'name', + label: 'Name', + widget: 'text', + required: true, + maxLength: 60, + placeholder: 'My research agent', + }, + ], + }, + ], +} + +/** Whether Haystack is configured to be deployable (base + key + workspace + template). */ +const isDeployConfigured = (settings: Settings): boolean => + Boolean( + settings.haystackBaseUrl && + settings.haystackApiKey && + settings.haystackWorkspace && + settings.haystackTemplatePipeline, + ) + +/** Map a Deepset pipeline status onto the normalized deploy lifecycle. */ +const mapStatus = (deepsetStatus: string): DeployStatus => { + const status = deepsetStatus.toUpperCase() + if (status === 'DEPLOYED') { + return 'running' + } + if (status.includes('FAIL')) { + return 'failed' + } + return 'pending' +} + +/** Derive a Deepset-safe pipeline name from a user-chosen display name. */ +const toPipelineRef = (name: string): string => { + const slug = + name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 40) || 'agent' + return `tb-${slug}-${Date.now().toString(36)}` +} + /** * Build the Haystack provider. Reads `HAYSTACK_PIPELINES` (JSON array) from * the injected `settings`. An empty / missing / malformed value yields an @@ -28,27 +108,63 @@ export const haystackProviderId = 'haystack' * and prod (`wss://` behind a reverse proxy) both produce correct URLs * without env-var pinning. */ -export const createHaystackProvider = (): AgentProvider => ({ - id: haystackProviderId, - list: (request: Request, settings: Settings): RemoteAgentDescriptor[] => { - const pipelines = parsePipelinesEnv(settings) - if (pipelines.length === 0) { - return [] - } - return pipelines.map((pipeline) => ({ - id: pipeline.id, - name: pipeline.name, - type: 'managed-acp', - transport: 'websocket', - // URL carries the public slug — the WS route resolves it back to the - // Deepset pipelineName / pipelineId from the same env-driven descriptor. - url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(pipeline.id)}`), - description: pipeline.description ?? null, - icon: pipeline.icon ?? null, - isSystem: 1, - })) - }, -}) +export const createHaystackProvider = (deps: HaystackProviderDeps = {}): AgentProvider => { + const managementClient = (settings: Settings) => + new DeepsetManagementClient( + { + haystackBaseUrl: settings.haystackBaseUrl, + haystackApiKey: settings.haystackApiKey, + haystackWorkspace: settings.haystackWorkspace, + }, + { fetchFn: deps.fetchFn }, + ) + + return { + id: haystackProviderId, + list: (request: Request, settings: Settings): RemoteAgentDescriptor[] => { + const pipelines = parsePipelinesEnv(settings) + if (pipelines.length === 0) { + return [] + } + return pipelines.map((pipeline) => ({ + id: pipeline.id, + name: pipeline.name, + type: 'managed-acp', + transport: 'websocket', + // URL carries the public slug — the WS route resolves it back to the + // Deepset pipelineName / pipelineId from the same env-driven descriptor. + url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(pipeline.id)}`), + description: pipeline.description ?? null, + icon: pipeline.icon ?? null, + isSystem: 1, + })) + }, + catalog: ({ settings }: ProviderContext): AgentDescriptor[] => + isDeployConfigured(settings) ? [haystackDescriptor] : [], + deploy: async (spec: AgentSpec, { settings }: ProviderContext): Promise => { + const name = typeof spec.name === 'string' ? spec.name : '' + const ref = toPipelineRef(name) + const client = managementClient(settings) + // Clone the owner-curated template's YAML, create under our `tb-` namespace, then deploy. + const queryYaml = await client.getPipelineYaml(settings.haystackTemplatePipeline) + await client.createPipeline({ name: ref, queryYaml }) + const deployed = await client.deployPipeline(ref) + return { deploymentId: encodeDeploymentId(haystackProviderId, ref), status: mapStatus(deployed.status) } + }, + status: async (ref: string, { request, settings }: ProviderContext): Promise => { + const pipeline = await managementClient(settings).getPipeline(ref) + const status = mapStatus(pipeline.status) + const connection: AgentConnection | null = + status === 'running' + ? { + url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(ref)}`), + transport: 'websocket', + } + : null + return { deploymentId: encodeDeploymentId(haystackProviderId, ref), status, detail: pipeline.status, connection } + }, + } +} /** * Parse `HAYSTACK_PIPELINES` from settings. The env var is a JSON-encoded diff --git a/backend/src/test-utils/settings.ts b/backend/src/test-utils/settings.ts index 82c6bdc31..474d69b92 100644 --- a/backend/src/test-utils/settings.ts +++ b/backend/src/test-utils/settings.ts @@ -63,6 +63,7 @@ export const createTestSettings = (overrides: Partial = {}): Settings haystackBaseUrl: '', haystackApiKey: '', haystackWorkspace: '', + haystackTemplatePipeline: '', haystackPipelines: '', minAppVersion: '', ...overrides, From 501389fabcd945fbb937b536d1d155c3afb92bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Wed, 29 Jul 2026 09:39:51 -0300 Subject: [PATCH 06/28] refactor(agents): support async provider list() --- backend/src/agents/discovery.ts | 7 ++--- backend/src/agents/routes.ts | 47 ++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/backend/src/agents/discovery.ts b/backend/src/agents/discovery.ts index 0b3627fcd..2ad51d0aa 100644 --- a/backend/src/agents/discovery.ts +++ b/backend/src/agents/discovery.ts @@ -29,9 +29,10 @@ export type AgentProvider = { * Matches `AgentDescriptor.provider`, so deploy requests route back here. */ id: string /** Returns descriptors visible to the caller. May read settings or the request - * (e.g. for WS URL host derivation). Throwing here is isolated per-provider — - * the discovery route swallows the failure and continues. */ - list: (request: Request, settings: Settings) => RemoteAgentDescriptor[] + * (e.g. for WS URL host derivation) and may be async (e.g. a live host query). + * Throwing / rejecting here is isolated per-provider — the discovery route + * swallows the failure and continues. */ + list: (request: Request, settings: Settings) => RemoteAgentDescriptor[] | Promise /** Curated creation descriptors (static). Absent → the provider is discovery-only * and not deployable. */ catalog?: (ctx: ProviderContext) => AgentDescriptor[] diff --git a/backend/src/agents/routes.ts b/backend/src/agents/routes.ts index 0747ac99f..8cff1cb45 100644 --- a/backend/src/agents/routes.ts +++ b/backend/src/agents/routes.ts @@ -37,41 +37,44 @@ export const createAgentsRoutes = (auth: Auth) => const sessionUser = session?.user as User | undefined return { user: sessionUser ?? null } }) - .get('/', ({ request, set, user }): AgentDiscoveryResponse | AgentsErrorResponse | { error: string } => { - if (!user) { - set.status = 401 - return { error: 'Unauthorized' } - } - if (user.isAnonymous) { - set.status = 403 - return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } - } + .get( + '/', + async ({ request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } - const settings = getSettings() - const enabledIds = getEnabledAgentsList(settings) - const allowedById = (id: string) => enabledIds.length === 0 || enabledIds.includes(id) + const settings = getSettings() + const enabledIds = getEnabledAgentsList(settings) + const allowedById = (id: string) => enabledIds.length === 0 || enabledIds.includes(id) - const agents = collectAgents(request, settings) - const filtered = agents.filter((descriptor) => allowedById(descriptor.id)) + const agents = await collectAgents(request, settings) + const filtered = agents.filter((descriptor) => allowedById(descriptor.id)) - return { - version: '1', - agents: filtered, - allowCustomAgents: settings.allowCustomAgents, - } - }) + return { + version: '1', + agents: filtered, + allowCustomAgents: settings.allowCustomAgents, + } + }, + ) /** * Asks every registered provider for its descriptors and concatenates the * results. A throwing provider is logged and skipped — one misbehaving plugin * never poisons the response for the others. */ -const collectAgents = (request: Request, settings: Settings): RemoteAgentDescriptor[] => { +const collectAgents = async (request: Request, settings: Settings): Promise => { const log = createStandaloneLogger(settings) const out: RemoteAgentDescriptor[] = [] for (const provider of getRegisteredProviders()) { try { - out.push(...provider.list(request, settings)) + out.push(...(await provider.list(request, settings))) } catch (error) { log.warn({ err: error, providerId: provider.id }, 'agent provider list() failed; skipping') } From ac1c0da95c48b4bcce0ccb40cbe63bb49a2844f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Wed, 29 Jul 2026 10:15:17 -0300 Subject: [PATCH 07/28] feat(haystack): discover pipelines from Deepset API --- backend/.env.example | 12 +- backend/src/api/powersync.test.ts | 1 - backend/src/config/settings.ts | 4 - backend/src/haystack/acp-server.test.ts | 1 - backend/src/haystack/files.test.ts | 1 - backend/src/haystack/index.ts | 11 +- backend/src/haystack/management-client.ts | 2 + backend/src/haystack/provider.test.ts | 56 +++++++- backend/src/haystack/provider.ts | 149 +++++++++++----------- backend/src/haystack/routes.test.ts | 45 ++++--- backend/src/haystack/routes.ts | 21 +-- backend/src/haystack/types.ts | 51 -------- backend/src/test-utils/settings.ts | 1 - 13 files changed, 179 insertions(+), 176 deletions(-) diff --git a/backend/.env.example b/backend/.env.example index a16845c84..71c0dac81 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -118,22 +118,16 @@ EXA_API_KEY= AGENT_DEPLOY=false # === Haystack / Deepset Cloud — optional === -# Configure to expose Deepset RAG pipelines as managed ACP agents. +# Configure to expose Deepset pipelines as managed ACP agents. Pipelines are +# discovered live from the Deepset API (GET /agents lists DEPLOYED, prompt-capable +# pipelines, excluding Thunderbolt-deployed `tb-*` ones) — no static list to maintain. # HAYSTACK_BASE_URL: Deepset Cloud root (e.g. https://api.cloud.deepset.ai). # HAYSTACK_API_KEY: bearer token used as `Authorization: Bearer …`. # HAYSTACK_WORKSPACE: workspace slug. URLs become # `${HAYSTACK_BASE_URL}/api/v1/workspaces/${HAYSTACK_WORKSPACE}/...`. -# HAYSTACK_PIPELINES: JSON array. Each entry needs: -# - id public slug surfaced to the FE (`rag-chat`). -# - name human-readable label. -# - pipelineName Deepset URL slug used in /pipelines//chat-stream. -# - pipelineId Deepset UUID, sent as `pipeline_id` when bootstrapping -# a `search_session`. -# - icon (optional) Phosphor icon name shown in the agent picker. HAYSTACK_BASE_URL= HAYSTACK_API_KEY= HAYSTACK_WORKSPACE= -HAYSTACK_PIPELINES='[{"id":"rag-chat","name":"RAG Chat","pipelineName":"rag-chat-pipeline","pipelineId":"15cf8b39-0000-0000-0000-000000000000","icon":"book"}]' # HAYSTACK_TEMPLATE_PIPELINE: name of an existing pipeline whose YAML is cloned when a # user deploys a Haystack agent (requires AGENT_DEPLOY=true). Empty = not deployable. HAYSTACK_TEMPLATE_PIPELINE= diff --git a/backend/src/api/powersync.test.ts b/backend/src/api/powersync.test.ts index 7e2695d4e..b6f3f20bf 100644 --- a/backend/src/api/powersync.test.ts +++ b/backend/src/api/powersync.test.ts @@ -77,7 +77,6 @@ const powersyncSettings: Settings = { haystackBaseUrl: '', haystackApiKey: '', haystackWorkspace: '', - haystackPipelines: '', haystackTemplatePipeline: '', minAppVersion: '', } diff --git a/backend/src/config/settings.ts b/backend/src/config/settings.ts index 19c28eae7..e5162d0b2 100644 --- a/backend/src/config/settings.ts +++ b/backend/src/config/settings.ts @@ -150,9 +150,6 @@ const settingsSchema = z haystackApiKey: z.string().default(''), // Deepset workspace slug. URLs are `${baseUrl}/api/v1/workspaces/${workspace}/...`. haystackWorkspace: z.string().default(''), - // JSON array of pipeline descriptors: [{id, name, pipelineName, pipelineId, description?, icon?}]. - // `id` is the public slug; `pipelineName` is the Deepset URL slug; `pipelineId` is the Deepset UUID. - haystackPipelines: z.string().default(''), // Name of an existing Deepset pipeline whose query_yaml is cloned as the template // for descriptor-driven deploys (THU-743). Empty = Haystack is not deployable. haystackTemplatePipeline: z.string().default(''), @@ -239,7 +236,6 @@ const parseSettings = (): Settings => { haystackBaseUrl: process.env.HAYSTACK_BASE_URL || '', haystackApiKey: process.env.HAYSTACK_API_KEY || '', haystackWorkspace: process.env.HAYSTACK_WORKSPACE || '', - haystackPipelines: process.env.HAYSTACK_PIPELINES || '', haystackTemplatePipeline: process.env.HAYSTACK_TEMPLATE_PIPELINE || '', } diff --git a/backend/src/haystack/acp-server.test.ts b/backend/src/haystack/acp-server.test.ts index 96e9cabc7..e556de35c 100644 --- a/backend/src/haystack/acp-server.test.ts +++ b/backend/src/haystack/acp-server.test.ts @@ -30,7 +30,6 @@ const buildSettings = (overrides: Partial = {}): Settings => haystackBaseUrl: 'https://haystack.test', haystackApiKey: 'test-key', haystackWorkspace: 'ws-test', - haystackPipelines: '', logLevel: 'INFO', ...overrides, }) as Settings diff --git a/backend/src/haystack/files.test.ts b/backend/src/haystack/files.test.ts index 273cbf4b1..c8f95c8fb 100644 --- a/backend/src/haystack/files.test.ts +++ b/backend/src/haystack/files.test.ts @@ -66,7 +66,6 @@ const haystackSettings = createTestSettings({ haystackBaseUrl: 'https://haystack.test', haystackApiKey: 'sekrit', haystackWorkspace: 'ws-test', - haystackPipelines: '', }) const buildApp = (auth: Auth, fetchFn: typeof fetch) => diff --git a/backend/src/haystack/index.ts b/backend/src/haystack/index.ts index fa7a213cb..6f1beefb4 100644 --- a/backend/src/haystack/index.ts +++ b/backend/src/haystack/index.ts @@ -3,14 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ export { createHaystackRoutes } from './routes' -export { createHaystackProvider, haystackProviderId, parsePipelinesEnv } from './provider' +export { createHaystackProvider, haystackProviderId, resolveHaystackPipeline } from './provider' export { HaystackAcpServer, type HaystackAcpDeps } from './acp-server' export { parseHaystackSseStream, HaystackSseParseError } from './sse-parser' -export { - haystackEventSchema, - haystackPipelineDescriptorSchema, - haystackPipelinesEnvSchema, - type HaystackEvent, - type HaystackPipelineDescriptor, - type HaystackSessionContext, -} from './types' +export { haystackEventSchema, type HaystackEvent, type HaystackSessionContext } from './types' diff --git a/backend/src/haystack/management-client.ts b/backend/src/haystack/management-client.ts index 295c24b18..a395c9d35 100644 --- a/backend/src/haystack/management-client.ts +++ b/backend/src/haystack/management-client.ts @@ -48,6 +48,8 @@ export const deepsetPipelineSchema = z.object({ pipeline_id: z.string(), status: z.string(), desired_status: z.string().nullable().optional(), + supports_prompt: z.boolean().optional(), + output_type: z.string().nullable().optional(), }) export type DeepsetPipeline = z.infer diff --git a/backend/src/haystack/provider.test.ts b/backend/src/haystack/provider.test.ts index 78d0ff7ad..382e276af 100644 --- a/backend/src/haystack/provider.test.ts +++ b/backend/src/haystack/provider.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from 'bun:test' import { createTestSettings } from '@/test-utils/settings' import type { ProviderContext } from '@/agents' -import { createHaystackProvider } from './provider' +import { createHaystackProvider, resolveHaystackPipeline } from './provider' const deployableSettings = () => createTestSettings({ @@ -107,3 +107,57 @@ describe('haystack provider — status', () => { expect(result.status).toBe('failed') }) }) + +describe('haystack provider — list', () => { + const request = new Request('http://localhost:8000/v1/agents') + const listBody = { + data: [ + { name: 'RAG', pipeline_id: 'p1', status: 'DEPLOYED', desired_status: 'DEPLOYED', supports_prompt: true }, + // Auto-idled but intended-deployed → still a usable agent (wakes on query). + { name: 'Napping', pipeline_id: 'p2', status: 'IDLE', desired_status: 'DEPLOYED', supports_prompt: true }, + // tb-* deploys ARE included for now (de-dup with the synced agents table is deferred). + { name: 'tb-mine-x', pipeline_id: 'p3', status: 'DEPLOYED', desired_status: 'DEPLOYED', supports_prompt: true }, + { name: 'Undeployed', pipeline_id: 'p4', status: 'IDLE', desired_status: 'UNDEPLOYED', supports_prompt: true }, + { name: 'Indexer', pipeline_id: 'p5', status: 'DEPLOYED', desired_status: 'DEPLOYED', supports_prompt: false }, + ], + } + + it('returns [] when Haystack is not configured', async () => { + expect(await createHaystackProvider().list(request, createTestSettings())).toEqual([]) + }) + + it('maps intended-deployed, prompt-capable pipelines (including idle + tb- ones)', async () => { + const { fetchFn, calls } = routedFetch(() => ({ status: 200, body: listBody })) + const list = await createHaystackProvider({ fetchFn }).list(request, deployableSettings()) + expect(list.map((a) => a.id)).toEqual(['RAG', 'Napping', 'tb-mine-x']) + expect(list[0]).toMatchObject({ name: 'RAG', type: 'managed-acp', transport: 'websocket', isSystem: 1 }) + expect(list[0].url).toContain('haystack/ws?pipeline=RAG') + expect(calls[0].url).toContain('/pipelines?limit=') + }) + + it('returns [] (never throws) when the host errors', async () => { + const { fetchFn } = routedFetch(() => ({ status: 500 })) + expect(await createHaystackProvider({ fetchFn }).list(request, deployableSettings())).toEqual([]) + }) +}) + +describe('resolveHaystackPipeline', () => { + it('resolves a pipeline by fetching its pipeline_id live', async () => { + const { fetchFn, calls } = routedFetch(() => ({ + status: 200, + body: { name: 'RAG', pipeline_id: 'pid-live', status: 'DEPLOYED' }, + })) + const resolved = await resolveHaystackPipeline('RAG', deployableSettings(), { fetchFn }) + expect(resolved).toEqual({ pipelineId: 'pid-live', pipelineName: 'RAG', supportsFiles: false }) + expect(calls[0].url).toContain('/pipelines/RAG') + }) + + it('returns null when Haystack is not configured', async () => { + expect(await resolveHaystackPipeline('RAG', createTestSettings())).toBeNull() + }) + + it('returns null (not throw) when the host lookup fails', async () => { + const { fetchFn } = routedFetch(() => ({ status: 404, body: { error: 'gone' } })) + expect(await resolveHaystackPipeline('missing', deployableSettings(), { fetchFn })).toBeNull() + }) +}) diff --git a/backend/src/haystack/provider.ts b/backend/src/haystack/provider.ts index 4901c14fd..7bc8db8f3 100644 --- a/backend/src/haystack/provider.ts +++ b/backend/src/haystack/provider.ts @@ -16,7 +16,6 @@ import type { DeployStatus, } from '@shared/agent-descriptors' import { DeepsetManagementClient } from './management-client' -import { haystackPipelinesEnvSchema } from './types' /** * Provider id registered into the agent discovery registry. The string is @@ -64,14 +63,13 @@ const haystackDescriptor: AgentDescriptor = { ], } -/** Whether Haystack is configured to be deployable (base + key + workspace + template). */ +/** Whether Haystack is configured to talk to Deepset at all (discovery + chat). */ +const isHaystackConfigured = (settings: Settings): boolean => + Boolean(settings.haystackBaseUrl && settings.haystackApiKey && settings.haystackWorkspace) + +/** Whether Haystack is also deployable (needs a template pipeline to clone). */ const isDeployConfigured = (settings: Settings): boolean => - Boolean( - settings.haystackBaseUrl && - settings.haystackApiKey && - settings.haystackWorkspace && - settings.haystackTemplatePipeline, - ) + isHaystackConfigured(settings) && Boolean(settings.haystackTemplatePipeline) /** Map a Deepset pipeline status onto the normalized deploy lifecycle. */ const mapStatus = (deepsetStatus: string): DeployStatus => { @@ -96,48 +94,87 @@ const toPipelineRef = (name: string): string => { return `tb-${slug}-${Date.now().toString(36)}` } +/** Construct a management client from settings (shared by the provider verbs and the WS resolver). */ +const makeManagementClient = (settings: Settings, deps: HaystackProviderDeps): DeepsetManagementClient => + new DeepsetManagementClient( + { + haystackBaseUrl: settings.haystackBaseUrl, + haystackApiKey: settings.haystackApiKey, + haystackWorkspace: settings.haystackWorkspace, + }, + { fetchFn: deps.fetchFn }, + ) + +/** A pipeline resolved for a live chat connection. */ +export type ResolvedPipeline = { pipelineId: string; pipelineName: string; supportsFiles: boolean } + +/** + * Resolve a `?pipeline=` slug to its Deepset identifiers for the WS route by + * looking it up live (the slug is the Deepset pipeline name). Returns null for + * unknown / unconfigured slugs so the caller can close the socket. Pipelines are + * treated as text-only for now (`supportsFiles: false`). + */ +export const resolveHaystackPipeline = async ( + slug: string, + settings: Settings, + deps: HaystackProviderDeps = {}, +): Promise => { + if (!isHaystackConfigured(settings)) { + return null + } + try { + const pipeline = await makeManagementClient(settings, deps).getPipeline(slug) + return { pipelineId: pipeline.pipeline_id, pipelineName: slug, supportsFiles: false } + } catch { + return null + } +} + /** - * Build the Haystack provider. Reads `HAYSTACK_PIPELINES` (JSON array) from - * the injected `settings`. An empty / missing / malformed value yields an - * empty descriptor list — we log and skip rather than throw so a deployment - * with no Haystack config doesn't fail other providers. + * Build the Haystack provider. `list()` fetches the workspace's pipelines live + * from Deepset — `DEPLOYED`, prompt-capable, and excluding Thunderbolt-deployed + * `tb-*` instances (those are user-owned and live in the synced `agents` table). + * On any host error it logs and returns `[]` so discovery never fails. * - * Each pipeline becomes a `managed-acp`, websocket-transport descriptor whose - * URL points at `/v1/haystack/ws?pipeline=`. The host is derived - * from the inbound `Request` via {@link buildWebSocketUrl} so dev (`ws://`) - * and prod (`wss://` behind a reverse proxy) both produce correct URLs - * without env-var pinning. + * Each pipeline becomes a `managed-acp`, websocket descriptor whose URL points at + * `/v1/haystack/ws?pipeline=`; the host is derived from the inbound request + * via {@link buildWebSocketUrl} so dev (`ws://`) and prod (`wss://`) both work. */ export const createHaystackProvider = (deps: HaystackProviderDeps = {}): AgentProvider => { - const managementClient = (settings: Settings) => - new DeepsetManagementClient( - { - haystackBaseUrl: settings.haystackBaseUrl, - haystackApiKey: settings.haystackApiKey, - haystackWorkspace: settings.haystackWorkspace, - }, - { fetchFn: deps.fetchFn }, - ) + const managementClient = (settings: Settings) => makeManagementClient(settings, deps) return { id: haystackProviderId, - list: (request: Request, settings: Settings): RemoteAgentDescriptor[] => { - const pipelines = parsePipelinesEnv(settings) - if (pipelines.length === 0) { + list: async (request: Request, settings: Settings): Promise => { + if (!isHaystackConfigured(settings)) { + return [] + } + try { + const pipelines = await managementClient(settings).listPipelines() + return pipelines + .filter( + // Deepset auto-idles pipelines (status flips DEPLOYED→IDLE), but they + // wake on query — so key off the intended `desired_status`, not the + // transient runtime status. Non-prompt pipelines (indexes) aren't chat agents. + // @todo Revisit de-duplication: this includes Thunderbolt-deployed `tb-*` + // pipelines, which the FE also stores in the synced agents table, so a + // deployed agent can currently appear in both lanes. + (p) => (p.desired_status ?? p.status) === 'DEPLOYED' && p.supports_prompt !== false, + ) + .map((p) => ({ + id: p.name, + name: p.name, + type: 'managed-acp', + transport: 'websocket', + url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(p.name)}`), + description: null, + icon: null, + isSystem: 1, + })) + } catch (err) { + createStandaloneLogger(settings).warn({ err }, 'haystack list pipelines failed; returning empty') return [] } - return pipelines.map((pipeline) => ({ - id: pipeline.id, - name: pipeline.name, - type: 'managed-acp', - transport: 'websocket', - // URL carries the public slug — the WS route resolves it back to the - // Deepset pipelineName / pipelineId from the same env-driven descriptor. - url: buildWebSocketUrl(request, `/haystack/ws?pipeline=${encodeURIComponent(pipeline.id)}`), - description: pipeline.description ?? null, - icon: pipeline.icon ?? null, - isSystem: 1, - })) }, catalog: ({ settings }: ProviderContext): AgentDescriptor[] => isDeployConfigured(settings) ? [haystackDescriptor] : [], @@ -165,33 +202,3 @@ export const createHaystackProvider = (deps: HaystackProviderDeps = {}): AgentPr }, } } - -/** - * Parse `HAYSTACK_PIPELINES` from settings. The env var is a JSON-encoded - * array of {@link haystackPipelinesEnvSchema} entries. Empty / missing values - * return `[]`. A malformed value also returns `[]` but is logged at WARN — - * silent dropping would hide a deployment-side typo, but throwing would - * cascade into a `GET /agents` 500 for unrelated providers (the discovery - * route catches the throw, but the operator wouldn't get a structured signal). - */ -export const parsePipelinesEnv = (settings: Settings) => { - const raw = settings.haystackPipelines.trim() - if (raw.length === 0) { - return [] - } - let parsedJson: unknown - try { - parsedJson = JSON.parse(raw) - } catch (err) { - const log = createStandaloneLogger(settings) - log.warn({ err }, 'HAYSTACK_PIPELINES is not valid JSON; ignoring') - return [] - } - const result = haystackPipelinesEnvSchema.safeParse(parsedJson) - if (!result.success) { - const log = createStandaloneLogger(settings) - log.warn({ issues: result.error.issues }, 'HAYSTACK_PIPELINES schema mismatch; ignoring') - return [] - } - return result.data -} diff --git a/backend/src/haystack/routes.test.ts b/backend/src/haystack/routes.test.ts index 35f11a27e..05cdd2dc3 100644 --- a/backend/src/haystack/routes.test.ts +++ b/backend/src/haystack/routes.test.ts @@ -99,6 +99,19 @@ const bearerProtocols = (bearerToken: string): string[] => [ `thunderbolt.bearer.${encodeWsBearer(bearerToken)}`, ] +/** + * Fake Deepset for the WS resolver (`resolveHaystackPipeline` → `GET /pipelines/:name`): + * `rag` resolves to a deployed pipeline; everything else 404s → the route closes 4001. + * Injected as `createApp`'s `fetchFn`, so it only affects the live slug lookup. + */ +const haystackFetch = (async (input: RequestInfo | URL) => { + const url = String(input) + if (/\/pipelines\/rag$/.test(url)) { + return new Response(JSON.stringify({ name: 'rag', pipeline_id: 'pipe-uuid', status: 'DEPLOYED' }), { status: 200 }) + } + return new Response(JSON.stringify({ error: 'not found' }), { status: 404, statusText: 'Not Found' }) +}) as typeof fetch + describe('WS /v1/haystack/ws — auth gating', () => { const cleanups: Array<() => Promise> = [] // Isolated PGlite instance for the real-`.listen()` upgrades in this suite: @@ -106,12 +119,12 @@ describe('WS /v1/haystack/ws — auth gating', () => { // singleton (head-of-line blocking under CI starvation; the 5s `initialize` // timeout was the canary). Closed once in afterAll to avoid exit-99. let iso: IsolatedTestDb - // Real settings come from `getSettings()` which reads `process.env`. Inject a - // valid pipelines config so the route's slug→descriptor lookup resolves; - // otherwise even authenticated upgrades would close 4001 with "unknown - // pipeline" before initialize can run. - const originalPipelinesEnv = process.env.HAYSTACK_PIPELINES + // Real settings come from `getSettings()` which reads `process.env`. Point + // Haystack at a fake host so it's "configured"; the route's live slug→pipeline + // lookup is served by the injected `haystackFetch` (otherwise even + // authenticated upgrades would close 4001 with "unknown pipeline"). const originalBaseUrlEnv = process.env.HAYSTACK_BASE_URL + const originalApiKeyEnv = process.env.HAYSTACK_API_KEY const originalWorkspaceEnv = process.env.HAYSTACK_WORKSPACE beforeAll(async () => { @@ -120,10 +133,8 @@ describe('WS /v1/haystack/ws — auth gating', () => { beforeEach(() => { process.env.HAYSTACK_BASE_URL = 'https://haystack.test' + process.env.HAYSTACK_API_KEY = 'sk-test' process.env.HAYSTACK_WORKSPACE = 'ws-test' - process.env.HAYSTACK_PIPELINES = JSON.stringify([ - { id: 'rag', name: 'RAG', pipelineName: 'rag-pipeline', pipelineId: 'pipe-uuid' }, - ]) clearSettingsCache() }) @@ -131,16 +142,16 @@ describe('WS /v1/haystack/ws — auth gating', () => { for (const cleanup of cleanups.splice(0)) { await cleanup() } - if (originalPipelinesEnv === undefined) { - delete process.env.HAYSTACK_PIPELINES - } else { - process.env.HAYSTACK_PIPELINES = originalPipelinesEnv - } if (originalBaseUrlEnv === undefined) { delete process.env.HAYSTACK_BASE_URL } else { process.env.HAYSTACK_BASE_URL = originalBaseUrlEnv } + if (originalApiKeyEnv === undefined) { + delete process.env.HAYSTACK_API_KEY + } else { + process.env.HAYSTACK_API_KEY = originalApiKeyEnv + } if (originalWorkspaceEnv === undefined) { delete process.env.HAYSTACK_WORKSPACE } else { @@ -232,7 +243,7 @@ describe('WS /v1/haystack/ws — auth gating', () => { }) it('opens the socket and answers initialize when a valid bearer is offered', async () => { - const handle = await createTestApp({ database: iso.db }) + const handle = await createTestApp({ database: iso.db, fetchFn: haystackFetch }) const port = await startApp(handle.app as unknown as RunningApp) cleanups.push(async () => { await stopApp(handle.app as unknown as RunningApp) @@ -267,15 +278,15 @@ describe('WS /v1/haystack/ws — auth gating', () => { client.close() }, 15000) - it('closes 4001 when the ?pipeline= slug is not in HAYSTACK_PIPELINES', async () => { - const handle = await createTestApp({ database: iso.db }) + it('closes 4001 when the ?pipeline= slug is not a known pipeline', async () => { + const handle = await createTestApp({ database: iso.db, fetchFn: haystackFetch }) const port = await startApp(handle.app as unknown as RunningApp) cleanups.push(async () => { await stopApp(handle.app as unknown as RunningApp) await handle.cleanup() }) - // `rag` is configured in beforeEach; `nope` is not — the route should reject + // `rag` resolves via the fake host; `nope` 404s — the route should reject // even with a valid bearer. const client = new WebSocket( `ws://127.0.0.1:${port}/v1/haystack/ws?pipeline=nope`, diff --git a/backend/src/haystack/routes.ts b/backend/src/haystack/routes.ts index 46debb381..3841db2b5 100644 --- a/backend/src/haystack/routes.ts +++ b/backend/src/haystack/routes.ts @@ -12,7 +12,7 @@ import type { User } from '@shared/types/auth' import { wsCarrierSubprotocol } from '@shared/ws-bearer' import { Elysia, t } from 'elysia' import { HaystackAcpServer, type HaystackAcpDeps } from './acp-server' -import { createHaystackProvider, parsePipelinesEnv } from './provider' +import { createHaystackProvider, resolveHaystackPipeline } from './provider' /** * Mount the Haystack ACP adapter routes. @@ -142,11 +142,12 @@ export const createHaystackRoutes = (settings: Settings, auth: Auth, deps?: Hays return } - // Resolve the public slug back to its Deepset identifiers. A missing - // descriptor here means a stale FE URL or a redeploy that dropped the - // pipeline — we close instead of opening to keep error surface tight. - const descriptor = parsePipelinesEnv(settings).find((p) => p.id === pipelineSlug) - if (!descriptor) { + // Resolve the slug to its Deepset identifiers: an env-declared descriptor, + // or a Thunderbolt-deployed `tb-*` pipeline looked up live. A null result + // means a stale FE URL or a dropped pipeline — close instead of opening to + // keep the error surface tight. + const resolved = await resolveHaystackPipeline(pipelineSlug, settings, { fetchFn: deps?.fetchFn }) + if (!resolved) { ws.close(wsCloseUnauthorized, 'unknown pipeline') return } @@ -155,14 +156,14 @@ export const createHaystackRoutes = (settings: Settings, auth: Auth, deps?: Hays send: (payload) => { ws.send(payload) }, - pipelineId: descriptor.pipelineId, - pipelineName: descriptor.pipelineName, - supportsFiles: descriptor.supportedContent.files, + pipelineId: resolved.pipelineId, + pipelineName: resolved.pipelineName, + supportsFiles: resolved.supportsFiles, settings, deps, }) ;(ws.data as unknown as { __haystackServer?: HaystackAcpServer }).__haystackServer = server - log.debug({ pipelineSlug, pipelineName: descriptor.pipelineName, userId: user.id }, 'haystack ws opened') + log.debug({ pipelineSlug, pipelineName: resolved.pipelineName, userId: user.id }, 'haystack ws opened') }, async message(ws, message) { const server = (ws.data as unknown as { __haystackServer?: HaystackAcpServer }).__haystackServer diff --git a/backend/src/haystack/types.ts b/backend/src/haystack/types.ts index 50a1c053a..fb92efe32 100644 --- a/backend/src/haystack/types.ts +++ b/backend/src/haystack/types.ts @@ -4,57 +4,6 @@ import { z } from 'zod' -/** - * Haystack pipeline descriptor as configured via the `HAYSTACK_PIPELINES` - * environment variable. The variable is a JSON array of these objects. Each - * pipeline becomes one {@link RemoteAgentDescriptor} on `GET /agents`. - * - * Deepset Cloud surfaces pipelines under two identifiers: - * - `pipelineName` — URL slug used in `/pipelines/${pipelineName}/chat-stream`. - * - `pipelineId` — the workspace-scoped UUID, used as the `pipeline_id` - * body field when bootstrapping a `search_session`. - * - * We keep them as separate fields because Deepset can rename a pipeline (slug - * changes) without re-issuing its UUID, and vice versa. - * - * - `id` — public slug we surface to the FE (`rag-chat`, etc). The - * only identifier the FE sees. - * - `name` — human-readable label for the agent picker. - * - `pipelineName`— Deepset URL slug. - * - `pipelineId` — Deepset UUID. - * - `description` — optional one-line description shown in the picker. - * - `icon` — optional Phosphor icon name; defaults applied by the - * provider when omitted. - * - `supportedContent` — which prompt content the pipeline accepts. `files: - * true` is load-bearing: it both advertises ACP - * `promptCapabilities.embeddedContext` to the client AND - * switches the run path to `temporary_files` upload + - * `search-stream` (generative pipelines), instead of the - * `search_sessions` + `chat-stream` path used by chat - * pipelines. Absent → `{ text: true, files: false }`, so - * existing chat configs need no change. - */ -const supportedContentSchema = z - .object({ - text: z.boolean().default(true), - files: z.boolean().default(false), - }) - .default({ text: true, files: false }) - -export const haystackPipelineDescriptorSchema = z.object({ - id: z.string().min(1), - name: z.string().min(1), - pipelineName: z.string().min(1), - pipelineId: z.string().min(1), - description: z.string().optional(), - icon: z.string().optional(), - supportedContent: supportedContentSchema, -}) - -export type HaystackPipelineDescriptor = z.infer - -export const haystackPipelinesEnvSchema = z.array(haystackPipelineDescriptorSchema) - /** * SSE event types emitted by Deepset's `/chat-stream` endpoint. Mirrors the * shape exercised by Deepset Cloud production traffic and the upstream diff --git a/backend/src/test-utils/settings.ts b/backend/src/test-utils/settings.ts index 474d69b92..8b7a8a340 100644 --- a/backend/src/test-utils/settings.ts +++ b/backend/src/test-utils/settings.ts @@ -64,7 +64,6 @@ export const createTestSettings = (overrides: Partial = {}): Settings haystackApiKey: '', haystackWorkspace: '', haystackTemplatePipeline: '', - haystackPipelines: '', minAppVersion: '', ...overrides, }) From 723642cef05f6c37bbfdd625a02599128cbf7009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Wed, 29 Jul 2026 10:30:08 -0300 Subject: [PATCH 08/28] feat(agents): add deploy, status, and catalog endpoints --- backend/src/agents/routes.test.ts | 141 ++++++++++++++++++++++++++++++ backend/src/agents/routes.ts | 124 +++++++++++++++++++++++++- 2 files changed, 264 insertions(+), 1 deletion(-) diff --git a/backend/src/agents/routes.test.ts b/backend/src/agents/routes.test.ts index 901f310aa..71a125c05 100644 --- a/backend/src/agents/routes.test.ts +++ b/backend/src/agents/routes.test.ts @@ -16,6 +16,7 @@ import type { Auth } from '@/auth/elysia-plugin' import { clearSettingsCache } from '@/config/settings' import type { RemoteAgentDescriptor } from '@shared/acp-types' +import { schemaVersionMismatch, type AgentDescriptor } from '@shared/agent-descriptors' import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { Elysia } from 'elysia' import { registerAgentProvider, resetAgentProvidersForTesting } from './discovery' @@ -163,3 +164,143 @@ describe('GET /agents', () => { expect(body.agents).toEqual([customDescriptor]) }) }) + +describe('agent deploy endpoints', () => { + const envKeys = ['AGENT_DEPLOY', 'ENABLED_AGENTS'] as const + let savedEnv: Partial> + + const descriptor: AgentDescriptor = { + id: 'haystack', + provider: 'haystack', + name: 'Haystack', + description: null, + icon: null, + schemaVersion: 3, + action: 'deploy', + steps: [ + { id: 'basics', title: 'Basics', fields: [{ key: 'name', label: 'Name', widget: 'text', required: true }] }, + ], + } + + /** A deployable provider whose verbs echo their inputs so tests can assert dispatch. */ + const registerDeployable = () => + registerAgentProvider({ + id: 'haystack', + list: () => [], + catalog: () => [descriptor], + deploy: (spec) => Promise.resolve({ deploymentId: `haystack:tb-${String(spec.name)}`, status: 'pending' }), + status: (ref) => Promise.resolve({ deploymentId: `haystack:${ref}`, status: 'running', connection: null }), + }) + + const post = (app: Elysia, path: string, body: unknown) => + app.handle( + new Request(`http://localhost${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }), + ) + + beforeEach(() => { + resetAgentProvidersForTesting() + savedEnv = {} + for (const key of envKeys) { + savedEnv[key] = process.env[key] + delete process.env[key] + } + process.env.AGENT_DEPLOY = 'true' + clearSettingsCache() + }) + + afterEach(() => { + resetAgentProvidersForTesting() + for (const key of envKeys) { + const saved = savedEnv[key] + if (saved === undefined) { + delete process.env[key] + } else { + process.env[key] = saved + } + } + clearSettingsCache() + }) + + it('GET /catalog returns the descriptors when the flag is on', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/catalog')) + expect(res.status).toBe(200) + const body = await res.json() + expect(body.version).toBe('1') + expect(body.descriptors.map((d: AgentDescriptor) => d.id)).toEqual(['haystack']) + }) + + it('GET /catalog returns 404 when the agentDeploy flag is off', async () => { + delete process.env.AGENT_DEPLOY + clearSettingsCache() + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/catalog')) + expect(res.status).toBe(404) + }) + + it('GET /catalog returns 403 for anonymous users', async () => { + const app = buildApp(buildAuth({ id: 'anon-1', isAnonymous: true })) + const res = await app.handle(new Request('http://localhost/agents/catalog')) + expect(res.status).toBe(403) + }) + + it('POST /deploy dispatches a valid spec to the provider', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await post(app, '/agents/deploy', { descriptorId: 'haystack', schemaVersion: 3, spec: { name: 'Bot' } }) + expect(res.status).toBe(200) + const body = await res.json() + expect(body).toEqual({ deploymentId: 'haystack:tb-Bot', status: 'pending' }) + }) + + it('POST /deploy returns 409 on a stale schema version', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await post(app, '/agents/deploy', { descriptorId: 'haystack', schemaVersion: 2, spec: { name: 'Bot' } }) + expect(res.status).toBe(409) + const body = await res.json() + expect(body.code).toBe(schemaVersionMismatch) + }) + + it('POST /deploy returns 400 when the spec fails re-validation', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + // `name` is required by the descriptor but missing here. + const res = await post(app, '/agents/deploy', { descriptorId: 'haystack', schemaVersion: 3, spec: {} }) + expect(res.status).toBe(400) + }) + + it('POST /deploy returns 404 for an unknown descriptor', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await post(app, '/agents/deploy', { descriptorId: 'ghost', schemaVersion: 3, spec: { name: 'x' } }) + expect(res.status).toBe(404) + }) + + it('GET /deployments/:id returns live status from the provider', async () => { + registerDeployable() + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/deployments/haystack:tb-bot')) + expect(res.status).toBe(200) + const body = await res.json() + expect(body).toEqual({ deploymentId: 'haystack:tb-bot', status: 'running', connection: null }) + }) + + it('GET /deployments/:id returns 400 for a malformed id', async () => { + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/deployments/nocolon')) + expect(res.status).toBe(400) + }) + + it('GET /deployments/:id returns 404 for an unknown provider', async () => { + const app = buildApp(buildAuth({ id: 'user-1', isAnonymous: false })) + const res = await app.handle(new Request('http://localhost/agents/deployments/ghost:ref')) + expect(res.status).toBe(404) + }) +}) diff --git a/backend/src/agents/routes.ts b/backend/src/agents/routes.ts index 8cff1cb45..6136ae9e9 100644 --- a/backend/src/agents/routes.ts +++ b/backend/src/agents/routes.ts @@ -7,11 +7,25 @@ import { getEnabledAgentsList, getSettings, type Settings } from '@/config/setti import { createStandaloneLogger } from '@/config/logger' import { safeErrorHandler } from '@/middleware/error-handling' import type { AgentDiscoveryResponse, RemoteAgentDescriptor } from '@shared/acp-types' +import { + deployRequestSchema, + schemaVersionMismatch, + specSchemaForDescriptor, + type AgentCatalogResponse, + type AgentDescriptor, + type AgentSpec, + type DeployResponse, + type DeploymentStatusResponse, +} from '@shared/agent-descriptors' import type { User } from '@shared/types/auth' import { Elysia } from 'elysia' -import { getRegisteredProviders } from './discovery' +import { decodeDeploymentId } from './deployment-id' +import { getProviderById, getRegisteredProviders, type ProviderContext } from './discovery' import type { AgentsErrorResponse } from './types' +/** Generic error body for the deploy endpoints (may carry an app-specific `code`). */ +type ErrorBody = { error: string; code?: string } + /** * Mounts `GET /agents`, the ACP discovery endpoint. * @@ -63,6 +77,93 @@ export const createAgentsRoutes = (auth: Auth) => } }, ) + .get('/catalog', async ({ request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } + const settings = getSettings() + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + return { version: '1', descriptors: collectCatalog({ request, settings, userId: user.id }) } + }) + .post('/deploy', async ({ body, request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } + const settings = getSettings() + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + const parsed = deployRequestSchema.safeParse(body) + if (!parsed.success) { + set.status = 400 + return { error: 'Invalid deploy request' } + } + const ctx: ProviderContext = { request, settings, userId: user.id } + const descriptor = collectCatalog(ctx).find((d) => d.id === parsed.data.descriptorId) + if (!descriptor) { + set.status = 404 + return { error: 'Unknown agent' } + } + // Reject a spec built against a stale descriptor before we act on it. + if (descriptor.schemaVersion !== parsed.data.schemaVersion) { + set.status = 409 + return { error: 'Schema version mismatch', code: schemaVersionMismatch } + } + const provider = getProviderById(descriptor.provider) + if (!provider?.deploy) { + set.status = 404 + return { error: 'Agent is not deployable' } + } + // Re-validate the spec server-side — the client is never trusted. + const specResult = specSchemaForDescriptor(descriptor).safeParse(parsed.data.spec) + if (!specResult.success) { + set.status = 400 + return { error: 'Invalid spec' } + } + return provider.deploy(specResult.data as AgentSpec, ctx) + }) + .get('/deployments/:id', async ({ params, request, set, user }): Promise => { + if (!user) { + set.status = 401 + return { error: 'Unauthorized' } + } + if (user.isAnonymous) { + set.status = 403 + return { error: 'Forbidden', code: 'ANONYMOUS_DISCOVERY_FORBIDDEN' } + } + const settings = getSettings() + if (!settings.agentDeploy) { + set.status = 404 + return { error: 'Not found' } + } + let decoded: { provider: string; ref: string } + try { + decoded = decodeDeploymentId(params.id) + } catch { + set.status = 400 + return { error: 'Invalid deployment id' } + } + const provider = getProviderById(decoded.provider) + if (!provider?.status) { + set.status = 404 + return { error: 'Unknown provider' } + } + return provider.status(decoded.ref, { request, settings, userId: user.id }) + }) /** * Asks every registered provider for its descriptors and concatenates the @@ -81,3 +182,24 @@ const collectAgents = async (request: Request, settings: Settings): Promise { + const log = createStandaloneLogger(ctx.settings) + const out: AgentDescriptor[] = [] + for (const provider of getRegisteredProviders()) { + if (!provider.catalog) { + continue + } + try { + out.push(...provider.catalog(ctx)) + } catch (error) { + log.warn({ err: error, providerId: provider.id }, 'agent provider catalog() failed; skipping') + } + } + return out +} From 4e69b53b242d4999db21eb8a9b14770360f9d8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Wed, 29 Jul 2026 10:37:04 -0300 Subject: [PATCH 09/28] feat(agents): add frontend deploy API client --- src/api/agent-deploy.test.ts | 88 ++++++++++++++++++++++++++++++++++++ src/api/agent-deploy.ts | 59 ++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 src/api/agent-deploy.test.ts create mode 100644 src/api/agent-deploy.ts diff --git a/src/api/agent-deploy.test.ts b/src/api/agent-deploy.test.ts new file mode 100644 index 000000000..92f657b8e --- /dev/null +++ b/src/api/agent-deploy.test.ts @@ -0,0 +1,88 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { describe, expect, it } from 'bun:test' +import { HttpError, type HttpClient } from '@/lib/http' +import type { AgentCatalogResponse, DeployRequest } from '@shared/agent-descriptors' +import { deployAgent, fetchAgentCatalog, getDeploymentStatus } from './agent-deploy' + +type Call = { url: string; options?: { json?: unknown } } + +/** Build a fake HttpClient whose get/post resolve to `body`, recording calls. */ +const makeClient = (body: unknown, opts: { throwStatus?: number } = {}) => { + const calls: Call[] = [] + const respond = (url: string, options?: { json?: unknown }) => { + calls.push({ url, options }) + return { + json: async () => { + if (opts.throwStatus) { + throw new HttpError(new Response(null, { status: opts.throwStatus })) + } + return body + }, + } + } + const client = { + get: (url: string) => respond(url), + post: (url: string, options?: { json?: unknown }) => respond(url, options), + } as unknown as HttpClient + return { client, calls } +} + +const cloudUrl = 'https://api.test/v1' + +const catalogBody: AgentCatalogResponse = { + version: '1', + descriptors: [ + { + id: 'haystack', + provider: 'haystack', + name: 'Haystack', + description: null, + icon: null, + schemaVersion: 1, + action: 'deploy', + steps: [{ id: 'basics', title: 'Basics', fields: [{ key: 'name', label: 'Name', widget: 'text' }] }], + }, + ], +} + +describe('fetchAgentCatalog', () => { + it('returns the descriptors from the catalog envelope', async () => { + const { client, calls } = makeClient(catalogBody) + const descriptors = await fetchAgentCatalog(cloudUrl, client) + expect(calls[0].url).toBe('https://api.test/v1/agents/catalog') + expect(descriptors.map((d) => d.id)).toEqual(['haystack']) + }) + + it('returns [] when the feature is disabled (404)', async () => { + const { client } = makeClient(null, { throwStatus: 404 }) + expect(await fetchAgentCatalog(cloudUrl, client)).toEqual([]) + }) + + it('rethrows non-404 errors', async () => { + const { client } = makeClient(null, { throwStatus: 500 }) + await expect(fetchAgentCatalog(cloudUrl, client)).rejects.toThrow() + }) +}) + +describe('deployAgent', () => { + it('POSTs the request and parses the response', async () => { + const { client, calls } = makeClient({ deploymentId: 'haystack:tb-x', status: 'pending' }) + const request: DeployRequest = { descriptorId: 'haystack', schemaVersion: 1, spec: { name: 'Bot' } } + const result = await deployAgent(cloudUrl, client, request) + expect(calls[0].url).toBe('https://api.test/v1/agents/deploy') + expect(calls[0].options?.json).toEqual(request) + expect(result).toEqual({ deploymentId: 'haystack:tb-x', status: 'pending' }) + }) +}) + +describe('getDeploymentStatus', () => { + it('encodes the deployment id and parses the status', async () => { + const { client, calls } = makeClient({ deploymentId: 'haystack:tb-x', status: 'running', connection: null }) + const result = await getDeploymentStatus(cloudUrl, client, 'haystack:tb-x') + expect(calls[0].url).toBe('https://api.test/v1/agents/deployments/haystack%3Atb-x') + expect(result.status).toBe('running') + }) +}) diff --git a/src/api/agent-deploy.ts b/src/api/agent-deploy.ts new file mode 100644 index 000000000..76294957e --- /dev/null +++ b/src/api/agent-deploy.ts @@ -0,0 +1,59 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Frontend client for the descriptor-driven agent deploy endpoints (THU-743). + * Uses the app's authenticated {@link HttpClient} and validates every response + * against the shared zod contract so a drifting backend surfaces loudly. + */ + +import { HttpError, type HttpClient } from '@/lib/http' +import { + agentCatalogResponseSchema, + deployResponseSchema, + deploymentStatusResponseSchema, + type AgentDescriptor, + type DeployRequest, + type DeployResponse, + type DeploymentStatusResponse, +} from '@shared/agent-descriptors' + +/** + * Fetch the deployable-agent catalog. Returns `[]` when the deploy feature is + * disabled server-side (the endpoint 404s) so callers can treat "off" and + * "nothing to deploy" identically. + */ +export const fetchAgentCatalog = async (cloudUrl: string, httpClient: HttpClient): Promise => { + try { + const data = await httpClient.get(`${cloudUrl}/agents/catalog`).json() + return agentCatalogResponseSchema.parse(data).descriptors + } catch (err) { + if (err instanceof HttpError && err.response.status === 404) { + return [] + } + throw err + } +} + +/** Start a deploy from a validated spec. Returns the deployment id + initial status. */ +export const deployAgent = async ( + cloudUrl: string, + httpClient: HttpClient, + request: DeployRequest, +): Promise => { + const data = await httpClient.post(`${cloudUrl}/agents/deploy`, { json: request }).json() + return deployResponseSchema.parse(data) +} + +/** Poll a deployment's live status (fetched by the backend from the host). */ +export const getDeploymentStatus = async ( + cloudUrl: string, + httpClient: HttpClient, + deploymentId: string, +): Promise => { + const data = await httpClient + .get(`${cloudUrl}/agents/deployments/${encodeURIComponent(deploymentId)}`) + .json() + return deploymentStatusResponseSchema.parse(data) +} From 90736851ec74ed3303b45d0f2d93b9f66706d441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=AD=20Adriano?= Date: Wed, 29 Jul 2026 10:45:36 -0300 Subject: [PATCH 10/28] feat(agents): add descriptor-driven form renderer --- .../descriptor-form/descriptor-form.test.tsx | 64 +++++++++ .../descriptor-form/descriptor-form.tsx | 125 ++++++++++++++++++ .../descriptor-form/submittable-spec.test.ts | 54 ++++++++ .../descriptor-form/submittable-spec.ts | 22 +++ 4 files changed, 265 insertions(+) create mode 100644 src/components/agents/descriptor-form/descriptor-form.test.tsx create mode 100644 src/components/agents/descriptor-form/descriptor-form.tsx create mode 100644 src/components/agents/descriptor-form/submittable-spec.test.ts create mode 100644 src/components/agents/descriptor-form/submittable-spec.ts diff --git a/src/components/agents/descriptor-form/descriptor-form.test.tsx b/src/components/agents/descriptor-form/descriptor-form.test.tsx new file mode 100644 index 000000000..1c6f58109 --- /dev/null +++ b/src/components/agents/descriptor-form/descriptor-form.test.tsx @@ -0,0 +1,64 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import '@testing-library/jest-dom' +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it } from 'bun:test' +import type { AgentDescriptor } from '@shared/agent-descriptors' +import { DescriptorForm } from './descriptor-form' + +afterEach(cleanup) + +const conditionalDescriptor: AgentDescriptor = { + id: 'x', + provider: 'x', + name: 'X', + description: 'Deploy X', + icon: null, + schemaVersion: 1, + action: 'deploy', + steps: [ + { + id: 's', + title: 'S', + fields: [ + { key: 'name', label: 'Name', widget: 'text', required: true, placeholder: 'name' }, + { key: 'mode', label: 'Mode', widget: 'text', default: 'curated' }, + { key: 'apiKey', label: 'API key', widget: 'password', visibleWhen: { field: 'mode', equals: 'byo' } }, + ], + }, + ], +} + +const nameOnly: AgentDescriptor = { + ...conditionalDescriptor, + steps: [ + { + id: 's', + title: 'S', + fields: [{ key: 'name', label: 'Name', widget: 'text', required: true, placeholder: 'name' }], + }, + ], +} + +describe('DescriptorForm', () => { + it('renders visible fields and hides ones whose visibleWhen guard is unmet', () => { + render( {}} />) + expect(screen.getByText('Name')).toBeInTheDocument() + expect(screen.getByText('Mode')).toBeInTheDocument() + // mode defaults to 'curated', so the byo-only field is hidden. + expect(screen.queryByText('API key')).not.toBeInTheDocument() + }) + + it('shows the submit label and the descriptor description', () => { + render( {}} submitLabel="Deploy agent" />) + expect(screen.getByRole('button', { name: 'Deploy agent' })).toBeInTheDocument() + expect(screen.getByText('Deploy X')).toBeInTheDocument() + }) + + it('surfaces a submit-time error above the button', () => { + render( {}} error="Deploy failed" />) + expect(screen.getByText('Deploy failed')).toBeInTheDocument() + }) +}) diff --git a/src/components/agents/descriptor-form/descriptor-form.tsx b/src/components/agents/descriptor-form/descriptor-form.tsx new file mode 100644 index 000000000..729fc492e --- /dev/null +++ b/src/components/agents/descriptor-form/descriptor-form.tsx @@ -0,0 +1,125 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Renders an {@link AgentDescriptor} as a form and produces a validated spec. + * The descriptor owns the structure (steps → fields → widgets, `visibleWhen`); + * react-hook-form owns state, with `zodResolver(specSchemaForDescriptor)` — the + * same validator the backend re-runs — as the single source of validation truth. + */ + +import { useForm, type ControllerRenderProps, type Resolver } from 'react-hook-form' +import { zodResolver } from '@hookform/resolvers/zod' +import { + defaultSpec, + specSchemaForDescriptor, + visibleFields, + type AgentDescriptor, + type AgentField, + type AgentSpec, +} from '@shared/agent-descriptors' +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { Textarea } from '@/components/ui/textarea' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form' +import { submittableSpec } from './submittable-spec' + +type DescriptorFormProps = { + descriptor: AgentDescriptor + onSubmit: (spec: AgentSpec) => void | Promise + submitLabel?: string + isSubmitting?: boolean + /** A submit-time error (e.g. a failed deploy) shown above the button. */ + error?: string | null +} + +const inlineOptions = (field: AgentField) => (field.source?.kind === 'inline' ? field.source.options : []) + +/** Render the control for a single field's widget. Unknown widgets degrade to a note. */ +const FieldControl = ({ field, rhf }: { field: AgentField; rhf: ControllerRenderProps }) => { + const value = typeof rhf.value === 'string' ? rhf.value : '' + switch (field.widget) { + case 'textarea': + return