diff --git a/.changeset/mcp-host-bridge.md b/.changeset/mcp-host-bridge.md new file mode 100644 index 00000000..7fe9fa2f --- /dev/null +++ b/.changeset/mcp-host-bridge.md @@ -0,0 +1,5 @@ +--- +"@herdctl/core": minor +--- + +Add host-side MCP server support for Docker agents. Servers with `host: true` are spawned on the host and bridged into the container via HTTP, enabling MCP servers that need host resources (filesystem, credentials) while the agent runs in Docker. Also fix MCP HTTP bridge URLs to use `host.docker.internal` so containers can reach host-side bridges. diff --git a/packages/core/src/config/schema.ts b/packages/core/src/config/schema.ts index e6d200f5..2c58ae2f 100644 --- a/packages/core/src/config/schema.ts +++ b/packages/core/src/config/schema.ts @@ -495,6 +495,8 @@ export const McpServerSchema = z.object({ args: z.array(z.string()).optional(), env: z.record(z.string(), z.string()).optional(), url: z.string().optional(), + /** When true and Docker is enabled, run this MCP server on the host and bridge into the container */ + host: z.boolean().optional(), }); // ============================================================================= diff --git a/packages/core/src/runner/runtime/__tests__/mcp-host-bridge.test.ts b/packages/core/src/runner/runtime/__tests__/mcp-host-bridge.test.ts new file mode 100644 index 00000000..ed774d18 --- /dev/null +++ b/packages/core/src/runner/runtime/__tests__/mcp-host-bridge.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it, vi } from "vitest"; +import type { McpServer } from "../../../config/index.js"; + +// Mock the MCP SDK before importing the module under test +const mockConnect = vi.fn().mockResolvedValue(undefined); +const mockClose = vi.fn().mockResolvedValue(undefined); +const mockListTools = vi.fn().mockResolvedValue({ + tools: [ + { + name: "search_notes", + description: "Search Bear notes", + inputSchema: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }, + { + name: "create_note", + description: "Create a Bear note", + inputSchema: { + type: "object", + properties: { title: { type: "string" }, text: { type: "string" } }, + }, + }, + ], +}); +const mockCallTool = vi.fn().mockResolvedValue({ + content: [{ type: "text", text: "Note found: Hello World" }], + isError: false, +}); + +vi.mock("@modelcontextprotocol/sdk/client/index.js", () => { + return { + Client: class MockClient { + connect = mockConnect; + close = mockClose; + listTools = mockListTools; + callTool = mockCallTool; + }, + }; +}); + +vi.mock("@modelcontextprotocol/sdk/client/stdio.js", () => { + return { + StdioClientTransport: class MockTransport {}, + }; +}); + +import { createMcpHostBridge, partitionMcpServers } from "../mcp-host-bridge.js"; + +describe("partitionMcpServers", () => { + it("separates host and container servers", () => { + const servers: Record = { + bear: { command: "node", args: ["bear.js"], host: true }, + gmail: { command: "npx", args: ["gmail-mcp"] }, + contacts: { command: "python3", args: ["-m", "contacts"], host: true }, + }; + + const [host, container] = partitionMcpServers(servers); + + expect(Object.keys(host)).toEqual(["bear", "contacts"]); + expect(Object.keys(container)).toEqual(["gmail"]); + expect(host.bear.host).toBe(true); + expect(container.gmail.host).toBeUndefined(); + }); + + it("returns empty records for undefined input", () => { + const [host, container] = partitionMcpServers(undefined); + expect(Object.keys(host)).toHaveLength(0); + expect(Object.keys(container)).toHaveLength(0); + }); + + it("puts all servers in container when none have host: true", () => { + const servers: Record = { + a: { command: "a" }, + b: { command: "b" }, + }; + const [host, container] = partitionMcpServers(servers); + expect(Object.keys(host)).toHaveLength(0); + expect(Object.keys(container)).toHaveLength(2); + }); + + it("puts all servers in host when all have host: true", () => { + const servers: Record = { + a: { command: "a", host: true }, + b: { command: "b", host: true }, + }; + const [host, container] = partitionMcpServers(servers); + expect(Object.keys(host)).toHaveLength(2); + expect(Object.keys(container)).toHaveLength(0); + }); +}); + +describe("createMcpHostBridge", () => { + it("throws if no command is specified", async () => { + await expect(createMcpHostBridge("test", { host: true })).rejects.toThrow( + "no command specified", + ); + }); + + it("connects to the MCP server and discovers tools", async () => { + const config: McpServer = { + command: "node", + args: ["/srv/mcp-servers/bear/index.js"], + env: { BEAR_DB_PATH: "/data/bear.sqlite" }, + host: true, + }; + + const handle = await createMcpHostBridge("bear", config); + + expect(mockConnect).toHaveBeenCalled(); + expect(mockListTools).toHaveBeenCalled(); + expect(handle.serverDef.name).toBe("bear"); + expect(handle.serverDef.tools).toHaveLength(2); + expect(handle.serverDef.tools[0].name).toBe("search_notes"); + expect(handle.serverDef.tools[1].name).toBe("create_note"); + }); + + it("tool handler delegates to client.callTool", async () => { + const handle = await createMcpHostBridge("bear", { + command: "node", + args: ["bear.js"], + host: true, + }); + + const result = await handle.serverDef.tools[0].handler({ query: "hello" }); + + expect(mockCallTool).toHaveBeenCalledWith({ + name: "search_notes", + arguments: { query: "hello" }, + }); + expect(result.content[0].text).toBe("Note found: Hello World"); + expect(result.isError).toBe(false); + }); + + it("close() calls client.close()", async () => { + const handle = await createMcpHostBridge("bear", { + command: "node", + args: ["bear.js"], + host: true, + }); + + await handle.close(); + expect(mockClose).toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/runner/runtime/cli-runtime.ts b/packages/core/src/runner/runtime/cli-runtime.ts index 43471771..30de94dd 100644 --- a/packages/core/src/runner/runtime/cli-runtime.ts +++ b/packages/core/src/runner/runtime/cli-runtime.ts @@ -18,6 +18,7 @@ import { execa, type Subprocess } from "execa"; import { createLogger } from "../../utils/logger.js"; import { transformMcpServers } from "../sdk-adapter.js"; import type { SDKMessage } from "../types.js"; +import { type McpHttpBridge, startMcpHttpBridge } from "./mcp-http-bridge.js"; import { getCliSessionDir, getCliSessionFile, waitForNewSessionFile } from "./cli-session-path.js"; import { CLISessionWatcher } from "./cli-session-watcher.js"; import type { RuntimeExecuteOptions, RuntimeInterface } from "./interface.js"; @@ -63,6 +64,14 @@ interface CLIRuntimeOptions { * container session files are visible (e.g., .herdctl/docker-sessions). */ sessionDirOverride?: string; + + /** + * Hostname for MCP HTTP bridge URLs + * + * Defaults to '127.0.0.1'. For Docker execution, set to 'host.docker.internal' + * so the container can reach host-side bridges. + */ + mcpBridgeHost?: string; } /** @@ -98,6 +107,7 @@ interface CLIRuntimeOptions { export class CLIRuntime implements RuntimeInterface { private processSpawner: ProcessSpawner; private sessionDirOverride?: string; + private mcpBridgeHost: string; constructor(options?: CLIRuntimeOptions) { // Default to local execa spawning with prompt via stdin @@ -111,6 +121,7 @@ export class CLIRuntime implements RuntimeInterface { })); this.sessionDirOverride = options?.sessionDirOverride; + this.mcpBridgeHost = options?.mcpBridgeHost ?? "127.0.0.1"; } /** * Execute an agent using the Claude CLI @@ -176,6 +187,82 @@ export class CLIRuntime implements RuntimeInterface { args.push("--mcp-config", mcpConfig); } + // Track env mutation so we can restore it (see CLAUDE_CODE_STREAM_CLOSE_TIMEOUT below) + let savedStreamCloseTimeout: string | undefined | null = null; // null = not mutated + + // Start HTTP bridges for injected MCP servers (e.g., file sender) + // Same pattern as container-runner: expose in-process handlers via HTTP, + // then pass as HTTP-type MCP servers in --mcp-config + const bridges: McpHttpBridge[] = []; + if (options.injectedMcpServers && Object.keys(options.injectedMcpServers).length > 0) { + for (const [name, def] of Object.entries(options.injectedMcpServers)) { + let bridge: McpHttpBridge; + try { + bridge = await startMcpHttpBridge(def); + } catch (bridgeError) { + // Clean up any bridges that started successfully before this failure + for (const b of bridges) { + try { + await b.close(); + } catch { + // best-effort cleanup + } + } + bridges.length = 0; + throw bridgeError; + } + bridges.push(bridge); + + // Build or extend the --mcp-config to include this HTTP server + // Find existing --mcp-config arg index to merge with it + const mcpConfigIdx = args.indexOf("--mcp-config"); + let mcpConfig: { mcpServers: Record }; + + if (mcpConfigIdx !== -1 && mcpConfigIdx + 1 < args.length) { + // Parse existing config and add the bridge + mcpConfig = JSON.parse(args[mcpConfigIdx + 1]); + } else { + mcpConfig = { mcpServers: {} }; + } + + mcpConfig.mcpServers[name] = { + type: "http", + url: `http://${this.mcpBridgeHost}:${bridge.port}/mcp`, + }; + + const configJson = JSON.stringify(mcpConfig); + if (mcpConfigIdx !== -1) { + args[mcpConfigIdx + 1] = configJson; + } else { + args.push("--mcp-config", configJson); + } + + logger.debug(`Started MCP HTTP bridge for '${name}' on port ${bridge.port}`); + } + + // Auto-add injected MCP tool patterns to allowedTools. + // Only needed when the agent has an explicit allowlist — without one, all tools + // (including injected MCP tools) are allowed by default. + const allowedToolsIdx = args.indexOf("--allowedTools"); + if (allowedToolsIdx !== -1 && allowedToolsIdx + 1 < args.length) { + const existing = args[allowedToolsIdx + 1]; + const injectedPatterns = Object.keys(options.injectedMcpServers).map( + (name) => `mcp__${name}__*`, + ); + args[allowedToolsIdx + 1] = [existing, ...injectedPatterns].join(","); + } + + // File uploads via MCP tools can take longer than the default 60s timeout. + // Save the original value so we can restore it in `finally` to avoid leaking + // state across concurrent jobs. + if (options.injectedMcpServers["herdctl-file-sender"]) { + if (!process.env.CLAUDE_CODE_STREAM_CLOSE_TIMEOUT) { + savedStreamCloseTimeout = undefined; // marker: was not set + process.env.CLAUDE_CODE_STREAM_CLOSE_TIMEOUT = "120000"; + } + } + } + // Add session options if (options.resume) { args.push("--resume", options.resume); diff --git a/packages/core/src/runner/runtime/container-runner.ts b/packages/core/src/runner/runtime/container-runner.ts index a71728ba..f8e2b92e 100644 --- a/packages/core/src/runner/runtime/container-runner.ts +++ b/packages/core/src/runner/runtime/container-runner.ts @@ -30,6 +30,11 @@ import { buildContainerEnv, buildContainerMounts, ContainerManager } from "./con import type { DockerConfig } from "./docker-config.js"; import type { RuntimeExecuteOptions, RuntimeInterface } from "./interface.js"; import { type McpHttpBridge, startMcpHttpBridge } from "./mcp-http-bridge.js"; +import { + createMcpHostBridge, + type McpHostBridgeHandle, + partitionMcpServers, +} from "./mcp-host-bridge.js"; import { SDKRuntime } from "./sdk-runtime.js"; const logger = createLogger("ContainerRunner"); @@ -83,6 +88,43 @@ export class ContainerRunner implements RuntimeInterface { // Get or create container const container = await this.manager.getOrCreateContainer(agent.name, this.config, mounts, env); + // Partition host: true MCP servers — these run on the host and get bridged + const hostBridges: McpHostBridgeHandle[] = []; + let effectiveOptions = options; + + if (agent.mcp_servers) { + const [hostServers, containerServers] = partitionMcpServers(agent.mcp_servers); + + if (Object.keys(hostServers).length > 0) { + // Start host-side MCP bridges + for (const [name, config] of Object.entries(hostServers)) { + const handle = await createMcpHostBridge(name, config); + hostBridges.push(handle); + } + + // Build effective options with host servers moved to injectedMcpServers + const injected = { ...(options.injectedMcpServers ?? {}) }; + for (const handle of hostBridges) { + injected[handle.serverDef.name] = handle.serverDef; + } + + effectiveOptions = { + ...options, + agent: { + ...agent, + mcp_servers: + Object.keys(containerServers).length > 0 ? containerServers : undefined, + }, + injectedMcpServers: injected, + }; + + logger.info( + `Partitioned MCP servers: ${Object.keys(hostServers).length} host, ` + + `${Object.keys(containerServers).length} container`, + ); + } + } + try { // Get container ID for docker exec const containerInfo = await container.inspect(); @@ -90,11 +132,11 @@ export class ContainerRunner implements RuntimeInterface { // Handle CLI runtime with session file watching if (this.wrapped instanceof CLIRuntime) { - yield* this.executeCLIRuntime(containerId, dockerSessionsDir, options); + yield* this.executeCLIRuntime(containerId, dockerSessionsDir, effectiveOptions); } // Handle SDK runtime with wrapper script else if (this.wrapped instanceof SDKRuntime) { - yield* this.executeSDKRuntime(container, options); + yield* this.executeSDKRuntime(container, effectiveOptions); } // Unknown runtime type else { @@ -112,6 +154,15 @@ export class ContainerRunner implements RuntimeInterface { // Container cleanup happens in finally block } finally { + // Close host MCP bridges (kill subprocess + MCP client) + for (const handle of hostBridges) { + try { + await handle.close(); + } catch (err) { + logger.error(`Failed to close host MCP bridge: ${err}`); + } + } + // For ephemeral containers, stop immediately after execution // This triggers AutoRemove so the container is cleaned up automatically if (this.config.ephemeral) { @@ -146,6 +197,7 @@ export class ContainerRunner implements RuntimeInterface { ): AsyncIterable { // Create CLI runtime with Docker-specific spawner const cliRuntime = new CLIRuntime({ + mcpBridgeHost: "host.docker.internal", processSpawner: (args, _cwd, prompt, signal) => { // Build docker exec command with prompt piped to stdin // Uses printf to avoid issues with newlines and special chars in prompt diff --git a/packages/core/src/runner/runtime/mcp-host-bridge.ts b/packages/core/src/runner/runtime/mcp-host-bridge.ts new file mode 100644 index 00000000..e1e90f6c --- /dev/null +++ b/packages/core/src/runner/runtime/mcp-host-bridge.ts @@ -0,0 +1,136 @@ +/** + * MCP Host Bridge + * + * Spawns an MCP server process on the host and wraps it as an InjectedMcpServerDef. + * Used by ContainerRunner to run `host: true` MCP servers outside Docker while + * making their tools available to the containerized agent via HTTP bridge. + * + * Uses the MCP SDK Client with StdioClientTransport to communicate with the + * spawned process. Discovers tools via tools/list and creates handler functions + * that delegate to tools/call. + * + * @module mcp-host-bridge + */ + +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import type { McpServer } from "../../config/index.js"; +import { createLogger } from "../../utils/logger.js"; +import type { InjectedMcpServerDef, InjectedMcpToolDef, McpToolCallResult } from "../types.js"; + +const logger = createLogger("McpHostBridge"); + +// ============================================================================= +// Types +// ============================================================================= + +export interface McpHostBridgeHandle { + /** The InjectedMcpServerDef with tool handlers that delegate to the host process */ + serverDef: InjectedMcpServerDef; + /** Close the MCP client and kill the subprocess */ + close: () => Promise; +} + +// ============================================================================= +// Host Bridge +// ============================================================================= + +/** + * Spawn a host-side MCP server process and wrap it as an InjectedMcpServerDef. + * + * The spawned process communicates via stdio using the MCP protocol. + * Tools are discovered via tools/list and each tool's handler delegates + * to tools/call on the subprocess. + * + * @param name - Server name (used for logging and as the InjectedMcpServerDef name) + * @param config - MCP server config with command, args, env + * @returns Handle with the server definition and a close function + */ +export async function createMcpHostBridge( + name: string, + config: McpServer, +): Promise { + if (!config.command) { + throw new Error(`MCP server '${name}' has host: true but no command specified`); + } + + const transport = new StdioClientTransport({ + command: config.command, + args: config.args, + env: config.env ? { ...process.env, ...config.env } as Record : undefined, + }); + + const client = new Client( + { name: `herdctl-host-${name}`, version: "1.0.0" }, + { capabilities: {} }, + ); + await client.connect(transport); + + // Discover tools from the MCP server + const toolsResult = await client.listTools(); + const tools: InjectedMcpToolDef[] = (toolsResult.tools ?? []).map((tool) => ({ + name: tool.name, + description: tool.description ?? "", + inputSchema: (tool.inputSchema ?? {}) as Record, + handler: async (args: Record): Promise => { + const result = await client.callTool({ name: tool.name, arguments: args }); + const content = Array.isArray(result.content) + ? (result.content as Array<{ type: string; text: string }>) + : [{ type: "text", text: JSON.stringify(result) }]; + return { + content, + isError: result.isError === true, + }; + }, + })); + + logger.info(`Host MCP bridge '${name}' connected with ${tools.length} tool(s)`); + + return { + serverDef: { + name, + version: "1.0.0", + tools, + }, + close: async () => { + try { + await client.close(); + } catch (err) { + logger.error(`Failed to close MCP host bridge '${name}': ${err}`); + } + }, + }; +} + +// ============================================================================= +// Partition Utility +// ============================================================================= + +/** + * Partition MCP servers into host-side and container-side groups. + * + * Servers with `host: true` are separated out so they can be spawned on + * the host and bridged into the container. Only meaningful when Docker + * is enabled — when Docker is off, all servers run in-process regardless. + * + * @param mcpServers - All configured MCP servers for the agent + * @returns Tuple of [hostServers, containerServers] + */ +export function partitionMcpServers( + mcpServers: Record | undefined, +): [Record, Record] { + const hostServers: Record = {}; + const containerServers: Record = {}; + + if (!mcpServers) return [hostServers, containerServers]; + + for (const [name, server] of Object.entries(mcpServers)) { + if (server.host) { + hostServers[name] = server; + } else { + containerServers[name] = server; + } + } + + return [hostServers, containerServers]; +}