diff --git a/packages/jimmy/src/cli/migrate.ts b/packages/jimmy/src/cli/migrate.ts index 9590f4fc..e0f80ad0 100644 --- a/packages/jimmy/src/cli/migrate.ts +++ b/packages/jimmy/src/cli/migrate.ts @@ -18,6 +18,7 @@ import { getInstanceVersion, getPendingMigrations, } from "../shared/version.js"; +import type { BuiltInEngineName } from "../shared/types.js"; const GREEN = "\x1b[32m"; const YELLOW = "\x1b[33m"; @@ -171,6 +172,16 @@ export async function runMigrate(opts: { check?: boolean; auto?: boolean }): Pro const defaultEngine = config.engines.default ?? "claude"; const engineConfig = config.engines[defaultEngine] ?? config.engines.claude; + // migrate operates by spawning the CLI engine and chatting with it. + // HTTP-loop engines (ollama, openai) added in V1 have no `bin` — they + // aren't installed as a local binary. If the operator's default engine + // is HTTP, fall back to claude for the migration step. + // (TS narrowing across config.engines[union] doesn't preserve the + // required `bin: string` on the claude/codex/gemini intersection, so + // we coalesce explicitly.) + const cliEngineBin: string = engineConfig.bin ?? config.engines.claude.bin; + const cliEngineName: BuiltInEngineName = engineConfig.bin ? defaultEngine : "claude"; + try { const prompt = [ `Apply all pending migrations in ${MIGRATIONS_DIR}.`, @@ -184,10 +195,10 @@ export async function runMigrate(opts: { check?: boolean; auto?: boolean }): Pro `Clean up the migrations/ directory when done.`, ].join("\n"); - const args = buildMigrateArgs(defaultEngine, prompt); - console.log(`${DIM}Engine: ${defaultEngine} (${engineConfig.bin})${RESET}\n`); + const args = buildMigrateArgs(cliEngineName, prompt); + console.log(`${DIM}Engine: ${cliEngineName} (${cliEngineBin})${RESET}\n`); - execFileSync(engineConfig.bin, args, { + execFileSync(cliEngineBin, args, { stdio: "inherit", cwd: JINN_HOME, }); diff --git a/packages/jimmy/src/cron/runner.ts b/packages/jimmy/src/cron/runner.ts index b6fe6c3d..7970d55d 100644 --- a/packages/jimmy/src/cron/runner.ts +++ b/packages/jimmy/src/cron/runner.ts @@ -1,4 +1,5 @@ -import type { CronJob, Connector, JinnConfig } from "../shared/types.js"; +import type { BuiltInEngineName, CronJob, Connector, JinnConfig } from "../shared/types.js"; +import { modelFor } from "../shared/types.js"; import { logger } from "../shared/logger.js"; import { appendRunLog } from "./jobs.js"; import { scanOrg, findEmployee } from "../gateway/org.js"; @@ -64,7 +65,7 @@ export async function runCronJob( { employee, engine: job.engine || employee?.engine || config.engines.default, - model: job.model || employee?.model || config.engines[(job.engine || config.engines.default) as "claude" | "codex" | "gemini"]?.model, + model: job.model || employee?.model || modelFor(config.engines, (job.engine || config.engines.default) as BuiltInEngineName), title: job.name, }, ); diff --git a/packages/jimmy/src/engines/README.md b/packages/jimmy/src/engines/README.md new file mode 100644 index 00000000..6e3627da --- /dev/null +++ b/packages/jimmy/src/engines/README.md @@ -0,0 +1,321 @@ +# Jin Engines + +Jin's gateway routes sessions to one of several engines. This document +covers the V1 HTTP-loop engines (`ollama`, `openai`) added in PR #13 +and how they compose with the existing CLI-spawning engines (`claude`, +`codex`, `gemini`). + +## Engine roster + +| Engine | Style | V1 in this PR | Notes | +|-----------|-------------|---------------|-------| +| `claude` | CLI spawn | unchanged | Default; full tool surface via Claude Code | +| `codex` | CLI spawn | unchanged | `sessions.fallbackEngine` target on Claude rate-limit | +| `gemini` | CLI spawn | unchanged | Optional; opt-in via `engines.gemini` config block | +| `ollama` | HTTP loop | **NEW** | Self-hosted; cost always $0 | +| `openai` | HTTP loop | **NEW** | Cloud API; cost from per-model pricing table | +| `mock` | in-process | unchanged | Tests only; not configurable via `engines.default` | + +## Compatibility matrix + +What works on the HTTP-loop engines vs. the CLI engines: + +| | claude/codex/gemini | ollama/openai (V1) | +|------------------------------|:-:|:-:| +| One-shot prompt | ✅ | ✅ | +| Tool calls (read/write/edit/bash/webfetch) | ✅ | ✅ | +| Streaming to web UI | ✅ | ❌ (non-streaming) | +| `resumeSessionId` | ✅ | ❌ (errors before any provider call) | +| `mcpConfigPath` | ✅ | ❌ (errors before any provider call) | +| `attachments` | ✅ | ❌ (errors before any provider call) | +| `cliFlags` | ✅ | ❌ (no CLI to flag) | +| Sub-agent spawn (`Agent` tool) | ✅ | ❌ | +| TodoWrite / Plan / Skill tools | ✅ | ❌ | +| Rate-limit fallback target | (claude→codex) | not eligible | + +A cron that uses any ❌ feature MUST stay on Claude. Eligibility is per-cron, not engine-wide. + +## Release notes (V1) + +### `apiKeyEnvVar` / `authTokenEnvVar` store the env var NAME, not the value + +The config fields `engines.openai.apiKeyEnvVar` and +`engines.ollama.authTokenEnvVar` take the **name** of an environment +variable (e.g. `"OPENAI_API_KEY"`), not the secret itself. The engine +reads `process.env[name]` at construction. + +The constructors validate the field against the POSIX env-var-name +regex `^[A-Z_][A-Z0-9_]*$`. If the value looks like a secret (contains +hyphens, lowercase, `/`, etc.) the engine throws at construction with +guidance: + +> `openai.apiKeyEnvVar must be an env var NAME matching [A-Z_][A-Z0-9_]* (e.g. "OPENAI_API_KEY"), not the secret value itself. Got a string of length 47. Set openai.apiKeyEnvVar to the env var name; put the actual secret in your shell / .env file.` + +The error never echoes the value (we don't want a misconfigured config +file leaking the secret into gateway logs). + +**Recommended pattern:** leave the fields at their defaults +(`OPENAI_API_KEY` / `OLLAMA_TOKEN`) and source the secret from +`~/.jinn/.env` per the existing jin convention. Only set +`apiKeyEnvVar`/`authTokenEnvVar` when you need to point at a +differently-named env var (e.g. running multiple OpenAI accounts). + +## Startup contract (read this before opting in) + +**Declaring `engines.ollama` or `engines.openai` in `config.yaml` is a +startup contract.** The gateway will refuse to boot if the block is +present but construction fails (e.g. missing `url` for ollama, missing +API key env var for openai). The error message includes the failing +engine name and the guidance: + +> → Fix the config OR remove the engines. block to opt out. + +To opt out cleanly, remove the entire block. To keep it enabled, ensure +the required url / env var is set before booting. + +This is intentional. Silent skip-on-construction-failure was the old +behavior in earlier review iterations; it was rejected because it lets +cron jobs that target `engine: "ollama"` succeed in writing run-log +entries while their actual sessions fail downstream. Loud boot failure +forces the operator to either fix the misconfiguration or make the +opt-out explicit. + +## Config + +Add one or both blocks under `engines:` in `~/.jinn/config.yaml`: + +```yaml +engines: + default: claude # cannot be "mock" + claude: # required, unchanged + bin: claude + model: opus + codex: # required, unchanged + bin: codex + model: gpt-5.4 + + # New: opt-in. Remove either block entirely to opt out. + ollama: + url: https://ollama.aga.my + model: qwen2.5:7b-instruct + maxTurns: 25 + timeoutMs: 300000 # whole-loop wall clock (ms) + providerTimeoutMs: 60000 # per-HTTP-call timeout (ms) + authTokenEnvVar: OLLAMA_TOKEN # optional; default OLLAMA_TOKEN + tools: + enabled: [read, write, edit, bash, webfetch] + bashAllowlist: + - git + - curl + - python3 + - sqlite3 + - jq + webfetch: + allowPrivate: false # default; true only for trusted internal use + + openai: + baseUrl: https://api.openai.com/v1 # default + apiKeyEnvVar: OPENAI_API_KEY # default + model: gpt-4o-mini + maxTurns: 25 + timeoutMs: 300000 + tools: + enabled: [read, write, edit, webfetch] # bash deliberately omitted here +``` + +### Tool configuration defaults (V1 deny-by-default) + +- `tools.enabled` undefined or `[]` → **text-only mode**. The model + receives an empty `tools` array; it cannot call any tool. Useful for + pure-classification / pure-summarization cron jobs. +- `tools.enabled: ["bash"]` with no `bashAllowlist` → bash tool is + registered but every call returns `error: "disabled"`. The deny-by- + default posture means you must explicitly list the executables you + trust. +- The hardcoded NEVER-LIST overrides any allowlist: + `sh, bash, zsh, fish, ksh, csh, tcsh, dash, ash, env, xargs, eval, + exec, source` are refused even if added to `bashAllowlist`. +- `python3` invocations get extra scrutiny: must include a positional + script path that resolves under `cwd` and exists; `-c`, `-m`, `-`, + `-i` flags are rejected. +- Filesystem tools (`read`, `write`, `edit`) are jailed under the + session `cwd` via two-stage check (lexical + realpath). Symbolic + links pointing outside the jail are refused; for `write`/`edit`, + any symlink leaf is refused regardless of target. +- `webfetch`: http/https only, max 5 same-scheme redirects, custom + DNS lookup validates the actual socket address at connect time + (DNS-rebinding mitigation), private/loopback/link-local IPs + refused unless `tools.webfetch.allowPrivate: true`. + +## Cost reporting + +| Engine | Cost calculation | Behavior on unknown model | +|--------|------------------|----------------------------| +| ollama | always `0` | n/a | +| openai | `(prompt_tokens × in_rate + completion_tokens × out_rate) / 1e6` from `providers/pricing.ts` | returns `cost: undefined` (NOT 0) so the `cost_log` row records `NULL` | + +Pricing uses `response.model` (what the provider actually billed) and +falls back to the requested model only when the response omits it. +An unknown model logs a `logger.warn` so it surfaces in the weekly +rollup as a "pricing gap" signal. + +## Audit log + +Every tool call produces an `AuditRow` with this exact shape: + +```ts +{ + toolName: string; // "read" | "write" | "edit" | "bash" | "webfetch" + argsSummary: string; // JSON.stringify(sanitized args) + durationMs: number; // wall-clock for the tool call + error: string | null; // short code or null on success + truncated: boolean; // any output stream truncated? + resultBytes: number | null; // pre-truncation byte count + exitCode: number | null; // bash only + httpStatus: number | null; // webfetch only +} +``` + +**Audit rows never contain the tool's output content** (stdout, stderr, +file body, HTTP response body). The model already saw that content in +its conversation; logging it twice doubles storage and creates a leak +surface for secrets the model observed. + +The `argsSummary` is sanitized before serialization: + +- Object keys matching `api_key`, `authorization`, `token`, `secret`, + `password`, `bearer`, `cookie` (case-insensitive) → value replaced + with `[redacted]`. +- URL strings with credentials are stripped: `https://user:pass@host` + loses the userinfo; `?api_key=...`, `?token=...`, `?password=...`, + `?signature=...` query values redacted while preserving the key + name as a debug signal. +- Long string values capped at 200 chars + `…[N more]` marker. +- Recursion depth capped at 5. + +In V1 the `AuditLogger` interface is pluggable but the sink is **not +yet wired to sqlite**. The engine wrappers accept an optional +`{ audit: AuditLogger }` constructor argument; production wiring (to +`~/.jinn/sessions/registry.db tool_call_log`) ships in a follow-up. +Until then, audit calls go to a no-op. Audit-sink failures (when wired) +do NOT abort the agent loop; they log via `logger.warn` so persistent +issues are visible. + +## Migration recipe + +Move a Claude cron to ollama or openai when: +- The cron's task is one of: BM polish, classification, summarization, + URL triage, transcript reading. +- The cron does NOT use sub-agents, MCP, attachments, or session resume. + +### Step-by-step + +1. **Pick the target engine.** Default to `openai` for tasks needing + strong reasoning; default to `ollama` for high-volume tasks where + cost matters more than ceiling quality. + +2. **Validate config block** in `config.yaml` before touching the cron. + Boot the gateway once after adding the block. If it doesn't start, + fix the error or remove the block. + +3. **Duplicate the cron job** in `~/.jinn/cron/jobs.json`: + - Same prompt + - `engine: "ollama"` or `engine: "openai"` + - `name: "-shadow"` + - `enabled: false` + - `schedule`: offset by a few minutes from the original + +4. **Enable the shadow.** Let it run alongside the Claude cron for + 3 wall-clock days, minimum 3 fires each. + +5. **Compare deliverables side-by-side.** What you actually need to + verify: + - Issue rows / news rows written with the same structural shape + - Headlines / classifications match on factual claims + - `cost_log.cost_usd` for the shadow is at least N× cheaper than + the Claude run (where N matches your savings target) + + **Do not rely on cron run-log status alone for migration monitoring.** + The run-log records the synchronous `route()` return value of + `sessions.manager.route()`. If the actual `runSession()` fails + asynchronously after route() has returned (engine missing, provider + error, max_turns), the run-log still says `success`. This is a + pre-existing limitation across all engines and is out of scope for + V1 to change. Use the SHADOW's actual delivered outputs, the + `sessions` table `last_error` column, and gateway logs (look for + `Engine "" not available` or `: :` lines) + to gate the migration decision. + +6. **If parity holds**: flip the original cron to the new engine, + leave the shadow disabled as a rollback artifact. Watch for one + more week before deleting the shadow entry. + +7. **If parity fails**: keep the original on Claude. Either tighten the + prompt for the new engine, switch target model, or accept that the + task isn't eligible for migration. + +### Rollback + +One JSON edit: flip `engine` back to `"claude"` on the original cron +entry. The next gateway reload (or restart) picks it up. + +## Operational visibility + +- `GET /api/status` returns one entry per **registered** engine. An + opt-in HTTP engine that's declared in config but failed construction + is NOT in the response (the gateway never booted in that state). +- Cron route failures (manager.ts:230) deliver the message + `Error: engine "" not available.` via the cron's configured + delivery connector (Telegram, WhatsApp, etc.). The error is also + logged at `error` level. +- `sessions.fallbackEngine` still only accepts `"codex"` — this is the + existing Claude rate-limit fallback, NOT a provider-fallback + mechanism. There is no automatic ollama→openai or openai→claude + routing in V1. + +## Known V1 limitations (carry into the docs of any cron that opts in) + +- No streaming responses (not exposed in web UI). +- No resume of previous sessions (`resumeSessionId` rejected before + provider call). +- No MCP server support. +- No file attachments. +- No sub-agent spawning (`Agent` tool isn't in the registry). +- No automatic provider-level fallback. +- Audit-log sqlite writer not yet wired (interface is pluggable; + default is no-op). +- Cron run-log `success` status is async-decoupled from session + outcome — see migration recipe step 5. + +## File map + +``` +packages/jimmy/src/engines/ +├── ollama.ts # Engine wrapper +├── openai.ts # Engine wrapper (reuses rejectUnsupported) +├── agentLoop.ts # Provider-agnostic loop +├── audit.ts # AuditLogger interface + sanitizer +├── providers/ +│ ├── ollama.ts # HTTP adapter +│ ├── openai.ts # HTTP adapter +│ ├── pricing.ts # Per-model rate table +│ └── types.ts # NormalizedToolCall, ProviderMessage, ... +├── tools/ +│ ├── index.ts # buildToolRegistry +│ ├── schemas.ts # JSON schemas exposed to the model +│ ├── cwdJail.ts # lexical + realpath jail +│ ├── ipBlocklist.ts # IPv4/IPv6 block ranges for webfetch +│ ├── read.ts / write.ts / edit.ts +│ ├── runCommand.ts # argv-only "bash" tool +│ └── webfetch.ts # http/https with DNS-rebinding mitigation +└── __tests__/ + ├── audit.test.ts + ├── agentLoop.test.ts + ├── wrappers.test.ts + ├── buildLookup.test.ts + ├── report-url.fixture.test.ts + └── fixtures/ + └── report-url/ + ├── input.txt + └── expected-classification.schema.json +``` diff --git a/packages/jimmy/src/engines/__tests__/agentLoop.test.ts b/packages/jimmy/src/engines/__tests__/agentLoop.test.ts new file mode 100644 index 00000000..24bfefe9 --- /dev/null +++ b/packages/jimmy/src/engines/__tests__/agentLoop.test.ts @@ -0,0 +1,472 @@ +import { describe, it, expect, vi } from "vitest"; +import { runAgentLoop, type AgentLoopOpts } from "../agentLoop.js"; +import type { + NormalizedToolCall, + ProviderCall, + ProviderCallResult, + ProviderMessage, +} from "../providers/types.js"; +import type { ToolExecutor } from "../tools/index.js"; +import type { AuditLogger, AuditRow } from "../audit.js"; +import type { JsonObject } from "../../shared/types.js"; + +// ─── Mock provider ─────────────────────────────────────────────────── + +/** + * Scriptable provider. Each call consumes the next script entry. + * If `scripts[i]` is a function, it's invoked with the request opts so + * the test can assert on the message history. If it's an Error, the + * provider throws (simulating parse/transport failure). + */ +type ProviderScript = ProviderCallResult | Error | ((opts: { messages: ProviderMessage[] }) => ProviderCallResult | Error); + +function mockProvider(scripts: ProviderScript[]): ProviderCall { + let i = 0; + return async (opts) => { + if (i >= scripts.length) throw new Error(`provider script exhausted at call ${i + 1}`); + let entry = scripts[i++]!; + if (typeof entry === "function") { + entry = entry({ messages: opts.messages }); + } + if (entry instanceof Error) throw entry; + return entry; + }; +} + +function assistantText(content: string, usage = { promptTokens: 10, completionTokens: 5 }): ProviderCallResult { + return { + message: { role: "assistant", content }, + finishReason: "stop", + usage, + billedModel: "gpt-4o-mini", + }; +} + +function assistantToolCall( + toolCalls: NormalizedToolCall[], + usage = { promptTokens: 10, completionTokens: 5 }, +): ProviderCallResult { + return { + message: { role: "assistant", content: "", toolCalls }, + finishReason: "tool_calls", + usage, + billedModel: "gpt-4o-mini", + }; +} + +function tc(name: string, args: JsonObject, id = `call_${Math.random().toString(36).slice(2, 10)}`): NormalizedToolCall { + return { id, name, arguments: args }; +} + +function fakeExec(content: string, ok = true): ToolExecutor { + return async () => ({ + ok, + content, + audit: { truncated: false, originalBytes: content.length }, + }); +} + +function baseOpts(overrides: Partial): AgentLoopOpts { + return { + provider: mockProvider([assistantText("default")]), + toolExecutors: new Map(), + toolSchemas: [], + model: "gpt-4o-mini", + userPrompt: "hi", + maxTurns: 5, + timeoutMs: 5000, + toolContext: { cwd: process.cwd() }, + ...overrides, + }; +} + +// ─── Text-only mode ────────────────────────────────────────────────── + +describe("agentLoop: text-only mode (no tools)", () => { + it("returns the assistant message after one turn with no tools exposed", async () => { + const provider = mockProvider([assistantText("hello back", { promptTokens: 12, completionTokens: 4 })]); + const result = await runAgentLoop(baseOpts({ provider })); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + expect(result.finalContent).toBe("hello back"); + expect(result.turns).toBe(1); + expect(result.promptTokens).toBe(12); + expect(result.completionTokens).toBe(4); + expect(result.billedModels).toEqual(["gpt-4o-mini"]); + expect(result.toolMessages).toEqual([]); + }); + + it("passes an empty tools array to the provider when none are registered", async () => { + const seen: number[] = []; + const provider: ProviderCall = async (opts) => { + seen.push(opts.tools.length); + return assistantText("ok"); + }; + await runAgentLoop(baseOpts({ provider })); + expect(seen).toEqual([0]); + }); +}); + +// ─── Single tool call ──────────────────────────────────────────────── + +describe("agentLoop: single tool turn", () => { + it("executes one tool call and then returns the terminal assistant message", async () => { + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x.txt" }, "c1")]), + assistantText("file said: hello"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("hello")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + expect(result.finalContent).toBe("file said: hello"); + expect(result.turns).toBe(2); + expect(result.toolMessages).toHaveLength(1); + expect(result.toolMessages[0].toolCallId).toBe("c1"); + expect(result.toolMessages[0].content).toBe("hello"); + }); +}); + +// ─── Multi-tool turn (parallel calls in one assistant turn) ────────── + +describe("agentLoop: multi-tool turn", () => { + it("executes both tool calls in a single assistant turn", async () => { + const provider = mockProvider([ + assistantToolCall([ + tc("read", { path: "a.txt" }, "c-a"), + tc("read", { path: "b.txt" }, "c-b"), + ]), + assistantText("done both"), + ]); + const toolExecutors = new Map(); + const exec = vi.fn(async (args: JsonObject) => ({ + ok: true, + content: `read ${(args as { path: string }).path}`, + audit: { truncated: false }, + })); + toolExecutors.set("read", exec as unknown as ToolExecutor); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + expect(exec).toHaveBeenCalledTimes(2); + expect(result.toolMessages).toHaveLength(2); + expect(result.toolMessages.map((m) => m.content)).toEqual(["read a.txt", "read b.txt"]); + }); +}); + +// ─── Max-turn exhaustion ──────────────────────────────────────────── + +describe("agentLoop: max_turns exhaustion", () => { + it("returns kind=max_turns when the model never stops calling tools", async () => { + const looping: ProviderScript[] = []; + for (let i = 0; i < 10; i++) { + looping.push(assistantToolCall([tc("read", { path: `${i}.txt` }, `c${i}`)])); + } + const provider = mockProvider(looping); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("...")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors, maxTurns: 3 })); + expect(result.kind).toBe("max_turns"); + if (result.kind !== "max_turns") return; + expect(result.turns).toBe(3); + }); +}); + +// ─── Wall-clock timeout: before provider call ──────────────────────── + +describe("agentLoop: timeout before provider call", () => { + it("returns kind=timeout when the deadline expires between turns", async () => { + // Turn 1: provider returns tool call instantly, tool burns 200ms. + // Budget = 100ms — so by the time the loop comes back for turn 2, + // the deadline has long passed and the gate-before-provider-call + // trips. + let providerCalls = 0; + const provider: ProviderCall = async () => { + providerCalls++; + if (providerCalls === 1) { + return assistantToolCall([tc("read", { path: "x" }, "c1")]); + } + return assistantText("never reached — should be gated out"); + }; + const slowTool: ToolExecutor = async () => { + await new Promise((r) => setTimeout(r, 200)); + return { ok: true, content: "done", audit: { truncated: false } }; + }; + const toolExecutors = new Map(); + toolExecutors.set("read", slowTool); + const result = await runAgentLoop( + baseOpts({ provider, toolExecutors, timeoutMs: 100 }), + ); + expect(result.kind).toBe("timeout"); + if (result.kind !== "timeout") return; + expect(result.message).toMatch(/before provider call/); + expect(providerCalls).toBe(1); + }); +}); + +// ─── Wall-clock timeout: before tool call ──────────────────────────── + +describe("agentLoop: timeout before tool call", () => { + it("returns kind=timeout if the deadline expires between provider and tool", async () => { + // Provider returns instantly with two tool calls. The first tool takes + // 100ms. Budget = 80ms so deadline passes mid-stream. + const provider = mockProvider([ + assistantToolCall([ + tc("read", { path: "a" }, "c1"), + tc("read", { path: "b" }, "c2"), + ]), + assistantText("never"), + ]); + const slowExec: ToolExecutor = async () => { + await new Promise((r) => setTimeout(r, 100)); + return { ok: true, content: "slow", audit: { truncated: false } }; + }; + const toolExecutors = new Map(); + toolExecutors.set("read", slowExec); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors, timeoutMs: 80 })); + expect(result.kind).toBe("timeout"); + if (result.kind !== "timeout") return; + expect(result.message).toMatch(/before tool call/); + }); +}); + +// ─── Unknown tool ──────────────────────────────────────────────────── + +describe("agentLoop: unknown tool", () => { + it("feeds an unknown_tool error back to the model and continues", async () => { + const provider = mockProvider([ + assistantToolCall([tc("fictional_tool", { x: 1 }, "c1")]), + // After receiving the error, the model gives up and answers. + assistantText("I tried; that tool doesn't exist."), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("...")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + expect(result.turns).toBe(2); + expect(result.toolMessages).toHaveLength(1); + const parsed = JSON.parse(result.toolMessages[0].content); + expect(parsed.error).toBe("unknown_tool"); + expect(parsed.requested).toBe("fictional_tool"); + expect(parsed.available).toEqual(["read"]); + }); +}); + +// ─── Tool that throws ──────────────────────────────────────────────── + +describe("agentLoop: tool executor exception", () => { + it("catches and surfaces tool exceptions as structured tool messages", async () => { + const throwingExec: ToolExecutor = async () => { + throw new Error("kaboom"); + }; + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")]), + assistantText("understood, moving on"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", throwingExec); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + const parsed = JSON.parse(result.toolMessages[0].content); + expect(parsed.error).toBe("tool_exception"); + expect(parsed.message).toBe("kaboom"); + }); +}); + +// ─── Provider parse/transport errors ───────────────────────────────── + +describe("agentLoop: provider parse / transport errors", () => { + it("aborts the loop with kind=provider_error when the adapter throws", async () => { + const provider = mockProvider([new Error("malformed tool_calls JSON")]); + const result = await runAgentLoop(baseOpts({ provider })); + expect(result.kind).toBe("provider_error"); + if (result.kind !== "provider_error") return; + expect(result.message).toMatch(/malformed/); + expect(result.turns).toBe(0); + }); + + it("aborts on provider error mid-loop", async () => { + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")]), + new Error("transport ECONNRESET"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + expect(result.kind).toBe("provider_error"); + if (result.kind !== "provider_error") return; + expect(result.message).toMatch(/ECONNRESET/); + expect(result.turns).toBe(1); // we did one full turn before the second provider call failed + }); +}); + +// ─── Audit logger integration ──────────────────────────────────────── + +describe("agentLoop: audit integration", () => { + it("records one AuditRow per tool call (including unknown / exception cases)", async () => { + const rows: AuditRow[] = []; + const audit: AuditLogger = { record: (r) => { rows.push(r); } }; + const provider = mockProvider([ + assistantToolCall([ + tc("read", { path: "x" }, "c1"), + tc("fictional", { y: 1 }, "c2"), + ]), + assistantText("done"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors, audit })); + expect(result.kind).toBe("ok"); + expect(rows).toHaveLength(2); + expect(rows[0].toolName).toBe("read"); + expect(rows[0].error).toBeNull(); + expect(rows[1].toolName).toBe("fictional"); + expect(rows[1].error).toBe("unknown_tool"); + }); + + it("audit failures do not break the loop", async () => { + const audit: AuditLogger = { + record: () => { + throw new Error("sqlite locked"); + }, + }; + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")]), + assistantText("done"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors, audit })); + expect(result.kind).toBe("ok"); + }); +}); + +// ─── Token accounting and model attribution ────────────────────────── + +describe("agentLoop: token + model accounting", () => { + it("accumulates prompt / completion tokens across turns", async () => { + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")], { promptTokens: 10, completionTokens: 3 }), + assistantText("done", { promptTokens: 25, completionTokens: 7 }), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + if (result.kind !== "ok") throw new Error("expected ok"); + expect(result.promptTokens).toBe(35); + expect(result.completionTokens).toBe(10); + }); + + it("dedupes billed model identifiers when the provider routes consistently", async () => { + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")]), + assistantText("done"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors })); + if (result.kind !== "ok") throw new Error("expected ok"); + expect(result.billedModels).toEqual(["gpt-4o-mini"]); + }); +}); + +// ─── durationMs on every result kind ───────────────────────────────── + +describe("agentLoop: durationMs is set on every result kind", () => { + it("ok result", async () => { + const r = await runAgentLoop(baseOpts({ provider: mockProvider([assistantText("hi")]) })); + expect(typeof r.durationMs).toBe("number"); + expect(r.durationMs).toBeGreaterThanOrEqual(0); + }); + + it("provider_error result", async () => { + const r = await runAgentLoop(baseOpts({ provider: mockProvider([new Error("nope")]) })); + expect(typeof r.durationMs).toBe("number"); + }); + + it("max_turns result", async () => { + const scripts = Array.from({ length: 8 }, () => + assistantToolCall([tc("read", { path: "x" }, "c")]), + ); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const r = await runAgentLoop( + baseOpts({ provider: mockProvider(scripts), toolExecutors, maxTurns: 2 }), + ); + expect(r.kind).toBe("max_turns"); + expect(typeof r.durationMs).toBe("number"); + }); + + it("timeout result", async () => { + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")]), + assistantText("never"), + ]); + const slow: ToolExecutor = async () => { + await new Promise((r) => setTimeout(r, 200)); + return { ok: true, content: "x", audit: { truncated: false } }; + }; + const toolExecutors = new Map(); + toolExecutors.set("read", slow); + const r = await runAgentLoop( + baseOpts({ provider, toolExecutors, timeoutMs: 100 }), + ); + expect(r.kind).toBe("timeout"); + expect(r.durationMs).toBeGreaterThanOrEqual(100); + }); +}); + +// ─── Audit-failure visibility ──────────────────────────────────────── + +describe("agentLoop: audit failures surface in logger.warn", () => { + it("audit.record() throwing produces a logger.warn line", async () => { + const { logger } = await import("../../shared/logger.js"); + const warnSpy = vi.spyOn(logger, "warn").mockImplementation(() => {}); + try { + const audit: AuditLogger = { + record: () => { + throw new Error("sqlite locked"); + }, + }; + const provider = mockProvider([ + assistantToolCall([tc("read", { path: "x" }, "c1")]), + assistantText("done"), + ]); + const toolExecutors = new Map(); + toolExecutors.set("read", fakeExec("ok")); + const result = await runAgentLoop(baseOpts({ provider, toolExecutors, audit })); + expect(result.kind).toBe("ok"); + expect(warnSpy).toHaveBeenCalledTimes(1); + const msg = String(warnSpy.mock.calls[0]?.[0] ?? ""); + expect(msg).toMatch(/audit sink failed/); + expect(msg).toMatch(/sqlite locked/); + expect(msg).toMatch(/"read"/); + } finally { + warnSpy.mockRestore(); + } + }); +}); + +// ─── Unknown-tool loop converges to max_turns ──────────────────────── + +describe("agentLoop: repeated unknown tool eventually hits max_turns", () => { + it("converges cleanly when the model keeps calling unknown tools", async () => { + const scripts = Array.from({ length: 5 }, () => + assistantToolCall([tc("nonexistent", { x: 1 }, "c-x")]), + ); + const provider = mockProvider(scripts); + const toolExecutors = new Map(); // empty registry + const result = await runAgentLoop( + baseOpts({ provider, toolExecutors, maxTurns: 4 }), + ); + expect(result.kind).toBe("max_turns"); + if (result.kind !== "max_turns") return; + expect(result.turns).toBe(4); + // Token accumulation continues across unknown-tool turns. + expect(result.promptTokens).toBe(40); // 4 turns × 10 + expect(result.completionTokens).toBe(20); // 4 turns × 5 + }); +}); diff --git a/packages/jimmy/src/engines/__tests__/audit.test.ts b/packages/jimmy/src/engines/__tests__/audit.test.ts new file mode 100644 index 00000000..03ebebf1 --- /dev/null +++ b/packages/jimmy/src/engines/__tests__/audit.test.ts @@ -0,0 +1,213 @@ +import { describe, it, expect } from "vitest"; +import { sanitizeArgsForAudit, buildAuditRow } from "../audit.js"; +import type { ToolResult } from "../tools/types.js"; + +describe("audit: sanitizeArgsForAudit redaction", () => { + it("redacts api_key field", () => { + const s = sanitizeArgsForAudit({ api_key: "sk-abc123", q: "hi" }); + const parsed = JSON.parse(s); + expect(parsed.api_key).toBe("[redacted]"); + expect(parsed.q).toBe("hi"); + }); + + it("redacts authorization header case-insensitively", () => { + const s = sanitizeArgsForAudit({ Authorization: "Bearer abc", method: "GET" }); + const parsed = JSON.parse(s); + expect(parsed.Authorization).toBe("[redacted]"); + }); + + it("redacts apiKey camelCase", () => { + const parsed = JSON.parse(sanitizeArgsForAudit({ apiKey: "x" })); + expect(parsed.apiKey).toBe("[redacted]"); + }); + + it("redacts nested secret fields", () => { + const s = sanitizeArgsForAudit({ + headers: { Authorization: "Bearer xyz", "X-Custom": "fine" }, + }); + const parsed = JSON.parse(s); + expect(parsed.headers.Authorization).toBe("[redacted]"); + expect(parsed.headers["X-Custom"]).toBe("fine"); + }); + + it("redacts password, token, secret, cookie keys", () => { + const s = sanitizeArgsForAudit({ + password: "p", + token: "t", + secret: "s", + cookie: "c", + keep: "ok", + }); + const p = JSON.parse(s); + expect(p.password).toBe("[redacted]"); + expect(p.token).toBe("[redacted]"); + expect(p.secret).toBe("[redacted]"); + expect(p.cookie).toBe("[redacted]"); + expect(p.keep).toBe("ok"); + }); + + it("truncates long string values to 200 chars + marker", () => { + const long = "x".repeat(500); + const s = sanitizeArgsForAudit({ blob: long }); + const p = JSON.parse(s); + expect(p.blob.length).toBeLessThan(300); + expect(p.blob).toMatch(/more\]$/); + }); + + it("caps deep nesting at depth-5", () => { + let v: unknown = "deep"; + for (let i = 0; i < 12; i++) v = { nested: v }; + // Cast through unknown — intentional test-only escape hatch since + // JsonObject's index signature is JsonValue and we want a deep tree. + const s = sanitizeArgsForAudit(v as never); + expect(s).toMatch(/depth-capped/); + }); + + it("walks arrays", () => { + const s = sanitizeArgsForAudit({ items: [{ token: "x" }, "fine"] }); + const p = JSON.parse(s); + expect(p.items[0].token).toBe("[redacted]"); + expect(p.items[1]).toBe("fine"); + }); +}); + +describe("audit: URL credential redaction", () => { + it("redacts ?api_key= query parameter", () => { + const s = sanitizeArgsForAudit({ url: "https://example.com/path?api_key=secret123&q=hi" }); + expect(s).not.toContain("secret123"); + // URLSearchParams URL-encodes the value, so we accept either form. + expect(s).toMatch(/api_key=(\[redacted\]|%5Bredacted%5D)/); + expect(s).toContain("q=hi"); // non-secret query params preserved + }); + + it("redacts ?token= and ?access_token=", () => { + const s = sanitizeArgsForAudit({ url: "https://x.com/?token=abc&access_token=xyz" }); + expect(s).not.toContain("abc"); + expect(s).not.toContain("xyz"); + }); + + it("strips https://user:password@host userinfo", () => { + const s = sanitizeArgsForAudit({ url: "https://user:hunter2@example.com/path" }); + expect(s).not.toContain("hunter2"); + expect(s).not.toContain("user:"); + }); + + it("leaves URLs without credentials untouched", () => { + const s = sanitizeArgsForAudit({ url: "https://example.com/path?q=hi" }); + const p = JSON.parse(s); + expect(p.url).toBe("https://example.com/path?q=hi"); + }); + + it("redacts URL secrets in arrays (e.g. webfetch redirect_chain)", () => { + const s = sanitizeArgsForAudit({ + chain: ["https://a.com/?api_key=A", "https://b.com/"] as never, + }); + expect(s).not.toContain("api_key=A"); + expect(s).toContain("https://b.com/"); + }); + + it("redacts URL secret BEFORE truncation so a long token can't survive at the tail", () => { + const longSecret = "k".repeat(500); + const s = sanitizeArgsForAudit({ url: `https://e.com/?api_key=${longSecret}` }); + expect(s).not.toContain(longSecret); + expect(s).not.toContain("kkkk"); // even a fragment of the token + }); + + it("variants: ?password=, ?secret=, ?signature=, ?sig=", () => { + const cases = [ + "https://x.com/?password=p", + "https://x.com/?secret=s", + "https://x.com/?signature=sig", + "https://x.com/?sig=short", + ]; + for (const url of cases) { + const s = sanitizeArgsForAudit({ url }); + expect(s).not.toMatch(/=p[^a-zA-Z]|=s[^a-zA-Z]|=sig[^a-zA-Z]|=short/); + } + }); + + it("doesn't crash on malformed URL strings (returns them untouched)", () => { + const s = sanitizeArgsForAudit({ url: "not a url at all" }); + const p = JSON.parse(s); + expect(p.url).toBe("not a url at all"); + }); +}); + +describe("audit: buildAuditRow keeps NO content / stdout / stderr / body", () => { + it("produces only the documented metadata keys", () => { + const result: ToolResult = { + ok: true, + content: "FULL FILE CONTENTS that must NEVER appear in audit", + audit: { + truncated: false, + originalBytes: 1234, + total_lines: 42, + returned_lines: 42, + }, + }; + const row = buildAuditRow("read", { path: "x.txt" }, result, 5); + // Whitelist of allowed keys + expect(Object.keys(row).sort()).toEqual( + ["argsSummary", "durationMs", "error", "exitCode", "httpStatus", "resultBytes", "toolName", "truncated"].sort(), + ); + // Sanity: no content leak + const serialized = JSON.stringify(row); + expect(serialized).not.toContain("FULL FILE CONTENTS"); + }); + + it("captures bash exit_code", () => { + const result: ToolResult = { + ok: false, + content: "[exit 2]", + audit: { + truncated: false, + truncated_stdout: false, + truncated_stderr: false, + original_stdout_bytes: 10, + original_stderr_bytes: 0, + exit_code: 2, + signal: null, + timeout: false, + duration_ms: 30, + command: "false", + args: [], + error: "nonzero_exit", + }, + }; + const row = buildAuditRow("bash", { command: "false", args: [] }, result, 30); + expect(row.exitCode).toBe(2); + expect(row.error).toBe("nonzero_exit"); + expect(row.resultBytes).toBe(10); // pulled from original_stdout_bytes + }); + + it("captures webfetch http_status", () => { + const result: ToolResult = { + ok: true, + content: "...", + audit: { + truncated: false, + original_bytes: 5000, + http_status: 200, + content_type: "text/html", + redirect_chain: ["https://example.com/"], + hops: 0, + }, + }; + const row = buildAuditRow("webfetch", { url: "https://example.com/" }, result, 50); + expect(row.httpStatus).toBe(200); + expect(row.resultBytes).toBe(5000); + expect(row.exitCode).toBeNull(); + }); + + it("redacts secrets in argsSummary", () => { + const result: ToolResult = { ok: true, content: "", audit: { truncated: false } }; + const row = buildAuditRow( + "webfetch", + { url: "https://api.example.com/x", authorization: "Bearer xyz" } as Record as never, + result, + 1, + ); + expect(row.argsSummary).toContain("[redacted]"); + expect(row.argsSummary).not.toContain("Bearer xyz"); + }); +}); diff --git a/packages/jimmy/src/engines/__tests__/fixtures/report-url/expected-classification.schema.json b/packages/jimmy/src/engines/__tests__/fixtures/report-url/expected-classification.schema.json new file mode 100644 index 00000000..4830da7a --- /dev/null +++ b/packages/jimmy/src/engines/__tests__/fixtures/report-url/expected-classification.schema.json @@ -0,0 +1,30 @@ +{ + "description": "Golden expected structure for the report-url triage output. Validates SHAPE, not exact prose — the model can phrase the reason however it likes as long as the fields and enum values stay stable. Used by report-url.fixture.test.ts to gate cron migrations from Claude → ollama/openai.", + "required": ["as", "reason", "confidence", "tenants"], + "fields": { + "as": { + "type": "string", + "enum": ["issue", "news", "both", "breaking"] + }, + "reason": { + "type": "string", + "minLength": 10, + "description": "Short human-readable rationale (one sentence)." + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tenants": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "category": { + "type": "string", + "optional": true + } + }, + "fixture_intent": "Pahang MB statement responding to flood-relief protests + opposition criticism + NGO call for tender review. Story is older than 1h, multi-stakeholder political content. Expected classification: as=issue (political response surface), tenants=[pahang], category=governance|crisis-comms|policy. NOT breaking (>1h old). NOT pure news (the political response framing makes it an issue." +} diff --git a/packages/jimmy/src/engines/__tests__/fixtures/report-url/input.txt b/packages/jimmy/src/engines/__tests__/fixtures/report-url/input.txt new file mode 100644 index 00000000..dbabf9b8 --- /dev/null +++ b/packages/jimmy/src/engines/__tests__/fixtures/report-url/input.txt @@ -0,0 +1,19 @@ +URL: https://pahang.warroom.cc/api/blog/example/2025-03-14/menteri-pahang-respons-bantahan-banjir +HTTP status: 200 +Content-Type: text/html + +PAGE TITLE: "Menteri Besar Pahang respons bantahan rakyat di kawasan banjir Temerloh" + +PAGE BODY: +Menteri Besar Pahang hari ini mengeluarkan kenyataan rasmi terhadap bantahan rakyat +di kawasan banjir Temerloh yang berleluasa sejak tiga hari lalu. Beliau berkata +kerajaan negeri sedang berusaha menambah kemudahan pusat pemindahan sementara +(PPS) dan menambah baik bekalan makanan, namun mengakui ada jurang dalam +tindak balas awal yang menyebabkan kemarahan komuniti. + +Pembangkang Pahang telah mengkritik tindak balas kerajaan negeri sebagai "lambat +dan tidak terurus", manakala kumpulan NGO tempatan menggesa pemeriksaan terhadap +kontraktor PPS yang dilantik tanpa tender terbuka. + +Ditulis pada: 2025-03-14 (lebih daripada 1 jam yang lalu) +Lokasi: Temerloh, Pahang diff --git a/packages/jimmy/src/engines/__tests__/report-url.fixture.test.ts b/packages/jimmy/src/engines/__tests__/report-url.fixture.test.ts new file mode 100644 index 00000000..a2feda49 --- /dev/null +++ b/packages/jimmy/src/engines/__tests__/report-url.fixture.test.ts @@ -0,0 +1,183 @@ +/** + * Golden classification fixture for the report-url triage cron. + * + * Why this test exists: + * When the cron migrates from Claude → ollama/openai we want a + * structural guarantee that downstream consumers (warroom_my.issue + * writer, sitrep dispatcher) keep working. The model's prose can drift + * between provider runs and is not under our control. The output + * SHAPE — field names, types, enum values — IS under our control via + * the system prompt and JSON-schema response shaping. + * + * What we assert: + * - The agent loop completes successfully (kind="ok") + * - finalContent parses as JSON + * - Required fields are present with the right types + * - `as` is one of the documented enum values + * - `confidence` is in [0,1] + * - `tenants` is a non-empty string array + * + * What we deliberately DO NOT assert: + * - Exact wording of `reason` + * - Specific tenant choice (model judgment) + * - The model's chain-of-thought + * + * This protects migration goals without making prose drift a blocker. + */ +import { describe, it, expect } from "vitest"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { runAgentLoop } from "../agentLoop.js"; +import type { ProviderCall } from "../providers/types.js"; + +const FIXTURE_DIR = path.dirname(fileURLToPath(import.meta.url)) + "/fixtures/report-url"; +const INPUT = fs.readFileSync(path.join(FIXTURE_DIR, "input.txt"), "utf8"); +const SCHEMA = JSON.parse(fs.readFileSync(path.join(FIXTURE_DIR, "expected-classification.schema.json"), "utf8")) as { + fields: Record; + required: string[]; +}; + +const TRIAGE_SYSTEM_PROMPT = `You are a news-URL triage classifier for the warroom monitoring system. +Given a URL + page content, decide how it should be filed. + +Respond with a JSON object only, no markdown fence, with these fields: + as: one of "issue" | "news" | "both" | "breaking" + reason: a short one-sentence rationale + confidence: number in [0, 1] + tenants: array of tenant slugs ("pahang" | "melaka" | "ns" | "selangor" | "editorial") + category: optional short category label + +Definitions: + - "issue": political content, governance criticism, narrative attacks — anything we'd draft a response for + - "news": neutral factual reporting — accidents, weather, court rulings on non-political cases + - "both": when the article reads as both + - "breaking": story age <1h AND at least one breaking criterion (deaths, VIP arrest, natural disaster, infra failure)`; + +// Stand-in provider that returns a single assistant turn with a JSON +// payload matching the documented schema. Stable across runs so the +// structural assertions below are deterministic. +const mockTriageProvider: ProviderCall = async () => ({ + message: { + role: "assistant", + content: JSON.stringify({ + as: "issue", + reason: "Political response to flood-relief criticism; multi-stakeholder governance content.", + confidence: 0.92, + tenants: ["pahang"], + category: "crisis-comms", + }), + }, + finishReason: "stop", + usage: { promptTokens: 480, completionTokens: 70 }, + billedModel: "gpt-4o-mini", +}); + +describe("report-url triage — golden fixture", () => { + it("agent loop produces a structurally valid classification", async () => { + const result = await runAgentLoop({ + provider: mockTriageProvider, + toolExecutors: new Map(), + toolSchemas: [], + model: "gpt-4o-mini", + systemPrompt: TRIAGE_SYSTEM_PROMPT, + userPrompt: INPUT, + maxTurns: 3, + timeoutMs: 10_000, + toolContext: { cwd: process.cwd() }, + }); + + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + + // Loop-level invariants + expect(result.turns).toBe(1); + expect(typeof result.durationMs).toBe("number"); + expect(result.promptTokens).toBeGreaterThan(0); + expect(result.completionTokens).toBeGreaterThan(0); + + // Parse the JSON envelope + const parsed = parseTriageOutput(result.finalContent); + expect(parsed).not.toBeNull(); + if (!parsed) return; + + // Required fields present + for (const key of SCHEMA.required) { + expect(parsed, `missing required field "${key}"`).toHaveProperty(key); + } + + // Field-by-field structural validation + const fields = SCHEMA.fields; + + expect(typeof parsed.as).toBe("string"); + expect(fields.as!.enum).toContain(parsed.as); + + expect(typeof parsed.reason).toBe("string"); + expect((parsed.reason as string).length).toBeGreaterThanOrEqual(fields.reason!.minLength!); + + expect(typeof parsed.confidence).toBe("number"); + expect(parsed.confidence as number).toBeGreaterThanOrEqual(fields.confidence!.minimum!); + expect(parsed.confidence as number).toBeLessThanOrEqual(fields.confidence!.maximum!); + + expect(Array.isArray(parsed.tenants)).toBe(true); + expect((parsed.tenants as unknown[]).length).toBeGreaterThanOrEqual(fields.tenants!.minItems!); + for (const t of parsed.tenants as unknown[]) { + expect(typeof t).toBe("string"); + } + + if (parsed.category !== undefined) { + expect(typeof parsed.category).toBe("string"); + } + }); + + it("rejects clearly malformed model output cleanly (structural guard)", async () => { + // What happens if a future model returns a stringified array instead + // of a JSON object? The loop completes, but downstream parsing fails + // and we detect it BEFORE writing to warroom_my.issue. + const badProvider: ProviderCall = async () => ({ + message: { + role: "assistant", + content: "[1, 2, 3]", // wrong shape entirely + }, + finishReason: "stop", + usage: { promptTokens: 100, completionTokens: 5 }, + billedModel: "gpt-4o-mini", + }); + + const result = await runAgentLoop({ + provider: badProvider, + toolExecutors: new Map(), + toolSchemas: [], + model: "gpt-4o-mini", + systemPrompt: TRIAGE_SYSTEM_PROMPT, + userPrompt: INPUT, + maxTurns: 1, + timeoutMs: 5_000, + toolContext: { cwd: process.cwd() }, + }); + + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") return; + const parsed = parseTriageOutput(result.finalContent); + // Array IS valid JSON but doesn't match our object schema. + expect(parsed).toBeNull(); + }); +}); + +/** + * Parse a model response into the triage object, tolerating common + * envelopes (raw JSON, ```json fenced blocks). Returns null on any + * structural failure so the test can assert "rejected cleanly". + */ +function parseTriageOutput(raw: string): Record | null { + // Strip markdown fences if present. + const fenceMatch = raw.match(/```(?:json)?\s*([\s\S]*?)```/i); + const body = (fenceMatch ? fenceMatch[1]! : raw).trim(); + try { + const j = JSON.parse(body); + if (j === null || typeof j !== "object" || Array.isArray(j)) return null; + return j as Record; + } catch { + return null; + } +} diff --git a/packages/jimmy/src/engines/__tests__/wrappers.test.ts b/packages/jimmy/src/engines/__tests__/wrappers.test.ts new file mode 100644 index 00000000..dade88f4 --- /dev/null +++ b/packages/jimmy/src/engines/__tests__/wrappers.test.ts @@ -0,0 +1,477 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { OllamaEngine } from "../ollama.js"; +import { OpenAIEngine } from "../openai.js"; +import type { ProviderCall, ProviderCallResult } from "../providers/types.js"; + +// We mock the two provider factories so the wrapper tests stay +// fully in-process — no HTTP, no fixture servers needed. The wrapper +// passes the provider call returned by these factories straight into +// the agent loop, so by controlling that we control the loop's +// response shape. +const mockOllamaProvider = vi.fn<(opts: object) => ProviderCall>(); +const mockOpenAIProvider = vi.fn<(opts: object) => ProviderCall>(); + +vi.mock("../providers/ollama.js", () => ({ + createOllamaProvider: (opts: object) => mockOllamaProvider(opts), +})); + +vi.mock("../providers/openai.js", () => ({ + createOpenAIProvider: (opts: object) => mockOpenAIProvider(opts), +})); + +// Defaults the provider factories return when a test doesn't override. +function setProviderResult(result: ProviderCallResult | Error): ProviderCall { + const fn: ProviderCall = async () => { + if (result instanceof Error) throw result; + return result; + }; + return fn; +} + +function okResult(content = "all done"): ProviderCallResult { + return { + message: { role: "assistant", content }, + finishReason: "stop", + usage: { promptTokens: 100, completionTokens: 30 }, + billedModel: "gpt-4o-mini", + }; +} + +beforeEach(() => { + mockOllamaProvider.mockReset(); + mockOpenAIProvider.mockReset(); + // Default to a successful provider call. + mockOllamaProvider.mockImplementation(() => setProviderResult(okResult())); + mockOpenAIProvider.mockImplementation(() => setProviderResult(okResult())); +}); + +// ─── Ollama: construction-time config validation ───────────────────── + +describe("OllamaEngine: construction-time validation", () => { + it("throws when config.url is missing", () => { + expect(() => new OllamaEngine({ url: "" as never })).toThrow(/url is required/); + }); + + it("succeeds with just a url", () => { + const engine = new OllamaEngine({ url: "https://ollama.example.com" }); + expect(engine.name).toBe("ollama"); + }); + + it("warns but constructs when tools.enabled has unknown names", () => { + // Just verify no throw — the warning goes through logger.warn. + expect( + () => + new OllamaEngine({ + url: "https://o.example.com", + tools: { enabled: ["read", "fictional_tool"] }, + }), + ).not.toThrow(); + }); + + it("reads OLLAMA_TOKEN env var by default", () => { + process.env.OLLAMA_TOKEN = "test-token-xyz"; + try { + new OllamaEngine({ url: "https://o.example.com" }); + expect(mockOllamaProvider).toHaveBeenCalledWith({ + baseUrl: "https://o.example.com", + token: "test-token-xyz", + }); + } finally { + delete process.env.OLLAMA_TOKEN; + } + }); + + it("respects custom authTokenEnvVar", () => { + process.env.CUSTOM_OLLAMA_TOKEN = "abc"; + try { + new OllamaEngine({ + url: "https://o.example.com", + authTokenEnvVar: "CUSTOM_OLLAMA_TOKEN", + }); + expect(mockOllamaProvider).toHaveBeenCalledWith({ + baseUrl: "https://o.example.com", + token: "abc", + }); + } finally { + delete process.env.CUSTOM_OLLAMA_TOKEN; + } + }); + + it("rejects authTokenEnvVar that looks like a token VALUE, not a NAME", () => { + // Common operator mistake: pasting the secret into the *EnvVar field. + // The guard catches this before we silently lookup process.env[secret] + // (which would always return undefined → engine constructs unauthed). + expect(() => new OllamaEngine({ + url: "https://o.example.com", + authTokenEnvVar: "sk-abc-XYZ-123", // hyphens + lowercase → not a valid env var name + })).toThrow(/authTokenEnvVar must be an env var NAME/); + }); + + it("rejects lowercase authTokenEnvVar", () => { + expect(() => new OllamaEngine({ + url: "https://o.example.com", + authTokenEnvVar: "ollama_token", + })).toThrow(/authTokenEnvVar must be an env var NAME/); + }); + + it("rejects empty authTokenEnvVar string", () => { + expect(() => new OllamaEngine({ + url: "https://o.example.com", + authTokenEnvVar: "", + })).toThrow(/authTokenEnvVar must be a non-empty env var NAME/); + }); + + it("error message does NOT echo the secret value (length only)", () => { + try { + new OllamaEngine({ + url: "https://o.example.com", + authTokenEnvVar: "sk-LEAK-DO-NOT-LOG-THIS", + }); + throw new Error("did not throw"); + } catch (err) { + const msg = (err as Error).message; + expect(msg).not.toContain("sk-LEAK-DO-NOT-LOG-THIS"); + expect(msg).toContain("length"); + } + }); +}); + +// ─── OpenAI: construction-time config validation ───────────────────── + +describe("OpenAIEngine: construction-time validation", () => { + it("throws when OPENAI_API_KEY env var is unset", () => { + const saved = process.env.OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + try { + expect(() => new OpenAIEngine({})).toThrow(/missing API key/); + } finally { + if (saved !== undefined) process.env.OPENAI_API_KEY = saved; + } + }); + + it("throws when OPENAI_API_KEY is empty", () => { + const saved = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = ""; + try { + expect(() => new OpenAIEngine({})).toThrow(/missing API key/); + } finally { + if (saved !== undefined) process.env.OPENAI_API_KEY = saved; + else delete process.env.OPENAI_API_KEY; + } + }); + + it("succeeds when OPENAI_API_KEY is set", () => { + process.env.OPENAI_API_KEY = "sk-test"; + try { + const engine = new OpenAIEngine({}); + expect(engine.name).toBe("openai"); + } finally { + delete process.env.OPENAI_API_KEY; + } + }); + + it("respects custom apiKeyEnvVar", () => { + process.env.CUSTOM_OPENAI_KEY = "sk-custom"; + try { + new OpenAIEngine({ apiKeyEnvVar: "CUSTOM_OPENAI_KEY" }); + expect(mockOpenAIProvider).toHaveBeenCalledWith({ + apiKey: "sk-custom", + baseUrl: undefined, + }); + } finally { + delete process.env.CUSTOM_OPENAI_KEY; + } + }); + + it("rejects apiKeyEnvVar that looks like a raw API key (sk-…)", () => { + expect(() => + new OpenAIEngine({ apiKeyEnvVar: "sk-proj-abc123" }), + ).toThrow(/apiKeyEnvVar must be an env var NAME/); + }); + + it("rejects apiKeyEnvVar that contains lowercase", () => { + expect(() => + new OpenAIEngine({ apiKeyEnvVar: "openai_api_key" }), + ).toThrow(/apiKeyEnvVar must be an env var NAME/); + }); + + it("rejects apiKeyEnvVar with leading digit", () => { + expect(() => new OpenAIEngine({ apiKeyEnvVar: "1KEY" })).toThrow( + /apiKeyEnvVar must be an env var NAME/, + ); + }); + + it("error message does NOT echo the secret value (length only)", () => { + try { + new OpenAIEngine({ apiKeyEnvVar: "sk-LEAK-DO-NOT-LOG-OAI" }); + throw new Error("did not throw"); + } catch (err) { + const msg = (err as Error).message; + expect(msg).not.toContain("sk-LEAK-DO-NOT-LOG-OAI"); + expect(msg).toContain("length"); + } + }); +}); + +// ─── Unsupported features rejected BEFORE any provider call ───────── + +describe("OllamaEngine: rejects unsupported features without provider call", () => { + let engine: OllamaEngine; + let providerCalls = 0; + + beforeEach(() => { + providerCalls = 0; + mockOllamaProvider.mockImplementation(() => async () => { + providerCalls++; + return okResult(); + }); + engine = new OllamaEngine({ url: "https://o.example.com", model: "qwen2.5:7b" }); + }); + + it("rejects resumeSessionId", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + resumeSessionId: "prev-sess", + }); + expect(r.error).toMatch(/resumeSessionId is not supported/); + expect(providerCalls).toBe(0); + }); + + it("rejects mcpConfigPath", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + mcpConfigPath: "/path/to/mcp.json", + }); + expect(r.error).toMatch(/MCP servers are not supported/); + expect(providerCalls).toBe(0); + }); + + it("rejects attachments", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + attachments: ["/path/to/file.txt"], + }); + expect(r.error).toMatch(/attachments are not supported/); + expect(providerCalls).toBe(0); + }); + + it("rejects cliFlags", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + cliFlags: ["--no-stream"], + }); + expect(r.error).toMatch(/cliFlags are not supported/); + expect(providerCalls).toBe(0); + }); + + it("ignores empty attachments / cliFlags arrays", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + attachments: [], + cliFlags: [], + }); + expect(r.error).toBeUndefined(); + expect(providerCalls).toBe(1); + }); +}); + +describe("OpenAIEngine: rejects unsupported features without provider call", () => { + let engine: OpenAIEngine; + let providerCalls = 0; + + beforeEach(() => { + process.env.OPENAI_API_KEY = "sk-test"; + providerCalls = 0; + mockOpenAIProvider.mockImplementation(() => async () => { + providerCalls++; + return okResult(); + }); + engine = new OpenAIEngine({ model: "gpt-4o-mini" }); + }); + + afterEach(() => { + delete process.env.OPENAI_API_KEY; + }); + + it("rejects resumeSessionId without making a provider call", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + resumeSessionId: "prev", + }); + expect(r.error).toMatch(/resumeSessionId is not supported/); + expect(providerCalls).toBe(0); + }); + + it("rejects mcpConfigPath", async () => { + const r = await engine.run({ + prompt: "hi", + cwd: "/tmp", + mcpConfigPath: "/a", + }); + expect(r.error).toMatch(/MCP servers are not supported/); + expect(providerCalls).toBe(0); + }); +}); + +// ─── Model resolution ──────────────────────────────────────────────── + +describe("OllamaEngine: model resolution", () => { + it("uses opts.model when provided", async () => { + let seenModel = ""; + mockOllamaProvider.mockImplementation(() => async (callOpts) => { + seenModel = callOpts.model; + return okResult(); + }); + const engine = new OllamaEngine({ url: "https://o", model: "config-model" }); + await engine.run({ prompt: "hi", cwd: "/tmp", model: "opts-model" }); + expect(seenModel).toBe("opts-model"); + }); + + it("falls back to config.model when opts.model is missing", async () => { + let seenModel = ""; + mockOllamaProvider.mockImplementation(() => async (callOpts) => { + seenModel = callOpts.model; + return okResult(); + }); + const engine = new OllamaEngine({ url: "https://o", model: "default-from-config" }); + await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(seenModel).toBe("default-from-config"); + }); + + it("returns error when neither opts.model nor config.model exists", async () => { + const engine = new OllamaEngine({ url: "https://o" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.error).toMatch(/no model resolved/); + expect(r.result).toBe(""); + }); +}); + +describe("OpenAIEngine: model resolution", () => { + beforeEach(() => { + process.env.OPENAI_API_KEY = "sk-test"; + }); + afterEach(() => { + delete process.env.OPENAI_API_KEY; + }); + + it("returns error when neither opts.model nor config.model exists", async () => { + const engine = new OpenAIEngine({}); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.error).toMatch(/no model resolved/); + }); +}); + +// ─── Session ID handling ───────────────────────────────────────────── + +describe("session id handling", () => { + it("preserves opts.sessionId when provided (Ollama)", async () => { + const engine = new OllamaEngine({ url: "https://o", model: "x" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp", sessionId: "my-sess-id" }); + expect(r.sessionId).toBe("my-sess-id"); + }); + + it("generates a sessionId when omitted (Ollama)", async () => { + const engine = new OllamaEngine({ url: "https://o", model: "x" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.sessionId).toMatch(/^[0-9a-f-]{36}$/); + }); + + it("preserves opts.sessionId when provided (OpenAI)", async () => { + process.env.OPENAI_API_KEY = "sk-test"; + try { + const engine = new OpenAIEngine({ model: "gpt-4o-mini" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp", sessionId: "openai-sess" }); + expect(r.sessionId).toBe("openai-sess"); + } finally { + delete process.env.OPENAI_API_KEY; + } + }); +}); + +// ─── Cost computation ──────────────────────────────────────────────── + +describe("cost computation", () => { + it("Ollama returns cost=0 regardless of model", async () => { + mockOllamaProvider.mockImplementation(() => async () => ({ + message: { role: "assistant", content: "done" }, + finishReason: "stop", + usage: { promptTokens: 9_999, completionTokens: 9_999 }, + billedModel: "qwen2.5:7b-instruct", + })); + const engine = new OllamaEngine({ url: "https://o", model: "qwen2.5:7b" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.cost).toBe(0); + }); + + it("OpenAI computes cost from billedModel and accumulated usage", async () => { + mockOpenAIProvider.mockImplementation(() => async () => ({ + message: { role: "assistant", content: "done" }, + finishReason: "stop", + usage: { promptTokens: 1_000_000, completionTokens: 500_000 }, + billedModel: "gpt-4o-mini", + })); + process.env.OPENAI_API_KEY = "sk-test"; + try { + const engine = new OpenAIEngine({ model: "gpt-4o-mini" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + // gpt-4o-mini: $0.15/M input, $0.60/M output + expect(r.cost).toBeCloseTo(0.15 + 0.30, 5); + } finally { + delete process.env.OPENAI_API_KEY; + } + }); + + it("OpenAI returns cost=undefined (NOT 0) for unknown billed model", async () => { + mockOpenAIProvider.mockImplementation(() => async () => ({ + message: { role: "assistant", content: "done" }, + finishReason: "stop", + usage: { promptTokens: 100, completionTokens: 50 }, + billedModel: "gpt-fictional-2030", + })); + process.env.OPENAI_API_KEY = "sk-test"; + try { + const engine = new OpenAIEngine({ model: "gpt-fictional-2030" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.cost).toBeUndefined(); + } finally { + delete process.env.OPENAI_API_KEY; + } + }); +}); + +// ─── EngineResult shape across loop kinds ──────────────────────────── + +describe("EngineResult shape mapping", () => { + it("happy path carries result, cost, durationMs, numTurns; no error", async () => { + mockOllamaProvider.mockImplementation(() => async () => ({ + message: { role: "assistant", content: "hello" }, + finishReason: "stop", + usage: { promptTokens: 10, completionTokens: 5 }, + billedModel: "qwen2.5:7b", + })); + const engine = new OllamaEngine({ url: "https://o", model: "qwen2.5:7b" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.result).toBe("hello"); + expect(r.cost).toBe(0); + expect(typeof r.durationMs).toBe("number"); + expect(r.numTurns).toBe(1); + expect(r.error).toBeUndefined(); + }); + + it("provider error path carries error, cost (from accumulated usage), turns", async () => { + mockOllamaProvider.mockImplementation(() => async () => { + throw new Error("ECONNREFUSED"); + }); + const engine = new OllamaEngine({ url: "https://o", model: "qwen2.5:7b" }); + const r = await engine.run({ prompt: "hi", cwd: "/tmp" }); + expect(r.error).toMatch(/provider_error.*ECONNREFUSED/); + expect(r.result).toBe(""); + expect(typeof r.durationMs).toBe("number"); + }); +}); diff --git a/packages/jimmy/src/engines/agentLoop.ts b/packages/jimmy/src/engines/agentLoop.ts new file mode 100644 index 00000000..52275a7f --- /dev/null +++ b/packages/jimmy/src/engines/agentLoop.ts @@ -0,0 +1,252 @@ +/** + * Provider-agnostic agent loop for the V1 HTTP engines (ollama, openai). + * + * Error taxonomy (distinct kinds, never collapsed): + * + * - Provider parse / transport errors (the adapter threw or returned a + * malformed payload) bubble up as `kind: "provider_error"` and abort + * the loop. The engine wrapper translates this to EngineResult.error. + * + * - Loop control errors — wall-clock timeout (`kind: "timeout"`) and + * max-turn exhaustion (`kind: "max_turns"`) — also abort the loop. + * + * - Tool execution errors stay model-visible: the tool returns + * `{ok: false, content, audit}`, we feed `content` back as a `tool` + * role message, the loop counts the turn, and the model decides what + * to do next. These never abort the loop. + * + * - Unknown tool calls (model invented a name that isn't registered) + * are treated the same as tool errors: synthetic + * `{ok: false, content: "Unknown tool: ..."}`, fed back as a tool + * message, turn counted, loop continues. + * + * - A tool executor that throws unexpectedly is also treated as a tool + * error (synthesized result), not a loop abort. The exception text + * surfaces in the tool message so the model can recover. + * + * Pre-call gates: + * - Before every provider call: check turns < maxTurns AND now < deadline. + * - Before every individual tool call (inside a multi-tool turn): same. + */ + +import type { ProviderCall, ProviderCallResult, ProviderMessage, ProviderToolDef } from "./providers/types.js"; +import type { ToolExecutionContext, ToolResult } from "./tools/types.js"; +import type { ToolExecutor } from "./tools/index.js"; +import { buildAuditRow, type AuditLogger, type AuditRow } from "./audit.js"; +import { logger } from "../shared/logger.js"; + +export interface AgentLoopOpts { + /** Provider adapter (openai or ollama). */ + provider: ProviderCall; + /** Tool name → executor (built per-engine via buildToolRegistry). */ + toolExecutors: Map; + /** Schemas exposed to the model — must match toolExecutors. */ + toolSchemas: ProviderToolDef[]; + /** Model to bill against. */ + model: string; + /** Optional system prompt. */ + systemPrompt?: string; + /** Initial user prompt. */ + userPrompt: string; + /** Max provider calls before aborting with kind="max_turns". */ + maxTurns: number; + /** Wall-clock budget for the whole loop, ms. */ + timeoutMs: number; + /** ToolExecutionContext shared by every tool call. */ + toolContext: ToolExecutionContext; + /** Optional audit sink. When present, every successful tool call is recorded. */ + audit?: AuditLogger; + /** Optional per-provider-call timeout override (ms). Default = remaining budget. */ + providerTimeoutMs?: number; +} + +interface AgentLoopUsage { + promptTokens: number; + completionTokens: number; + /** Distinct models actually billed (may include router-routed alts). */ + billedModels: string[]; +} + +interface AgentLoopBase extends AgentLoopUsage { + turns: number; + /** Wall-clock duration of the whole loop, in milliseconds. */ + durationMs: number; +} + +export interface AgentLoopOk extends AgentLoopBase { + kind: "ok"; + finalContent: string; + /** Each `{role: tool}` message accumulated during the loop. Mostly for tests. */ + toolMessages: ProviderMessage[]; +} + +export interface AgentLoopErr extends AgentLoopBase { + kind: "provider_error" | "max_turns" | "timeout"; + message: string; +} + +export type AgentLoopResult = AgentLoopOk | AgentLoopErr; + +export async function runAgentLoop(opts: AgentLoopOpts): Promise { + const messages: ProviderMessage[] = []; + if (opts.systemPrompt) messages.push({ role: "system", content: opts.systemPrompt }); + messages.push({ role: "user", content: opts.userPrompt }); + + const loopStart = Date.now(); + const deadline = loopStart + opts.timeoutMs; + let promptTokens = 0; + let completionTokens = 0; + const billedModels: string[] = []; + const toolMessages: ProviderMessage[] = []; + + for (let turn = 0; turn < opts.maxTurns; turn++) { + // Gate 1: wall-clock check BEFORE provider call. + if (Date.now() >= deadline) { + return { + kind: "timeout", + message: `loop deadline exceeded before provider call at turn ${turn}`, + turns: turn, + durationMs: Date.now() - loopStart, + promptTokens, + completionTokens, + billedModels, + }; + } + + const remaining = deadline - Date.now(); + const providerTimeout = Math.min(opts.providerTimeoutMs ?? remaining, remaining); + + let providerResult: ProviderCallResult; + try { + providerResult = await opts.provider({ + messages, + tools: opts.toolSchemas, + model: opts.model, + timeoutMs: providerTimeout, + }); + } catch (err) { + return { + kind: "provider_error", + message: (err as Error).message, + turns: turn, + durationMs: Date.now() - loopStart, + promptTokens, + completionTokens, + billedModels, + }; + } + + promptTokens += providerResult.usage.promptTokens; + completionTokens += providerResult.usage.completionTokens; + if (!billedModels.includes(providerResult.billedModel)) { + billedModels.push(providerResult.billedModel); + } + + // Append the assistant turn to history. + messages.push(providerResult.message); + + const toolCalls = providerResult.message.toolCalls ?? []; + if (toolCalls.length === 0) { + // Terminal: model decided not to call any tool. + return { + kind: "ok", + finalContent: providerResult.message.content, + turns: turn + 1, + durationMs: Date.now() - loopStart, + promptTokens, + completionTokens, + billedModels, + toolMessages, + }; + } + + // Execute each tool call sequentially. + for (const tc of toolCalls) { + // Gate 2: wall-clock check BEFORE every individual tool call. + if (Date.now() >= deadline) { + return { + kind: "timeout", + message: `loop deadline exceeded before tool call "${tc.name}" at turn ${turn}`, + turns: turn + 1, + durationMs: Date.now() - loopStart, + promptTokens, + completionTokens, + billedModels, + }; + } + + const executor = opts.toolExecutors.get(tc.name); + const callStart = Date.now(); + let result: ToolResult; + + if (!executor) { + // Unknown tool — synthesize a structured error result, do NOT throw. + const known = [...opts.toolExecutors.keys()]; + result = { + ok: false, + content: JSON.stringify({ + error: "unknown_tool", + requested: tc.name, + available: known, + }), + audit: { truncated: false, error: "unknown_tool" }, + }; + } else { + try { + result = await executor(tc.arguments, opts.toolContext); + } catch (err) { + // Executor threw unexpectedly. Surface as a structured tool + // error so the model can recover, and record the exception in + // audit. Do NOT abort the loop. + result = { + ok: false, + content: JSON.stringify({ + error: "tool_exception", + message: (err as Error).message, + }), + audit: { truncated: false, error: "tool_exception" }, + }; + } + } + + const toolDurationMs = Date.now() - callStart; + + if (opts.audit) { + const row = buildAuditRow(tc.name, tc.arguments, result, toolDurationMs); + await safeAudit(opts.audit, row, tc.name); + } + + const toolMessage: ProviderMessage = { + role: "tool", + content: result.content, + toolCallId: tc.id, + name: tc.name, + }; + messages.push(toolMessage); + toolMessages.push(toolMessage); + } + } + + // Loop exited without a terminal assistant message. + return { + kind: "max_turns", + message: `loop reached maxTurns=${opts.maxTurns} without a final assistant message`, + turns: opts.maxTurns, + durationMs: Date.now() - loopStart, + promptTokens, + completionTokens, + billedModels, + }; +} + +async function safeAudit(audit: AuditLogger, row: AuditRow, toolName: string): Promise { + try { + await audit.record(row); + } catch (err) { + // Audit failures must NOT break the loop, but they MUST be visible — + // log via the gateway logger so persistent sink issues are surfaced + // (e.g. sqlite-locked, disk full, schema drift). + const msg = (err as Error)?.message ?? String(err); + logger.warn(`agentLoop: audit sink failed for tool "${toolName}": ${msg}`); + } +} diff --git a/packages/jimmy/src/engines/audit.ts b/packages/jimmy/src/engines/audit.ts new file mode 100644 index 00000000..c58456e8 --- /dev/null +++ b/packages/jimmy/src/engines/audit.ts @@ -0,0 +1,178 @@ +/** + * Audit-log writer for tool calls inside the HTTP-loop engines. + * + * KEY INVARIANT: AuditRow NEVER carries full tool output (stdout, stderr, + * file body, HTTP response body). Audit is for forensic / cost-attribution + * use — the model already sees the body in its conversation. Logging it + * twice doubles storage and creates a leak surface for secrets that the + * model saw but we shouldn't persist on disk. + * + * Only metadata flows through: tool name, sanitized args, duration, + * exit code / http status, truncation flags, byte counts, error code. + * + * The actual sink (sqlite write, log file, telemetry pipe) is injected + * by the engine wrapper in Phase 7. Phase 6 only defines the abstract + * AuditLogger interface so the loop is testable. + */ + +import type { JsonObject, JsonValue } from "../shared/types.js"; +import type { ToolResult } from "./tools/types.js"; + +/** Header names + JSON keys that look like secrets and get redacted in audit. */ +const SECRET_KEY_PATTERNS = [ + /api[_-]?key/i, + /authorization/i, + /^auth$/i, + /token/i, + /secret/i, + /password/i, + /^bearer$/i, + /cookie/i, +]; + +const MAX_AUDIT_STRING_CHARS = 200; +const MAX_AUDIT_DEPTH = 5; + +/** + * Query-string parameter names that look like credentials. Matched + * case-insensitively against URL.searchParams keys; the VALUE is + * replaced with "[redacted]". (We don't strip the key itself — the + * presence of `?api_key=...` is itself useful debugging signal.) + */ +const SECRET_QUERY_PATTERNS = [ + /api[_-]?key/i, + /access[_-]?token/i, + /^token$/i, + /^auth$/i, + /authorization/i, + /secret/i, + /password/i, + /^key$/i, + /sig(nature)?/i, +]; + +function isSecretKey(key: string): boolean { + return SECRET_KEY_PATTERNS.some((re) => re.test(key)); +} + +function isSecretQueryParam(name: string): boolean { + return SECRET_QUERY_PATTERNS.some((re) => re.test(name)); +} + +/** + * Strip credentials from a URL string. Removes userinfo (https://user:pass@host) + * entirely and redacts query-string values for keys matching SECRET_QUERY_PATTERNS. + * Returns the original string unchanged if not a parseable URL. + */ +function redactUrl(s: string): string { + // Cheap guard: only attempt parsing for http(s)://-shaped strings. + if (!/^https?:\/\//i.test(s)) return s; + let u: URL; + try { + u = new URL(s); + } catch { + return s; + } + let touched = false; + if (u.username || u.password) { + u.username = ""; + u.password = ""; + touched = true; + } + for (const [k] of u.searchParams.entries()) { + if (isSecretQueryParam(k)) { + u.searchParams.set(k, "[redacted]"); + touched = true; + } + } + return touched ? u.toString() : s; +} + +function redact(value: JsonValue, depth: number): JsonValue { + if (depth > MAX_AUDIT_DEPTH) return "[depth-capped]"; + if (typeof value === "string") { + // Strip credentials from URL-shaped strings BEFORE truncation, so a + // long token sitting at the tail can't survive by being chopped off + // partway through. + const cleaned = redactUrl(value); + if (cleaned.length <= MAX_AUDIT_STRING_CHARS) return cleaned; + return cleaned.slice(0, MAX_AUDIT_STRING_CHARS) + `…[${cleaned.length - MAX_AUDIT_STRING_CHARS} more]`; + } + if (Array.isArray(value)) { + return value.map((v) => redact(v, depth + 1)); + } + if (value !== null && typeof value === "object") { + const out: Record = {}; + for (const [k, v] of Object.entries(value)) { + if (isSecretKey(k)) { + out[k] = "[redacted]"; + } else { + out[k] = redact(v as JsonValue, depth + 1); + } + } + return out; + } + return value; +} + +/** Serialize tool arguments for the audit row — keys redacted, strings capped. */ +export function sanitizeArgsForAudit(args: JsonObject): string { + return JSON.stringify(redact(args, 0)); +} + +export interface AuditRow { + /** Tool name as the model invoked it. */ + toolName: string; + /** JSON.stringify of sanitized args (secrets redacted, long strings capped). */ + argsSummary: string; + /** Wall-clock duration of the tool call in milliseconds. */ + durationMs: number; + /** Short error code, or null on success. */ + error: string | null; + /** Whether any output stream was truncated. */ + truncated: boolean; + /** Pre-truncation byte count where the tool reports it, else null. */ + resultBytes: number | null; + /** Process exit code for bash, else null. */ + exitCode: number | null; + /** HTTP status for webfetch, else null. */ + httpStatus: number | null; +} + +export interface AuditLogger { + record(row: AuditRow): void | Promise; +} + +/** Build an AuditRow from a tool call's args + result + measured duration. */ +export function buildAuditRow( + toolName: string, + args: JsonObject, + result: ToolResult, + durationMs: number, +): AuditRow { + const audit = result.audit; + const resultBytes = pickNumber(audit, [ + "originalBytes", + "original_bytes", + "file_bytes", + "original_stdout_bytes", + ]); + return { + toolName, + argsSummary: sanitizeArgsForAudit(args), + durationMs, + error: audit.error == null ? null : String(audit.error), + truncated: !!audit.truncated, + resultBytes, + exitCode: pickNumber(audit, ["exit_code"]), + httpStatus: pickNumber(audit, ["http_status"]), + }; +} + +function pickNumber(audit: ToolResult["audit"], keys: string[]): number | null { + for (const k of keys) { + const v = audit[k]; + if (typeof v === "number") return v; + } + return null; +} diff --git a/packages/jimmy/src/engines/ollama.ts b/packages/jimmy/src/engines/ollama.ts new file mode 100644 index 00000000..052dc165 --- /dev/null +++ b/packages/jimmy/src/engines/ollama.ts @@ -0,0 +1,199 @@ +/** + * Ollama engine wrapper — implements the Engine interface by composing + * the Ollama provider adapter (Phase 2), the tool registry (Phase 6), + * and the agent loop (Phase 6). + * + * V1 posture: non-streaming, no resume, no MCP, no attachments, no + * cliFlags. Each of those produces a clean EngineResult.error BEFORE + * any provider HTTP call. + * + * Construction-time validation: + * - config.url is required (throws if missing) + * - if config.authTokenEnvVar is set, the env var is consulted + * (no error if absent — Ollama may run unauthenticated) + * + * Per-call validation: + * - opts.model || config.model || error + * - reject unsupported features before touching the network + */ + +import { randomUUID } from "node:crypto"; +import type { Engine, EngineRunOpts, EngineResult, OllamaConfig } from "../shared/types.js"; +import type { JsonObject, JsonValue } from "../shared/types.js"; +import { logger } from "../shared/logger.js"; +import { createOllamaProvider } from "./providers/ollama.js"; +import { ollamaCostFor } from "./providers/pricing.js"; +import { buildToolRegistry, type ToolRegistry } from "./tools/index.js"; +import { runAgentLoop, type AgentLoopResult } from "./agentLoop.js"; +import type { AuditLogger } from "./audit.js"; +import type { ProviderCall } from "./providers/types.js"; + +const DEFAULT_MAX_TURNS = 25; +const DEFAULT_LOOP_TIMEOUT_MS = 300_000; +const DEFAULT_PROVIDER_TIMEOUT_MS = 60_000; + +export class OllamaEngine implements Engine { + name = "ollama" as const; + + private readonly provider: ProviderCall; + private readonly toolRegistry: ToolRegistry; + private readonly defaultModel: string | undefined; + private readonly maxTurns: number; + private readonly loopTimeoutMs: number; + private readonly providerTimeoutMs: number; + private readonly audit: AuditLogger | undefined; + private readonly toolOpts: Record; + + constructor(config: OllamaConfig, opts: { audit?: AuditLogger } = {}) { + if (!config.url) { + throw new Error("ollama: config.url is required (engines.ollama.url in config.yaml)"); + } + + const tokenEnvVar = config.authTokenEnvVar ?? "OLLAMA_TOKEN"; + assertEnvVarName("ollama.authTokenEnvVar", tokenEnvVar); + const token = process.env[tokenEnvVar]; + + this.provider = createOllamaProvider({ baseUrl: config.url, token }); + this.toolRegistry = buildToolRegistry(config.tools); + if (this.toolRegistry.unknownRequested.length > 0) { + logger.warn( + `ollama: engines.ollama.tools.enabled lists unknown names: ${this.toolRegistry.unknownRequested.join(", ")}`, + ); + } + + this.defaultModel = config.model; + this.maxTurns = config.maxTurns ?? DEFAULT_MAX_TURNS; + this.loopTimeoutMs = config.timeoutMs ?? DEFAULT_LOOP_TIMEOUT_MS; + this.providerTimeoutMs = config.providerTimeoutMs ?? DEFAULT_PROVIDER_TIMEOUT_MS; + this.audit = opts.audit; + this.toolOpts = buildToolOpts(config); + } + + async run(runOpts: EngineRunOpts): Promise { + const sessionId = runOpts.sessionId ?? randomUUID(); + + const unsupported = rejectUnsupported(this.name, runOpts); + if (unsupported) { + return { sessionId, result: "", error: unsupported }; + } + + const model = runOpts.model || this.defaultModel; + if (!model) { + return { + sessionId, + result: "", + error: `ollama: no model resolved (set engines.ollama.model in config or pass opts.model)`, + }; + } + + const loopResult = await runAgentLoop({ + provider: this.provider, + toolExecutors: this.toolRegistry.executors, + toolSchemas: this.toolRegistry.schemas, + model, + systemPrompt: runOpts.systemPrompt, + userPrompt: runOpts.prompt, + maxTurns: this.maxTurns, + timeoutMs: this.loopTimeoutMs, + providerTimeoutMs: this.providerTimeoutMs, + toolContext: { + cwd: runOpts.cwd, + sessionId, + engineName: this.name, + toolOpts: this.toolOpts, + }, + audit: this.audit, + }); + + return mapLoopResult(sessionId, loopResult, this.name); + } +} + +/** Ollama cost is always 0 (self-hosted). */ +function mapLoopResult(sessionId: string, r: AgentLoopResult, engineName: string): EngineResult { + const cost = ollamaCostFor(r.billedModels[0] ?? "", r.promptTokens, r.completionTokens); + if (r.kind === "ok") { + return { + sessionId, + result: r.finalContent, + cost, + durationMs: r.durationMs, + numTurns: r.turns, + }; + } + return { + sessionId, + result: "", + error: `${engineName}: ${r.kind}: ${r.message}`, + cost, + durationMs: r.durationMs, + numTurns: r.turns, + }; +} + +/** + * Translate the user-facing EngineToolsConfig shape into the + * ToolExecutionContext.toolOpts shape the tool executors read at runtime. + * Keeps config naming friendly (`bashAllowlist`) while preserving the + * runtime key the tool expects (`bash.allowlist`). + */ +function buildToolOpts(config: OllamaConfig): Record { + const t = config.tools; + if (!t) return {}; + const out: Record = {}; + if (t.bashAllowlist || t.bash) { + out.bash = { + allowlist: (t.bashAllowlist ?? []) as JsonValue, + ...(t.bash ?? {}), + } as JsonObject; + } + if (t.read) out.read = { ...t.read } as JsonObject; + if (t.webfetch) out.webfetch = { ...t.webfetch } as JsonObject; + return out; +} + +/** + * Validate per-call options BEFORE any provider call. Returns a + * descriptive error string if the request is incompatible with V1 + * semantics, or undefined if it can proceed. + */ +export function rejectUnsupported(engineName: string, runOpts: EngineRunOpts): string | undefined { + if (runOpts.resumeSessionId) { + return `${engineName}: resumeSessionId is not supported in V1; multi-turn resume is reserved for claude/codex/gemini`; + } + if (runOpts.mcpConfigPath) { + return `${engineName}: MCP servers are not supported in V1`; + } + if (runOpts.attachments && runOpts.attachments.length > 0) { + return `${engineName}: attachments are not supported in V1`; + } + if (runOpts.cliFlags && runOpts.cliFlags.length > 0) { + return `${engineName}: cliFlags are not supported (engine is HTTP-based, has no CLI)`; + } + return undefined; +} + +/** + * Posix-shell env var names: leading [A-Z_], then [A-Z0-9_]. Refuses + * lowercase or anything containing "-", "/", ".", etc. — catches the + * common operator mistake of pasting the secret VALUE + * (e.g. "sk-abc123…") into apiKeyEnvVar / authTokenEnvVar instead of + * an env var NAME (e.g. "OPENAI_API_KEY"). The error message NEVER + * echoes the value — we don't know if it's a secret, so we mask it. + */ +const ENV_VAR_NAME_RE = /^[A-Z_][A-Z0-9_]*$/; + +export function assertEnvVarName(fieldPath: string, value: string): void { + if (typeof value !== "string" || value.length === 0) { + throw new Error( + `${fieldPath} must be a non-empty env var NAME (e.g. "OPENAI_API_KEY"). Did you paste the secret value by mistake?`, + ); + } + if (!ENV_VAR_NAME_RE.test(value)) { + throw new Error( + `${fieldPath} must be an env var NAME matching [A-Z_][A-Z0-9_]* (e.g. "OPENAI_API_KEY"), ` + + `not the secret value itself. Got a string of length ${value.length}. ` + + `Set ${fieldPath} to the env var name; put the actual secret in your shell / .env file.`, + ); + } +} diff --git a/packages/jimmy/src/engines/openai.ts b/packages/jimmy/src/engines/openai.ts new file mode 100644 index 00000000..ffc68994 --- /dev/null +++ b/packages/jimmy/src/engines/openai.ts @@ -0,0 +1,155 @@ +/** + * OpenAI engine wrapper — same shape as OllamaEngine but reads + * apiKey from env at construction and computes cost via the OpenAI + * pricing table (Phase 2). cost is `undefined` when the billed model + * is not in the pricing table — the cost_log row goes to NULL and the + * weekly rollup surfaces the gap. + * + * Construction-time validation: + * - API key required (read from process.env[config.apiKeyEnvVar ?? + * "OPENAI_API_KEY"]); throws if missing. + * + * Per-call validation mirrors OllamaEngine: rejectUnsupported() runs + * BEFORE any provider call. + */ + +import { randomUUID } from "node:crypto"; +import type { Engine, EngineRunOpts, EngineResult, OpenAIConfig } from "../shared/types.js"; +import type { JsonObject, JsonValue } from "../shared/types.js"; +import { logger } from "../shared/logger.js"; +import { createOpenAIProvider } from "./providers/openai.js"; +import { openaiCostFor } from "./providers/pricing.js"; +import { buildToolRegistry, type ToolRegistry } from "./tools/index.js"; +import { runAgentLoop, type AgentLoopResult } from "./agentLoop.js"; +import type { AuditLogger } from "./audit.js"; +import type { ProviderCall } from "./providers/types.js"; +import { rejectUnsupported, assertEnvVarName } from "./ollama.js"; + +const DEFAULT_MAX_TURNS = 25; +const DEFAULT_LOOP_TIMEOUT_MS = 300_000; +const DEFAULT_PROVIDER_TIMEOUT_MS = 60_000; + +export class OpenAIEngine implements Engine { + name = "openai" as const; + + private readonly provider: ProviderCall; + private readonly toolRegistry: ToolRegistry; + private readonly defaultModel: string | undefined; + private readonly maxTurns: number; + private readonly loopTimeoutMs: number; + private readonly providerTimeoutMs: number; + private readonly audit: AuditLogger | undefined; + private readonly toolOpts: Record; + + constructor(config: OpenAIConfig, opts: { audit?: AuditLogger } = {}) { + const apiKeyEnvVar = config.apiKeyEnvVar ?? "OPENAI_API_KEY"; + assertEnvVarName("openai.apiKeyEnvVar", apiKeyEnvVar); + const apiKey = process.env[apiKeyEnvVar]; + if (!apiKey || apiKey.length === 0) { + throw new Error( + `openai: missing API key — env var "${apiKeyEnvVar}" is unset or empty`, + ); + } + this.provider = createOpenAIProvider({ apiKey, baseUrl: config.baseUrl }); + this.toolRegistry = buildToolRegistry(config.tools); + if (this.toolRegistry.unknownRequested.length > 0) { + logger.warn( + `openai: engines.openai.tools.enabled lists unknown names: ${this.toolRegistry.unknownRequested.join(", ")}`, + ); + } + + this.defaultModel = config.model; + this.maxTurns = config.maxTurns ?? DEFAULT_MAX_TURNS; + this.loopTimeoutMs = config.timeoutMs ?? DEFAULT_LOOP_TIMEOUT_MS; + this.providerTimeoutMs = config.providerTimeoutMs ?? DEFAULT_PROVIDER_TIMEOUT_MS; + this.audit = opts.audit; + this.toolOpts = buildToolOpts(config); + } + + async run(runOpts: EngineRunOpts): Promise { + const sessionId = runOpts.sessionId ?? randomUUID(); + + const unsupported = rejectUnsupported(this.name, runOpts); + if (unsupported) { + return { sessionId, result: "", error: unsupported }; + } + + const model = runOpts.model || this.defaultModel; + if (!model) { + return { + sessionId, + result: "", + error: `openai: no model resolved (set engines.openai.model in config or pass opts.model)`, + }; + } + + const loopResult = await runAgentLoop({ + provider: this.provider, + toolExecutors: this.toolRegistry.executors, + toolSchemas: this.toolRegistry.schemas, + model, + systemPrompt: runOpts.systemPrompt, + userPrompt: runOpts.prompt, + maxTurns: this.maxTurns, + timeoutMs: this.loopTimeoutMs, + providerTimeoutMs: this.providerTimeoutMs, + toolContext: { + cwd: runOpts.cwd, + sessionId, + engineName: this.name, + toolOpts: this.toolOpts, + }, + audit: this.audit, + }); + + return mapLoopResult(sessionId, loopResult, this.name); + } +} + +/** + * Compute USD cost from accumulated usage. Uses billedModels[0] (the + * first model the provider actually billed against — may differ from + * the requested model if the provider routes silently). Returns + * undefined when the model is not in the pricing table. + */ +function mapLoopResult(sessionId: string, r: AgentLoopResult, engineName: string): EngineResult { + const billed = r.billedModels[0] ?? ""; + const cost = openaiCostFor(billed, r.promptTokens, r.completionTokens); + if (cost === undefined && billed) { + logger.warn( + `openai: unknown pricing for model "${billed}"; cost_log row will record NULL`, + ); + } + if (r.kind === "ok") { + return { + sessionId, + result: r.finalContent, + cost, + durationMs: r.durationMs, + numTurns: r.turns, + }; + } + return { + sessionId, + result: "", + error: `${engineName}: ${r.kind}: ${r.message}`, + cost, + durationMs: r.durationMs, + numTurns: r.turns, + }; +} + +function buildToolOpts(config: OpenAIConfig): Record { + const t = config.tools; + if (!t) return {}; + const out: Record = {}; + if (t.bashAllowlist || t.bash) { + out.bash = { + allowlist: (t.bashAllowlist ?? []) as JsonValue, + ...(t.bash ?? {}), + } as JsonObject; + } + if (t.read) out.read = { ...t.read } as JsonObject; + if (t.webfetch) out.webfetch = { ...t.webfetch } as JsonObject; + return out; +} diff --git a/packages/jimmy/src/engines/providers/__tests__/ollama.test.ts b/packages/jimmy/src/engines/providers/__tests__/ollama.test.ts new file mode 100644 index 00000000..0c32a2ee --- /dev/null +++ b/packages/jimmy/src/engines/providers/__tests__/ollama.test.ts @@ -0,0 +1,296 @@ +import { describe, it, expect, vi } from "vitest"; +import { createOllamaProvider } from "../ollama.js"; +import type { ProviderCallOpts } from "../types.js"; + +function mockJsonResponse(body: unknown, init: { status?: number; statusText?: string } = {}) { + const status = init.status ?? 200; + const fn: typeof fetch = async (_input, _init) => { + return new Response(JSON.stringify(body), { + status, + statusText: init.statusText ?? "", + headers: { "Content-Type": "application/json" }, + }); + }; + return vi.fn(fn); +} + +function baseOpts(overrides: Partial = {}): ProviderCallOpts { + return { + messages: [ + { role: "system", content: "you are helpful" }, + { role: "user", content: "hi" }, + ], + tools: [], + model: "qwen2.5:7b-instruct", + ...overrides, + }; +} + +describe("providers/ollama — construction", () => { + it("throws if baseUrl is empty at construction", () => { + expect(() => createOllamaProvider({ baseUrl: "" })).toThrow(/missing baseUrl/); + }); +}); + +describe("providers/ollama — happy path text only", () => { + it("returns assistant content and stop finish reason", async () => { + const fetchFn = mockJsonResponse({ + model: "qwen2.5:7b-instruct", + message: { role: "assistant", content: "hello from ollama" }, + done: true, + done_reason: "stop", + prompt_eval_count: 20, + eval_count: 5, + }); + const call = createOllamaProvider({ baseUrl: "https://ollama.example.com" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.message.content).toBe("hello from ollama"); + expect(r.message.toolCalls).toBeUndefined(); + expect(r.finishReason).toBe("stop"); + expect(r.usage).toEqual({ promptTokens: 20, completionTokens: 5 }); + expect(r.billedModel).toBe("qwen2.5:7b-instruct"); + }); + + it("model that ignores tools and returns plain text is treated as a normal text response", async () => { + const fetchFn = mockJsonResponse({ + model: "qwen2.5:7b-instruct", + // No tool_calls field at all — model ignored the tools schema. + message: { role: "assistant", content: "I would call search('x') but I'll just describe it instead." }, + done: true, + done_reason: "stop", + prompt_eval_count: 40, + eval_count: 12, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + const r = await call(baseOpts({ + fetchFn, + tools: [{ name: "search", description: "x", parameters: { type: "object" } }], + })); + expect(r.finishReason).toBe("stop"); + expect(r.message.toolCalls).toBeUndefined(); + expect(r.message.content).toContain("search"); + }); +}); + +describe("providers/ollama — tool_calls normalization", () => { + it("accepts arguments as an OBJECT on the wire (typical Ollama)", async () => { + const fetchFn = mockJsonResponse({ + model: "qwen2.5:7b-instruct", + message: { + role: "assistant", + content: "", + tool_calls: [ + { function: { name: "search", arguments: { q: "hello", limit: 3 } } }, + ], + }, + done: true, + prompt_eval_count: 8, + eval_count: 4, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.finishReason).toBe("tool_calls"); + expect(r.message.toolCalls).toHaveLength(1); + expect(r.message.toolCalls![0].name).toBe("search"); + expect(r.message.toolCalls![0].arguments).toEqual({ q: "hello", limit: 3 }); + }); + + it("accepts arguments as a JSON STRING (some clients/models)", async () => { + const fetchFn = mockJsonResponse({ + model: "qwen2.5:7b-instruct", + message: { + role: "assistant", + content: "", + tool_calls: [ + { function: { name: "search", arguments: '{"q":"hello"}' } }, + ], + }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.message.toolCalls![0].arguments).toEqual({ q: "hello" }); + }); + + it("synthesizes stable call_ id when id is missing", async () => { + const fetchFn = mockJsonResponse({ + message: { + role: "assistant", + content: "", + tool_calls: [{ function: { name: "noop", arguments: {} } }], + }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.message.toolCalls![0].id).toMatch(/^call_[0-9a-f-]{36}$/); + }); + + it("treats empty / null / undefined arguments as {}", async () => { + for (const args of [null, undefined, "", {}]) { + const fetchFn = mockJsonResponse({ + message: { + role: "assistant", + content: "", + tool_calls: [{ function: { name: "noop", arguments: args } }], + }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.message.toolCalls![0].arguments).toEqual({}); + } + }); + + it("throws when arguments parses to an array (non-object)", async () => { + const fetchFn = mockJsonResponse({ + message: { + role: "assistant", + content: "", + tool_calls: [{ function: { name: "x", arguments: "[1,2,3]" } }], + }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/must parse to an object, got array/); + }); + + it("throws when arguments is an unparseable JSON string", async () => { + const fetchFn = mockJsonResponse({ + message: { + role: "assistant", + content: "", + tool_calls: [{ function: { name: "x", arguments: "{not json" } }], + }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/arguments JSON parse failed/); + }); + + it("throws when tool_call is missing function.name", async () => { + const fetchFn = mockJsonResponse({ + message: { + role: "assistant", + content: "", + tool_calls: [{ function: { arguments: {} } }], + }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/missing function.name/); + }); +}); + +describe("providers/ollama — auth", () => { + it("sends Authorization: Bearer header when token is provided", async () => { + const fetchFn = mockJsonResponse({ + message: { role: "assistant", content: "ok" }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o", token: "secret-token-xyz" }); + await call(baseOpts({ fetchFn })); + const init = fetchFn.mock.calls[0]![1] as RequestInit; + const headers = init.headers as Record; + expect(headers["Authorization"]).toBe("Bearer secret-token-xyz"); + }); + + it("omits Authorization header when no token is provided", async () => { + const fetchFn = mockJsonResponse({ + message: { role: "assistant", content: "ok" }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await call(baseOpts({ fetchFn })); + const init = fetchFn.mock.calls[0]![1] as RequestInit; + const headers = init.headers as Record; + expect(headers["Authorization"]).toBeUndefined(); + }); +}); + +describe("providers/ollama — HTTP errors", () => { + it("throws with status on non-2xx", async () => { + const fetchFn = vi.fn(async () => + new Response("model not found", { status: 404, statusText: "Not Found" }) + ); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/HTTP 404.*model not found/); + expect(fetchFn).toHaveBeenCalledTimes(1); + }); + + it("throws with transport error message when fetch rejects", async () => { + const fetchFn = vi.fn(async () => { + throw new Error("DNS resolution failed"); + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/transport error.*DNS resolution failed/); + }); +}); + +describe("providers/ollama — request shape", () => { + it("hits /api/chat with model + messages + stream:false", async () => { + const fetchFn = mockJsonResponse({ + message: { role: "assistant", content: "ok" }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://ollama.example.com/" }); // trailing slash + await call(baseOpts({ fetchFn })); + const [url, init] = fetchFn.mock.calls[0]!; + expect(String(url)).toBe("https://ollama.example.com/api/chat"); + const body = JSON.parse((init as RequestInit).body as string); + expect(body.model).toBe("qwen2.5:7b-instruct"); + expect(body.stream).toBe(false); + expect(Array.isArray(body.messages)).toBe(true); + }); + + it("round-trips assistant toolCalls with arguments as objects (Ollama wire format)", async () => { + const fetchFn = mockJsonResponse({ + message: { role: "assistant", content: "done" }, + done: true, + prompt_eval_count: 1, + eval_count: 1, + }); + const call = createOllamaProvider({ baseUrl: "https://o" }); + await call({ + ...baseOpts({ fetchFn }), + messages: [ + { role: "user", content: "do thing" }, + { + role: "assistant", + content: "", + toolCalls: [{ id: "c1", name: "search", arguments: { q: "x" } }], + }, + { role: "tool", content: '{"results":[]}', toolCallId: "c1", name: "search" }, + ], + }); + const body = JSON.parse(fetchFn.mock.calls[0]![1]!.body as string); + const asstMsg = body.messages[1]; + // Ollama: arguments object on the wire, not stringified + expect(asstMsg.tool_calls[0].function.arguments).toEqual({ q: "x" }); + const toolMsg = body.messages[2]; + expect(toolMsg).toEqual({ + role: "tool", + content: '{"results":[]}', + tool_call_id: "c1", + name: "search", + }); + }); +}); diff --git a/packages/jimmy/src/engines/providers/__tests__/openai.test.ts b/packages/jimmy/src/engines/providers/__tests__/openai.test.ts new file mode 100644 index 00000000..451010ef --- /dev/null +++ b/packages/jimmy/src/engines/providers/__tests__/openai.test.ts @@ -0,0 +1,311 @@ +import { describe, it, expect, vi } from "vitest"; +import { createOpenAIProvider } from "../openai.js"; +import type { ProviderCallOpts } from "../types.js"; + +/** Build a fetch-shaped mock from a single canned response object. */ +function mockJsonResponse(body: unknown, init: { status?: number; statusText?: string } = {}) { + const status = init.status ?? 200; + const fn: typeof fetch = async (_input, _init) => { + return new Response(JSON.stringify(body), { + status, + statusText: init.statusText ?? "", + headers: { "Content-Type": "application/json" }, + }); + }; + return vi.fn(fn); +} + +function baseOpts(overrides: Partial = {}): ProviderCallOpts { + return { + messages: [ + { role: "system", content: "you are helpful" }, + { role: "user", content: "hi" }, + ], + tools: [], + model: "gpt-4o-mini", + ...overrides, + }; +} + +describe("providers/openai — construction", () => { + it("throws if apiKey is empty at construction", () => { + expect(() => createOpenAIProvider({ apiKey: "" })).toThrow(/missing apiKey/); + }); +}); + +describe("providers/openai — happy path (text only)", () => { + it("returns assistant message with no toolCalls and finishReason=stop", async () => { + const fetchFn = mockJsonResponse({ + id: "resp_1", + model: "gpt-4o-mini-2024-07-18", + choices: [ + { + index: 0, + message: { role: "assistant", content: "hello back" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 12, completion_tokens: 4, total_tokens: 16 }, + }); + + const call = createOpenAIProvider({ apiKey: "sk-test" }); + const r = await call(baseOpts({ fetchFn })); + + expect(r.message.role).toBe("assistant"); + expect(r.message.content).toBe("hello back"); + expect(r.message.toolCalls).toBeUndefined(); + expect(r.finishReason).toBe("stop"); + expect(r.usage).toEqual({ promptTokens: 12, completionTokens: 4, totalTokens: 16 }); + expect(r.billedModel).toBe("gpt-4o-mini-2024-07-18"); + }); + + it("falls back to requested model when response.model is missing", async () => { + const fetchFn = mockJsonResponse({ + choices: [{ message: { role: "assistant", content: "x" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + const r = await call(baseOpts({ fetchFn, model: "gpt-4o-mini" })); + expect(r.billedModel).toBe("gpt-4o-mini"); + }); +}); + +describe("providers/openai — tool_calls normalization", () => { + it("parses JSON-string arguments into a JS object", async () => { + const fetchFn = mockJsonResponse({ + model: "gpt-4o-mini", + choices: [ + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { + id: "call_abc", + type: "function", + function: { + name: "search", + arguments: '{"q":"hello","limit":3}', + }, + }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { prompt_tokens: 8, completion_tokens: 12 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + const r = await call(baseOpts({ fetchFn })); + + expect(r.finishReason).toBe("tool_calls"); + expect(r.message.toolCalls).toHaveLength(1); + expect(r.message.toolCalls![0]).toEqual({ + id: "call_abc", + name: "search", + arguments: { q: "hello", limit: 3 }, + }); + }); + + it("synthesizes a stable id when the response omits one", async () => { + const fetchFn = mockJsonResponse({ + choices: [ + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { type: "function", function: { name: "noop", arguments: "{}" } }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.message.toolCalls![0].id).toMatch(/^call_[0-9a-f-]{36}$/); + }); + + it("returns finishReason=tool_calls when tool_calls present even if wire says 'stop'", async () => { + const fetchFn = mockJsonResponse({ + choices: [ + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { id: "c1", type: "function", function: { name: "x", arguments: "{}" } }, + ], + }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + const r = await call(baseOpts({ fetchFn })); + expect(r.finishReason).toBe("tool_calls"); + }); + + it("throws on malformed JSON in tool_calls arguments", async () => { + const fetchFn = mockJsonResponse({ + choices: [ + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { id: "c1", type: "function", function: { name: "x", arguments: "{not valid json" } }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/arguments JSON parse failed/); + }); + + it("throws when tool_call arguments parses to a non-object (array, primitive)", async () => { + const fetchFn = mockJsonResponse({ + choices: [ + { + message: { + role: "assistant", + content: "", + tool_calls: [ + { id: "c1", type: "function", function: { name: "x", arguments: "[1,2,3]" } }, + ], + }, + finish_reason: "tool_calls", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/must parse to an object/); + }); + + it("throws when tool_call is missing function.name", async () => { + const fetchFn = mockJsonResponse({ + choices: [ + { + message: { + role: "assistant", + content: "", + tool_calls: [{ id: "c1", type: "function", function: { arguments: "{}" } }], + }, + finish_reason: "tool_calls", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/missing function.name/); + }); +}); + +describe("providers/openai — HTTP errors and no-retry policy", () => { + it("throws with status + body excerpt on non-2xx response", async () => { + const fetchFn = vi.fn(async () => + new Response("rate limit exceeded", { status: 429, statusText: "Too Many Requests" }) + ); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/HTTP 429.*rate limit exceeded/); + expect(fetchFn).toHaveBeenCalledTimes(1); // V1: no retries + }); + + it("throws with transport error message when fetch rejects", async () => { + const fetchFn = vi.fn(async () => { + throw new Error("ECONNRESET"); + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/transport error.*ECONNRESET/); + expect(fetchFn).toHaveBeenCalledTimes(1); + }); + + it("throws when response body is not valid JSON", async () => { + const fetchFn = vi.fn(async () => + new Response("not json at all", { + status: 200, + headers: { "Content-Type": "application/json" }, + }) + ); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await expect(call(baseOpts({ fetchFn }))).rejects.toThrow(/response JSON parse failed/); + }); +}); + +describe("providers/openai — request shape", () => { + it("sends Authorization: Bearer header and JSON body with tools when provided", async () => { + const fetchFn = mockJsonResponse({ + choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test-123", baseUrl: "https://example.com/v1/" }); + await call(baseOpts({ + fetchFn, + tools: [ + { name: "search", description: "search the web", parameters: { type: "object", properties: { q: { type: "string" } } } }, + ], + })); + + const [url, init] = fetchFn.mock.calls[0]!; + expect(String(url)).toBe("https://example.com/v1/chat/completions"); // trailing slash on baseUrl is normalized + const headers = (init as RequestInit).headers as Record; + expect(headers["Authorization"]).toBe("Bearer sk-test-123"); + expect(headers["Content-Type"]).toBe("application/json"); + const body = JSON.parse((init as RequestInit).body as string); + expect(body.model).toBe("gpt-4o-mini"); + expect(body.stream).toBe(false); + expect(body.tools).toHaveLength(1); + expect(body.tools[0]).toEqual({ + type: "function", + function: { + name: "search", + description: "search the web", + parameters: { type: "object", properties: { q: { type: "string" } } }, + }, + }); + }); + + it("omits tools field when no tools are provided", async () => { + const fetchFn = mockJsonResponse({ + choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await call(baseOpts({ fetchFn })); + const init = fetchFn.mock.calls[0]![1] as RequestInit; + const body = JSON.parse(init.body as string); + expect(body.tools).toBeUndefined(); + }); + + it("serializes assistant→tool round-trip with arguments as JSON string", async () => { + const fetchFn = mockJsonResponse({ + choices: [{ message: { role: "assistant", content: "done" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + const call = createOpenAIProvider({ apiKey: "sk-test" }); + await call({ + ...baseOpts({ fetchFn }), + messages: [ + { role: "user", content: "do thing" }, + { + role: "assistant", + content: "", + toolCalls: [{ id: "c1", name: "search", arguments: { q: "x" } }], + }, + { role: "tool", content: "{\"results\":[]}", toolCallId: "c1" }, + ], + }); + const body = JSON.parse(fetchFn.mock.calls[0]![1]!.body as string); + const asstMsg = body.messages[1]; + expect(asstMsg.tool_calls[0].function.arguments).toBe('{"q":"x"}'); // serialized to string on the wire + const toolMsg = body.messages[2]; + expect(toolMsg).toEqual({ role: "tool", tool_call_id: "c1", content: '{"results":[]}' }); + }); +}); diff --git a/packages/jimmy/src/engines/providers/__tests__/pricing.test.ts b/packages/jimmy/src/engines/providers/__tests__/pricing.test.ts new file mode 100644 index 00000000..de6e4831 --- /dev/null +++ b/packages/jimmy/src/engines/providers/__tests__/pricing.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest"; +import { openaiRate, openaiCostFor, ollamaCostFor } from "../pricing.js"; + +describe("providers/pricing — openaiRate", () => { + it("returns rate for known model", () => { + const r = openaiRate("gpt-4o-mini"); + expect(r).toBeDefined(); + expect(r!.in).toBeGreaterThan(0); + expect(r!.out).toBeGreaterThan(r!.in); + }); + + it("returns undefined for unknown model — never falls back to 0", () => { + expect(openaiRate("gpt-fictional-2026")).toBeUndefined(); + }); + + it("returns undefined for empty model id", () => { + expect(openaiRate("")).toBeUndefined(); + }); +}); + +describe("providers/pricing — openaiCostFor", () => { + it("computes USD cost for a known model from token counts", () => { + // gpt-4o-mini: $0.15/M input, $0.60/M output + const cost = openaiCostFor("gpt-4o-mini", 1_000_000, 500_000); + expect(cost).toBeCloseTo(0.15 + 0.30, 5); + }); + + it("returns undefined for unknown model (not 0)", () => { + expect(openaiCostFor("gpt-fictional-2026", 1000, 1000)).toBeUndefined(); + }); + + it("returns 0 for known model with zero tokens", () => { + expect(openaiCostFor("gpt-4o-mini", 0, 0)).toBe(0); + }); +}); + +describe("providers/pricing — ollamaCostFor", () => { + it("always returns 0 regardless of model and tokens", () => { + expect(ollamaCostFor("qwen2.5:7b", 0, 0)).toBe(0); + expect(ollamaCostFor("qwen2.5:7b", 1_000_000, 500_000)).toBe(0); + expect(ollamaCostFor("any-fictional-model", 999_999_999, 999_999_999)).toBe(0); + }); +}); diff --git a/packages/jimmy/src/engines/providers/ollama.ts b/packages/jimmy/src/engines/providers/ollama.ts new file mode 100644 index 00000000..e2475be9 --- /dev/null +++ b/packages/jimmy/src/engines/providers/ollama.ts @@ -0,0 +1,245 @@ +/** + * Ollama Chat Completions provider adapter. + * + * Scope (V1): + * - Non-streaming. Sets `stream: false` on the request body. + * - Optional bearer token (set via OllamaAuth.token). + * - No auto-retries. + * - Tool call normalization differs from OpenAI in two ways: + * 1. Ollama's `message.tool_calls[i].function.arguments` may arrive + * as a JSON object (typical) OR as a JSON string (some models). + * We accept either and normalize to a JS object. + * 2. Ollama tool support is uneven across model families. We + * synthesize a stable `id` (call_) when missing so the + * tool-response round-trip still works. + * - If a model ignores the tools schema and replies with plain text that + * "looks like" a tool call (e.g. "I would call search(q='x')"), we do + * NOT try to parse it. That ambiguity belongs to a higher-level + * fallback layer, not the adapter. + * - Cost is always 0 (self-hosted; pricing.ts.ollamaCostFor handles this). + */ + +import { randomUUID } from "node:crypto"; +import type { + NormalizedToolCall, + OllamaAuth, + ProviderCall, + ProviderCallOpts, + ProviderCallResult, + ProviderFinishReason, + ProviderMessage, +} from "./types.js"; +import type { JsonObject, JsonValue } from "../../shared/types.js"; + +const DEFAULT_TIMEOUT_MS = 60_000; + +interface OllamaToolCallWire { + // Ollama may omit id entirely; we synthesize one when missing. + id?: string; + function?: { + name?: string; + // Object on the wire (per Ollama docs) but some clients/models send a string. + arguments?: JsonObject | string; + }; +} + +interface OllamaResponseWire { + model?: string; + message?: { + role?: string; + content?: string; + tool_calls?: OllamaToolCallWire[]; + }; + done?: boolean; + done_reason?: string; + // Token counts in Ollama's native shape. + prompt_eval_count?: number; + eval_count?: number; +} + +export function createOllamaProvider(auth: OllamaAuth): ProviderCall { + const baseUrl = auth.baseUrl.replace(/\/+$/, ""); + if (!baseUrl) { + throw new Error("ollama: missing baseUrl at provider construction"); + } + + return async function callOllama(opts: ProviderCallOpts): Promise { + const fetchFn = opts.fetchFn ?? fetch; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + const body = buildRequestBody(opts); + const headers: Record = { "Content-Type": "application/json" }; + if (auth.token) headers["Authorization"] = `Bearer ${auth.token}`; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error(`ollama: HTTP timeout after ${timeoutMs}ms`)), timeoutMs); + + let res: Response; + try { + res = await fetchFn(`${baseUrl}/api/chat`, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: controller.signal, + }); + } catch (err) { + throw new Error(`ollama: HTTP transport error: ${(err as Error).message}`); + } finally { + clearTimeout(timer); + } + + if (!res.ok) { + const errBody = await safeReadText(res); + throw new Error(`ollama: HTTP ${res.status} ${res.statusText}: ${truncate(errBody, 500)}`); + } + + let parsed: OllamaResponseWire; + try { + parsed = (await res.json()) as OllamaResponseWire; + } catch (err) { + throw new Error(`ollama: response JSON parse failed: ${(err as Error).message}`); + } + + return interpretResponse(parsed, opts.model); + }; +} + +function buildRequestBody(opts: ProviderCallOpts): JsonObject { + const body: JsonObject = { + model: opts.model, + messages: opts.messages.map(serializeMessage), + stream: false, + }; + if (opts.tools.length > 0) { + body.tools = opts.tools.map((t) => ({ + type: "function", + function: { + name: t.name, + description: t.description, + parameters: t.parameters, + }, + })); + } + return body; +} + +function serializeMessage(m: ProviderMessage): JsonObject { + if (m.role === "assistant") { + const out: JsonObject = { role: "assistant", content: m.content }; + if (m.toolCalls && m.toolCalls.length > 0) { + // Ollama's wire format accepts object arguments; pass through directly. + out.tool_calls = m.toolCalls.map((tc) => ({ + id: tc.id, + function: { + name: tc.name, + arguments: tc.arguments as JsonValue, + }, + })); + } + return out; + } + if (m.role === "tool") { + // Ollama accepts {role:"tool", content, tool_call_id, name} — name is + // helpful when present. + const out: JsonObject = { role: "tool", content: m.content }; + if (m.toolCallId) out.tool_call_id = m.toolCallId; + if (m.name) out.name = m.name; + return out; + } + return { role: m.role, content: m.content }; +} + +function interpretResponse(parsed: OllamaResponseWire, requestedModel: string): ProviderCallResult { + const wireMsg = parsed.message; + if (!wireMsg) { + throw new Error("ollama: response missing message"); + } + const role = wireMsg.role ?? "assistant"; + if (role !== "assistant") { + throw new Error(`ollama: expected assistant role in response, got "${role}"`); + } + + const toolCalls: NormalizedToolCall[] = (wireMsg.tool_calls ?? []).map(normalizeToolCall); + + const message: ProviderMessage = { + role: "assistant", + content: wireMsg.content ?? "", + }; + if (toolCalls.length > 0) message.toolCalls = toolCalls; + + const finishReason = normalizeFinishReason(parsed.done_reason, toolCalls.length > 0); + const billedModel = parsed.model && parsed.model.length > 0 ? parsed.model : requestedModel; + + return { + message, + finishReason, + usage: { + promptTokens: parsed.prompt_eval_count ?? 0, + completionTokens: parsed.eval_count ?? 0, + }, + billedModel, + }; +} + +function normalizeToolCall(tc: OllamaToolCallWire): NormalizedToolCall { + const name = tc.function?.name ?? ""; + if (!name) { + throw new Error("ollama: tool_call missing function.name"); + } + const id = tc.id && tc.id.length > 0 ? tc.id : `call_${randomUUID()}`; + const rawArgs = tc.function?.arguments; + const args = parseOllamaArgs(name, rawArgs); + return { id, name, arguments: args }; +} + +/** + * Accept either a JSON object or a JSON-encoded string and return a plain + * object. Empty / null / undefined collapse to `{}`. Throws if the value is + * non-object after parsing (arrays, primitives) — the agent loop should + * never have to guess the argument shape. + */ +function parseOllamaArgs(toolName: string, raw: JsonObject | string | undefined | null): JsonObject { + if (raw === undefined || raw === null) return {}; + if (typeof raw === "object" && !Array.isArray(raw)) { + return raw; + } + if (typeof raw === "string") { + if (raw === "") return {}; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error(`ollama: tool_call "${toolName}" arguments JSON parse failed: ${(err as Error).message}`); + } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`ollama: tool_call "${toolName}" arguments must parse to an object, got ${Array.isArray(parsed) ? "array" : typeof parsed}`); + } + return parsed as JsonObject; + } + throw new Error(`ollama: tool_call "${toolName}" arguments has unexpected type ${typeof raw}`); +} + +function normalizeFinishReason(raw: string | undefined, hadToolCalls: boolean): ProviderFinishReason { + if (hadToolCalls) return "tool_calls"; + switch (raw) { + case "stop": + case undefined: + return "stop"; + case "length": + return "length"; + default: + return "unknown"; + } +} + +async function safeReadText(res: Response): Promise { + try { + return await res.text(); + } catch { + return ""; + } +} + +function truncate(s: string, n: number): string { + return s.length <= n ? s : s.slice(0, n) + "…"; +} diff --git a/packages/jimmy/src/engines/providers/openai.ts b/packages/jimmy/src/engines/providers/openai.ts new file mode 100644 index 00000000..3419cc9f --- /dev/null +++ b/packages/jimmy/src/engines/providers/openai.ts @@ -0,0 +1,241 @@ +/** + * OpenAI Chat Completions provider adapter. + * + * Scope (V1): + * - Non-streaming. Streaming is out of scope until web UI demand exists. + * - No auto-retries. Transport / HTTP non-2xx / parse errors are thrown + * and surface as engine errors. + * - Tool-call shape parsed from `response.choices[0].message.tool_calls`. + * Arguments come as JSON-strings on the wire; we parse to JS object + * before handing to the agent loop (NormalizedToolCall.arguments is + * always object-typed). + * - Billed model from `response.model` (falls back to requested model + * only when the response omits it — providers can route to a different + * tier silently and cost lookup must follow the actual billing). + */ + +import { randomUUID } from "node:crypto"; +import type { + NormalizedToolCall, + OpenAIAuth, + ProviderCall, + ProviderCallOpts, + ProviderCallResult, + ProviderFinishReason, + ProviderMessage, +} from "./types.js"; +import type { JsonObject } from "../../shared/types.js"; + +const DEFAULT_BASE_URL = "https://api.openai.com/v1"; +const DEFAULT_TIMEOUT_MS = 60_000; + +interface OpenAIToolCallWire { + id?: string; + type?: string; + function?: { + name?: string; + arguments?: string; + }; +} + +interface OpenAIChoiceWire { + index?: number; + message?: { + role?: string; + content?: string | null; + tool_calls?: OpenAIToolCallWire[]; + }; + finish_reason?: string; +} + +interface OpenAIResponseWire { + id?: string; + model?: string; + choices?: OpenAIChoiceWire[]; + usage?: { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + }; +} + +/** + * Build a call function bound to the given auth. The returned function + * matches ProviderCall and can be passed to the agent loop directly. + */ +export function createOpenAIProvider(auth: OpenAIAuth): ProviderCall { + const baseUrl = (auth.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + if (!auth.apiKey) { + throw new Error("openai: missing apiKey at provider construction"); + } + + return async function callOpenAI(opts: ProviderCallOpts): Promise { + const fetchFn = opts.fetchFn ?? fetch; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + const body = buildRequestBody(opts); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error(`openai: HTTP timeout after ${timeoutMs}ms`)), timeoutMs); + + let res: Response; + try { + res = await fetchFn(`${baseUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${auth.apiKey}`, + }, + body: JSON.stringify(body), + signal: controller.signal, + }); + } catch (err) { + throw new Error(`openai: HTTP transport error: ${(err as Error).message}`); + } finally { + clearTimeout(timer); + } + + if (!res.ok) { + const errBody = await safeReadText(res); + throw new Error(`openai: HTTP ${res.status} ${res.statusText}: ${truncate(errBody, 500)}`); + } + + let parsed: OpenAIResponseWire; + try { + parsed = (await res.json()) as OpenAIResponseWire; + } catch (err) { + throw new Error(`openai: response JSON parse failed: ${(err as Error).message}`); + } + + return interpretResponse(parsed, opts.model); + }; +} + +function buildRequestBody(opts: ProviderCallOpts): JsonObject { + const body: JsonObject = { + model: opts.model, + messages: opts.messages.map(serializeMessage), + stream: false, + }; + if (opts.tools.length > 0) { + body.tools = opts.tools.map((t) => ({ + type: "function", + function: { + name: t.name, + description: t.description, + parameters: t.parameters, + }, + })); + } + return body; +} + +function serializeMessage(m: ProviderMessage): JsonObject { + if (m.role === "assistant") { + const out: JsonObject = { role: "assistant", content: m.content }; + if (m.toolCalls && m.toolCalls.length > 0) { + out.tool_calls = m.toolCalls.map((tc) => ({ + id: tc.id, + type: "function", + function: { + name: tc.name, + // OpenAI requires arguments as a string on the wire. + arguments: JSON.stringify(tc.arguments), + }, + })); + } + return out; + } + if (m.role === "tool") { + return { + role: "tool", + tool_call_id: m.toolCallId ?? "", + content: m.content, + }; + } + return { role: m.role, content: m.content }; +} + +function interpretResponse(parsed: OpenAIResponseWire, requestedModel: string): ProviderCallResult { + const choice = parsed.choices?.[0]; + if (!choice || !choice.message) { + throw new Error("openai: response missing choices[0].message"); + } + const wireRole = choice.message.role ?? "assistant"; + if (wireRole !== "assistant") { + throw new Error(`openai: expected assistant role in response, got "${wireRole}"`); + } + + const toolCalls: NormalizedToolCall[] = (choice.message.tool_calls ?? []).map(normalizeToolCall); + + const message: ProviderMessage = { + role: "assistant", + content: choice.message.content ?? "", + }; + if (toolCalls.length > 0) message.toolCalls = toolCalls; + + const finishReason = normalizeFinishReason(choice.finish_reason); + // If the wire layer reported tool_calls but finish_reason was something + // odd, trust the tool_calls signal — the agent loop needs to execute them. + const effectiveFinish: ProviderFinishReason = + toolCalls.length > 0 ? "tool_calls" : finishReason; + + const billedModel = parsed.model && parsed.model.length > 0 ? parsed.model : requestedModel; + + return { + message, + finishReason: effectiveFinish, + usage: { + promptTokens: parsed.usage?.prompt_tokens ?? 0, + completionTokens: parsed.usage?.completion_tokens ?? 0, + totalTokens: parsed.usage?.total_tokens, + }, + billedModel, + }; +} + +function normalizeToolCall(tc: OpenAIToolCallWire): NormalizedToolCall { + const id = tc.id && tc.id.length > 0 ? tc.id : `call_${randomUUID()}`; + const name = tc.function?.name ?? ""; + if (!name) { + throw new Error("openai: tool_call missing function.name"); + } + let args: JsonObject = {}; + const rawArgs = tc.function?.arguments; + if (rawArgs !== undefined && rawArgs !== null && rawArgs !== "") { + try { + const parsed = JSON.parse(rawArgs); + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`tool_call "${name}" arguments must parse to an object, got ${typeof parsed}`); + } + args = parsed as JsonObject; + } catch (err) { + throw new Error(`openai: tool_call "${name}" arguments JSON parse failed: ${(err as Error).message}`); + } + } + return { id, name, arguments: args }; +} + +function normalizeFinishReason(raw: string | undefined): ProviderFinishReason { + switch (raw) { + case "stop": + return "stop"; + case "tool_calls": + return "tool_calls"; + case "length": + return "length"; + default: + return "unknown"; + } +} + +async function safeReadText(res: Response): Promise { + try { + return await res.text(); + } catch { + return ""; + } +} + +function truncate(s: string, n: number): string { + return s.length <= n ? s : s.slice(0, n) + "…"; +} diff --git a/packages/jimmy/src/engines/providers/pricing.ts b/packages/jimmy/src/engines/providers/pricing.ts new file mode 100644 index 00000000..629f0280 --- /dev/null +++ b/packages/jimmy/src/engines/providers/pricing.ts @@ -0,0 +1,72 @@ +/** + * Per-model pricing for cost reporting from provider responses. + * + * Rates are USD per 1 million tokens, broken down by input vs output. + * OpenAI does not separate cache writes from base input in their billing + * surface (cached input is auto-discounted in `usage.prompt_tokens_details` + * if the SDK reports it); we treat `prompt_tokens` as one bucket for V1. + * + * Pricing snapshot: 2026-05. + */ + +export interface ModelRate { + /** USD per 1M prompt (input) tokens. */ + in: number; + /** USD per 1M completion (output) tokens. */ + out: number; +} + +/** OpenAI model price table. Keys match the model id used in the API. */ +const OPENAI_PRICING: Record = { + // GPT-4o family + "gpt-4o": { in: 2.50, out: 10.00 }, + "gpt-4o-2024-08-06": { in: 2.50, out: 10.00 }, + "gpt-4o-2024-11-20": { in: 2.50, out: 10.00 }, + "gpt-4o-mini": { in: 0.15, out: 0.60 }, + "gpt-4o-mini-2024-07-18": { in: 0.15, out: 0.60 }, + + // GPT-4.1 family + "gpt-4.1": { in: 2.00, out: 8.00 }, + "gpt-4.1-mini": { in: 0.40, out: 1.60 }, + "gpt-4.1-nano": { in: 0.10, out: 0.40 }, + + // o1 / o3 reasoning models + "o1": { in: 15.00, out: 60.00 }, + "o1-mini": { in: 1.10, out: 4.40 }, + "o3-mini": { in: 1.10, out: 4.40 }, + "o3": { in: 2.00, out: 8.00 }, + "o4-mini": { in: 1.10, out: 4.40 }, + + // GPT-5 family + "gpt-5": { in: 1.25, out: 10.00 }, + "gpt-5-mini": { in: 0.25, out: 2.00 }, + "gpt-5-nano": { in: 0.05, out: 0.40 }, +}; + +/** + * Look up the rate for an OpenAI model. Returns undefined if not in the + * table — callers must report cost as undefined rather than zero so the + * pricing gap is visible in cost_log + the weekly rollup. + */ +export function openaiRate(model: string): ModelRate | undefined { + return OPENAI_PRICING[model]; +} + +/** + * Compute USD cost for a single OpenAI completion. Returns undefined when + * no pricing entry exists for the model — the caller should pass through + * to cost_log as NULL and log a warning. Never returns 0 for missing data. + */ +export function openaiCostFor(model: string, promptTokens: number, completionTokens: number): number | undefined { + const rate = openaiRate(model); + if (!rate) return undefined; + return (promptTokens * rate.in) / 1e6 + (completionTokens * rate.out) / 1e6; +} + +/** + * Ollama runs locally / self-hosted, no per-token billing. Always returns 0 + * so cost_log rows from ollama sessions are recorded but show $0 spend. + */ +export function ollamaCostFor(_model: string, _promptTokens: number, _completionTokens: number): number { + return 0; +} diff --git a/packages/jimmy/src/engines/providers/types.ts b/packages/jimmy/src/engines/providers/types.ts new file mode 100644 index 00000000..eb0204eb --- /dev/null +++ b/packages/jimmy/src/engines/providers/types.ts @@ -0,0 +1,120 @@ +/** + * Shared types for HTTP-based provider adapters (openai, ollama). + * + * The adapter layer's job is to normalize the wire-format differences + * between providers so the agent loop never sees provider-specific shapes. + * `NormalizedToolCall` is the lingua franca: every adapter returns assistant + * messages in this exact shape, regardless of how the wire protocol + * delivered them. + */ + +import type { JsonObject } from "../../shared/types.js"; + +/** + * A tool invocation requested by the model. The adapter is responsible for: + * - Parsing whatever shape the wire protocol used (e.g. OpenAI's + * `tool_calls[].function.arguments` is a JSON-string; Ollama may return + * it as an object already). + * - Always producing `arguments` as a real JS object, never a string. + * - Synthesizing a stable `id` when the provider doesn't supply one + * (Ollama tool support varies). + */ +export interface NormalizedToolCall { + /** Stable id used to round-trip the `tool` role message back to the model. */ + id: string; + /** Tool name as the model requested it. May not match a registered tool. */ + name: string; + /** Parsed argument object. Empty object if the model supplied no arguments. */ + arguments: JsonObject; +} + +/** + * One message in a provider chat completion request/response. + * + * Mirrors the OpenAI chat-completion shape because both providers accept it + * (Ollama added compatibility with the OpenAI format in 0.1.30+). The + * adapter handles any wire-format differences internally. + */ +export type ProviderRole = "system" | "user" | "assistant" | "tool"; + +export interface ProviderMessage { + role: ProviderRole; + /** Plain text content. Empty string is valid (e.g. assistant returning only tool_calls). */ + content: string; + /** Set when role==="assistant" and the model requested tool execution. */ + toolCalls?: NormalizedToolCall[]; + /** Set when role==="tool". Echoes the tool_call.id this is a response to. */ + toolCallId?: string; + /** Optional tool name on a `tool` role message; some providers require it. */ + name?: string; +} + +/** + * Tool definition presented to the model. JSON-schema parameters, OpenAI + * function-calling format. Ollama accepts the same shape on its OpenAI- + * compatible endpoint and on `/api/chat` since 0.3.x. + */ +export interface ProviderToolDef { + name: string; + description: string; + parameters: JsonObject; +} + +export interface ProviderUsage { + promptTokens: number; + completionTokens: number; + totalTokens?: number; +} + +export type ProviderFinishReason = "stop" | "tool_calls" | "length" | "unknown"; + +export interface ProviderCallResult { + /** The assistant message produced by this turn. May carry toolCalls. */ + message: ProviderMessage; + finishReason: ProviderFinishReason; + usage: ProviderUsage; + /** + * The model that was actually billed (response.model from the provider). + * For cost lookup, prefer this over the requested model — providers can + * route to a different tier silently. Falls back to the requested model + * if the response omits it. + */ + billedModel: string; +} + +export interface ProviderCallOpts { + messages: ProviderMessage[]; + tools: ProviderToolDef[]; + model: string; + /** Per-call HTTP timeout in ms. Default 60_000. */ + timeoutMs?: number; + /** + * Optional fetch override. Used by tests to inject mocked HTTP responses. + * Production callers leave this undefined to use the global `fetch`. + */ + fetchFn?: typeof fetch; +} + +/** + * Adapter call function shape. Each provider module exports one of these. + * Throws on transport errors, HTTP non-2xx, or malformed payloads (so the + * agent loop can catch and surface a clear error). Never auto-retries — V1 + * policy is fail-fast. + */ +export type ProviderCall = (opts: ProviderCallOpts) => Promise; + +/** + * Auth options passed at module/engine construction time, not per-call. + */ +export interface OpenAIAuth { + apiKey: string; + /** Defaults to https://api.openai.com/v1 */ + baseUrl?: string; +} + +export interface OllamaAuth { + /** e.g. https://ollama.aga.my */ + baseUrl: string; + /** Optional bearer token. Sent as `Authorization: Bearer ` if set. */ + token?: string; +} diff --git a/packages/jimmy/src/engines/tools/__tests__/buildLookup.test.ts b/packages/jimmy/src/engines/tools/__tests__/buildLookup.test.ts new file mode 100644 index 00000000..d5796b96 --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/buildLookup.test.ts @@ -0,0 +1,93 @@ +/** + * Direct unit coverage for buildLookup — the function passed to + * http.request({lookup}). The integration test in webfetch.test.ts proves + * the chain end-to-end via fixture server requests, but this file exercises + * the validator path on its own so a regression in the address-check + * surfaces immediately, independent of HTTP server state. + * + * Note on the broader DNS-rebinding guarantee: + * Node's documented contract for http.request's `lookup` option is that the + * function returned here IS invoked at socket-connect time, and the address + * it passes back IS what the socket dials. We rely on that contract; + * exercising it under a stub is not feasible without monkey-patching + * internal node:http machinery. + */ +import { describe, it, expect } from "vitest"; +import { buildLookup } from "../webfetch.js"; + +function callLookup( + fn: ReturnType, + hostname: string, +): Promise<{ err: NodeJS.ErrnoException | null; address?: string; family?: number }> { + return new Promise((resolve) => { + // node:net's LookupFunction has an overloaded signature; we invoke the + // 3-arg form (hostname, options, callback) the http module uses. + (fn as unknown as ( + h: string, + opts: object, + cb: (err: NodeJS.ErrnoException | null, address?: string, family?: number) => void, + ) => void)(hostname, {}, (err, address, family) => { + resolve({ err, address, family }); + }); + }); +} + +describe("buildLookup (strict: allowPrivate=false)", () => { + const lookup = buildLookup(false); + + it("rejects localhost because it resolves to a loopback address", async () => { + const r = await callLookup(lookup, "localhost"); + expect(r.err).toBeTruthy(); + expect(r.err!.code).toMatch(/blocked_loopback/); + }); + + it("returns an address for a routable public hostname", async () => { + // 1.1.1.1.nip.io is a public DNS service that maps hostnames of the + // form a.b.c.d.nip.io → a.b.c.d. It's stable enough to use as a + // public-resolution probe. If the test environment has no network, + // this test will skip (we tolerate ENOTFOUND). + const r = await callLookup(lookup, "one.one.one.one"); + if (r.err && (r.err.code === "ENOTFOUND" || r.err.code === "EAI_AGAIN")) { + return; // offline test env — skip + } + expect(r.err).toBeNull(); + expect(r.address).toBeTruthy(); + expect(r.family === 4 || r.family === 6).toBe(true); + }, 10_000); + + it("rejects an IPv4 literal hostname when it resolves to a private address", async () => { + // dns.lookup of an IPv4 literal returns that literal immediately, + // exercising our validator on the connect-time address. + const r = await callLookup(lookup, "10.0.0.1"); + expect(r.err).toBeTruthy(); + expect(r.err!.code).toMatch(/blocked_private/); + }); + + it("rejects [::1] equivalent hostname", async () => { + const r = await callLookup(lookup, "::1"); + expect(r.err).toBeTruthy(); + expect(r.err!.code).toMatch(/blocked_loopback/); + }); + + it("rejects 169.254.169.254 (AWS metadata endpoint)", async () => { + const r = await callLookup(lookup, "169.254.169.254"); + expect(r.err).toBeTruthy(); + expect(r.err!.code).toMatch(/blocked_link_local/); + }); +}); + +describe("buildLookup (permissive: allowPrivate=true)", () => { + const lookup = buildLookup(true); + + it("accepts localhost when allowPrivate=true", async () => { + const r = await callLookup(lookup, "localhost"); + expect(r.err).toBeNull(); + expect(r.address).toBeTruthy(); + }); + + it("accepts 10.0.0.1 literal when allowPrivate=true", async () => { + const r = await callLookup(lookup, "10.0.0.1"); + expect(r.err).toBeNull(); + expect(r.address).toBe("10.0.0.1"); + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/cwdJail.test.ts b/packages/jimmy/src/engines/tools/__tests__/cwdJail.test.ts new file mode 100644 index 00000000..8b222684 --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/cwdJail.test.ts @@ -0,0 +1,174 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { resolveInJail, JailViolation } from "../cwdJail.js"; + +let jail: string; +let realJail: string; +let outside: string; + +beforeEach(async () => { + jail = await fs.mkdtemp(path.join(os.tmpdir(), "cwd-jail-")); + realJail = await fs.realpath(jail); + outside = await fs.mkdtemp(path.join(os.tmpdir(), "cwd-jail-outside-")); +}); + +afterEach(async () => { + await fs.rm(jail, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); +}); + +describe("resolveInJail — happy path (lexical)", () => { + it("resolves a simple relative path under cwd", async () => { + await expect(resolveInJail(jail, "foo.txt")).resolves.toBe(path.join(realJail, "foo.txt")); + }); + + it("resolves a nested relative path that doesn't yet exist", async () => { + await expect(resolveInJail(jail, "sub/dir/new.json")).resolves.toBe( + path.join(realJail, "sub/dir/new.json"), + ); + }); + + it("normalizes redundant './' segments", async () => { + await expect(resolveInJail(jail, "./foo/./bar.txt")).resolves.toBe( + path.join(realJail, "foo/bar.txt"), + ); + }); + + it("normalizes internal '..' that stays inside the jail", async () => { + await expect(resolveInJail(jail, "foo/../bar.txt")).resolves.toBe( + path.join(realJail, "bar.txt"), + ); + }); + + it("accepts an absolute path that is already under cwd", async () => { + await expect(resolveInJail(jail, path.join(jail, "x/y.txt"))).resolves.toBe( + path.join(realJail, "x/y.txt"), + ); + }); + + it("allows the cwd itself ('.')", async () => { + await expect(resolveInJail(jail, ".")).resolves.toBe(realJail); + }); +}); + +describe("resolveInJail — lexical_escape", () => { + it("rejects a leading '..' with reason=lexical_escape", async () => { + await expect(resolveInJail(jail, "../escape.txt")).rejects.toMatchObject({ + name: "JailViolation", + reason: "lexical_escape", + }); + }); + + it("rejects an absolute path outside cwd", async () => { + await expect(resolveInJail(jail, "/etc/passwd")).rejects.toMatchObject({ + reason: "lexical_escape", + }); + }); + + it("rejects a sibling-prefix path", async () => { + // jail = /tmp/cwd-jail-XYZ; sibling /tmp/cwd-jail-XYZ-EXTRA must be rejected. + const sibling = `${jail}-EXTRA`; + await expect(resolveInJail(jail, sibling)).rejects.toMatchObject({ + reason: "lexical_escape", + }); + }); + + it("JailViolation carries requestedPath, cwd, and reason", async () => { + try { + await resolveInJail(jail, "../oops"); + throw new Error("did not throw"); + } catch (err) { + expect(err).toBeInstanceOf(JailViolation); + expect((err as JailViolation).requestedPath).toBe("../oops"); + expect((err as JailViolation).cwd).toBe(jail); + expect((err as JailViolation).reason).toBe("lexical_escape"); + } + }); +}); + +describe("resolveInJail — realpath_escape (symlink-based)", () => { + it("rejects a leaf-symlink that points outside the jail", async () => { + const target = path.join(outside, "secret.txt"); + await fs.writeFile(target, "TOP-SECRET"); + await fs.symlink(target, path.join(jail, "link")); + // Without rejectSymlinkLeaf, the leaf-symlink check is skipped — but + // the realpath check still rejects because the target is outside. + await expect(resolveInJail(jail, "link")).rejects.toMatchObject({ + reason: "realpath_escape", + }); + }); + + it("rejects when the path traverses a parent-directory symlink that escapes", async () => { + await fs.writeFile(path.join(outside, "leak.txt"), "PARENT-LEAK"); + await fs.symlink(outside, path.join(jail, "esc")); + await expect(resolveInJail(jail, "esc/leak.txt")).rejects.toMatchObject({ + reason: "realpath_escape", + }); + }); + + it("allows a leaf-symlink whose target stays INSIDE the jail (no rejectSymlinkLeaf)", async () => { + const realFile = path.join(jail, "real.txt"); + await fs.writeFile(realFile, "inside"); + await fs.symlink(realFile, path.join(jail, "alias")); + // Resolved canonical path is the realpath of the target. + await expect(resolveInJail(jail, "alias")).resolves.toBe(await fs.realpath(realFile)); + }); +}); + +describe("resolveInJail — symlink_leaf (write/edit posture)", () => { + it("rejects a leaf-symlink with rejectSymlinkLeaf=true, even if target is inside the jail", async () => { + const realFile = path.join(jail, "real.txt"); + await fs.writeFile(realFile, "inside"); + await fs.symlink(realFile, path.join(jail, "alias")); + await expect(resolveInJail(jail, "alias", { rejectSymlinkLeaf: true })).rejects.toMatchObject({ + reason: "symlink_leaf", + }); + }); + + it("does not reject the leaf when it's a regular file under rejectSymlinkLeaf", async () => { + await fs.writeFile(path.join(jail, "regular.txt"), "x"); + await expect(resolveInJail(jail, "regular.txt", { rejectSymlinkLeaf: true })).resolves.toBe( + path.join(realJail, "regular.txt"), + ); + }); + + it("does not reject a non-existent leaf under rejectSymlinkLeaf (write-new-file case)", async () => { + await expect(resolveInJail(jail, "new-file.txt", { rejectSymlinkLeaf: true })).resolves.toBe( + path.join(realJail, "new-file.txt"), + ); + }); + + it("still rejects parent-symlink-escape under rejectSymlinkLeaf", async () => { + await fs.symlink(outside, path.join(jail, "esc")); + await expect( + resolveInJail(jail, "esc/new-file.txt", { rejectSymlinkLeaf: true }), + ).rejects.toMatchObject({ reason: "realpath_escape" }); + }); +}); + +describe("resolveInJail — malformed input", () => { + it("rejects non-string input", async () => { + await expect( + // @ts-expect-error — intentional bad type + resolveInJail(jail, 42), + ).rejects.toThrow(/path must be a string/); + }); + + it("rejects empty string", async () => { + await expect(resolveInJail(jail, "")).rejects.toThrow(/non-empty string/); + }); + + it("rejects NUL bytes in path", async () => { + await expect(resolveInJail(jail, "foo\0bar")).rejects.toThrow(/NUL bytes/); + }); +}); + +describe("resolveInJail — cwd normalization", () => { + it("treats a trailing-slash cwd as equivalent", async () => { + await expect(resolveInJail(jail + path.sep, "foo.txt")).resolves.toBe( + path.join(realJail, "foo.txt"), + ); + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/fs-tools-jail.test.ts b/packages/jimmy/src/engines/tools/__tests__/fs-tools-jail.test.ts new file mode 100644 index 00000000..43a2616a --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/fs-tools-jail.test.ts @@ -0,0 +1,175 @@ +/** + * Regression tests for the symlink escape, size-cap, and write-to-cwd + * hardening landed in Phase 3a. Every test in this file describes a + * scenario where the unhardened tool surface previously leaked, clobbered, + * or wasted memory — and the new behavior should refuse cleanly. + */ + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { readTool } from "../read.js"; +import { writeTool } from "../write.js"; +import { editTool } from "../edit.js"; +import type { ToolExecutionContext } from "../types.js"; + +let jail: string; +let outside: string; +let ctx: ToolExecutionContext; + +beforeEach(async () => { + jail = await fs.mkdtemp(path.join(os.tmpdir(), "jin-jail-regr-")); + outside = await fs.mkdtemp(path.join(os.tmpdir(), "jin-jail-outside-")); + ctx = { cwd: jail }; +}); + +afterEach(async () => { + await fs.rm(jail, { recursive: true, force: true }); + await fs.rm(outside, { recursive: true, force: true }); +}); + +// ─── Symlink escape regressions ────────────────────────────────────── + +describe("read: symlink escape", () => { + it("does NOT read through a leaf-symlink that escapes the jail", async () => { + const secretPath = path.join(outside, "secret.txt"); + await fs.writeFile(secretPath, "TOP-SECRET"); + await fs.symlink(secretPath, path.join(jail, "inner-link")); + + const r = await readTool({ path: "inner-link" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("realpath_escape"); + expect(r.content).not.toContain("TOP-SECRET"); + }); + + it("does NOT traverse a parent-directory symlink that escapes", async () => { + await fs.writeFile(path.join(outside, "leak.txt"), "PARENT-LEAK"); + await fs.symlink(outside, path.join(jail, "esc")); + + const r = await readTool({ path: "esc/leak.txt" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("realpath_escape"); + expect(r.content).not.toContain("PARENT-LEAK"); + }); + + it("DOES read through a leaf-symlink whose target stays inside the jail", async () => { + // read intentionally permits symlinks-to-inside; the realpath check + // proves there's no escape. (write/edit are stricter — see below.) + const real = path.join(jail, "real.txt"); + await fs.writeFile(real, "inside-content"); + await fs.symlink(real, path.join(jail, "alias")); + + const r = await readTool({ path: "alias" }, ctx); + expect(r.ok).toBe(true); + expect(r.content).toBe("inside-content"); + }); +}); + +describe("write: symlink escape", () => { + it("does NOT clobber via a leaf-symlink that points outside", async () => { + const victim = path.join(outside, "victim.txt"); + await fs.writeFile(victim, "ORIGINAL"); + await fs.symlink(victim, path.join(jail, "link")); + + const r = await writeTool({ path: "link", content: "CLOBBERED" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("symlink_leaf"); + expect(await fs.readFile(victim, "utf8")).toBe("ORIGINAL"); + }); + + it("REFUSES to write through any leaf-symlink, even to a target inside the jail", async () => { + const real = path.join(jail, "real.txt"); + await fs.writeFile(real, "before"); + await fs.symlink(real, path.join(jail, "alias")); + + const r = await writeTool({ path: "alias", content: "changed" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("symlink_leaf"); + expect(await fs.readFile(real, "utf8")).toBe("before"); + }); + + it("does NOT create a new file via a parent-dir symlink that escapes", async () => { + await fs.symlink(outside, path.join(jail, "esc")); + const r = await writeTool({ path: "esc/new.txt", content: "x" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("realpath_escape"); + // No file should appear in `outside`. + await expect(fs.stat(path.join(outside, "new.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); + +describe("edit: symlink escape", () => { + it("does NOT modify a file via a leaf-symlink that escapes", async () => { + const victim = path.join(outside, "victim.txt"); + await fs.writeFile(victim, "before-MARKER-after"); + await fs.symlink(victim, path.join(jail, "link")); + + const r = await editTool({ path: "link", old_string: "MARKER", new_string: "EDITED" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("symlink_leaf"); + expect(await fs.readFile(victim, "utf8")).toBe("before-MARKER-after"); + }); +}); + +// ─── Size caps ──────────────────────────────────────────────────────── + +describe("read: file-size cap", () => { + it("refuses files larger than 5MB with error=too_large", async () => { + const big = "x".repeat(6 * 1024 * 1024); // 6MB + await fs.writeFile(path.join(jail, "big.txt"), big); + + const r = await readTool({ path: "big.txt" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("too_large"); + expect(r.audit.file_bytes).toBe(big.length); + }); + + it("allows files exactly at 5MB", async () => { + const exact = "x".repeat(5 * 1024 * 1024); + await fs.writeFile(path.join(jail, "exact.txt"), exact); + + const r = await readTool({ path: "exact.txt" }, ctx); + expect(r.ok).toBe(true); + // Model-output truncation still kicks in at 64k by default. + expect(r.audit.truncated).toBe(true); + }); +}); + +describe("edit: file-size cap", () => { + it("refuses files larger than 5MB with error=too_large (no read attempted)", async () => { + const big = "x".repeat(6 * 1024 * 1024) + "MARKER"; + await fs.writeFile(path.join(jail, "big.txt"), big); + + const r = await editTool({ path: "big.txt", old_string: "MARKER", new_string: "Y" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("too_large"); + // File unchanged + const after = await fs.readFile(path.join(jail, "big.txt"), "utf8"); + expect(after.endsWith("MARKER")).toBe(true); + }); +}); + +// ─── Write-to-cwd-itself guard ─────────────────────────────────────── + +describe("write: refusing to overwrite the cwd directory", () => { + it("rejects path='.' with error=is_cwd_dir (does NOT mkdir parent-of-cwd)", async () => { + // The is_cwd_dir guard short-circuits BEFORE the recursive mkdir + // runs, so we don't need to inspect the parent's listing (which is + // flappy on Linux CI — `/tmp` has system mounts like .ICE-unix / + // .X11-unix that come and go between calls). The audit code + // assertion alone proves the guard fired. + const r = await writeTool({ path: ".", content: "anything" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("is_cwd_dir"); + // The jail dir itself still exists and was not converted to a file. + const jailStat = await fs.stat(jail); + expect(jailStat.isDirectory()).toBe(true); + }); + + it("rejects an absolute path equal to cwd", async () => { + const r = await writeTool({ path: jail, content: "anything" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("is_cwd_dir"); + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/fs-tools.test.ts b/packages/jimmy/src/engines/tools/__tests__/fs-tools.test.ts new file mode 100644 index 00000000..a4282ecb --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/fs-tools.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { readTool } from "../read.js"; +import { writeTool } from "../write.js"; +import { editTool } from "../edit.js"; +import type { ToolExecutionContext } from "../types.js"; + +let tmpDir: string; +let ctx: ToolExecutionContext; + +beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "jin-fs-tools-")); + ctx = { cwd: tmpDir }; +}); + +afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); +}); + +// ─── read ──────────────────────────────────────────────────────────── + +describe("tools/read", () => { + it("reads a file and returns its content", async () => { + await fs.writeFile(path.join(tmpDir, "a.txt"), "line1\nline2\nline3"); + const r = await readTool({ path: "a.txt" }, ctx); + expect(r.ok).toBe(true); + expect(r.content).toBe("line1\nline2\nline3"); + expect(r.audit.truncated).toBe(false); + expect(r.audit.total_lines).toBe(3); + expect(r.audit.returned_lines).toBe(3); + }); + + it("applies 1-indexed offset", async () => { + await fs.writeFile(path.join(tmpDir, "a.txt"), "a\nb\nc\nd\ne"); + const r = await readTool({ path: "a.txt", offset: 3 }, ctx); + expect(r.content).toBe("c\nd\ne"); + }); + + it("respects limit", async () => { + await fs.writeFile(path.join(tmpDir, "a.txt"), "a\nb\nc\nd\ne"); + const r = await readTool({ path: "a.txt", offset: 2, limit: 2 }, ctx); + expect(r.content).toBe("b\nc"); + expect(r.audit.returned_lines).toBe(2); + }); + + it("truncates when content exceeds maxChars and appends a marker", async () => { + const big = "x".repeat(80_000); + await fs.writeFile(path.join(tmpDir, "big.txt"), big); + const r = await readTool({ path: "big.txt" }, ctx); + expect(r.audit.truncated).toBe(true); + expect(r.content.length).toBeGreaterThan(64_000); + expect(r.content.length).toBeLessThan(64_500); + expect(r.content).toMatch(/\[truncated: 64000 of 80000 characters\]/); + expect(r.audit.originalBytes).toBe(80_000); + }); + + it("honors a per-engine maxChars override from ctx.toolOpts", async () => { + await fs.writeFile(path.join(tmpDir, "a.txt"), "x".repeat(2000)); + const r = await readTool( + { path: "a.txt" }, + { ...ctx, toolOpts: { read: { maxChars: 500 } } }, + ); + expect(r.audit.truncated).toBe(true); + expect(r.content).toMatch(/\[truncated: 500 of 2000/); + }); + + it("returns ok:false with lexical_escape on '..' escape", async () => { + const r = await readTool({ path: "../escape.txt" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("lexical_escape"); + expect(r.content).toMatch(/violates jail/); + }); + + it("returns ok:false with ENOENT when file missing", async () => { + const r = await readTool({ path: "nope.txt" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("ENOENT"); + }); + + it("rejects non-string path with bad_args", async () => { + const r = await readTool({ path: 42 as unknown as string }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); + + it("rejects negative offset with bad_args", async () => { + await fs.writeFile(path.join(tmpDir, "a.txt"), "x"); + const r = await readTool({ path: "a.txt", offset: 0 }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); +}); + +// ─── write ─────────────────────────────────────────────────────────── + +describe("tools/write", () => { + it("writes a new file under cwd", async () => { + const r = await writeTool({ path: "new.txt", content: "hello" }, ctx); + expect(r.ok).toBe(true); + expect(await fs.readFile(path.join(tmpDir, "new.txt"), "utf8")).toBe("hello"); + expect(r.audit.bytes_written).toBe(5); + }); + + it("overwrites an existing file", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "old"); + const r = await writeTool({ path: "x.txt", content: "new" }, ctx); + expect(r.ok).toBe(true); + expect(await fs.readFile(path.join(tmpDir, "x.txt"), "utf8")).toBe("new"); + }); + + it("creates parent directories recursively under cwd", async () => { + const r = await writeTool({ path: "a/b/c/file.txt", content: "nested" }, ctx); + expect(r.ok).toBe(true); + expect(await fs.readFile(path.join(tmpDir, "a/b/c/file.txt"), "utf8")).toBe("nested"); + }); + + it("rejects jail escape", async () => { + const r = await writeTool({ path: "../outside.txt", content: "x" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("lexical_escape"); + }); + + it("rejects missing content arg", async () => { + // Cast through unknown because JsonObject's index signature is JsonValue; + // the test intentionally passes an under-specified shape. + const r = await writeTool({ path: "x.txt" } as unknown as Record, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); + + it("accepts an empty content string as a valid 'truncate to empty' op", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "old"); + const r = await writeTool({ path: "x.txt", content: "" }, ctx); + expect(r.ok).toBe(true); + expect(await fs.readFile(path.join(tmpDir, "x.txt"), "utf8")).toBe(""); + }); + + it("computes UTF-8 byte length, not char length, for multibyte content", async () => { + const content = "héllo"; // 6 bytes UTF-8, 5 chars + const r = await writeTool({ path: "x.txt", content }, ctx); + expect(r.ok).toBe(true); + expect(r.audit.bytes_written).toBe(6); + }); +}); + +// ─── edit ──────────────────────────────────────────────────────────── + +describe("tools/edit", () => { + it("replaces a unique occurrence", async () => { + await fs.writeFile(path.join(tmpDir, "x.ts"), "const foo = 1;\nconst bar = 2;"); + const r = await editTool({ path: "x.ts", old_string: "const foo = 1;", new_string: "const foo = 42;" }, ctx); + expect(r.ok).toBe(true); + expect(r.audit.replacements).toBe(1); + expect(await fs.readFile(path.join(tmpDir, "x.ts"), "utf8")).toBe("const foo = 42;\nconst bar = 2;"); + }); + + it("refuses when old_string matches multiple times and replace_all is false", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "dup\ndup\ndup"); + const r = await editTool({ path: "x.txt", old_string: "dup", new_string: "X" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("ambiguous"); + expect(r.audit.matches).toBe(3); + expect(await fs.readFile(path.join(tmpDir, "x.txt"), "utf8")).toBe("dup\ndup\ndup"); // unchanged + }); + + it("replaces all when replace_all=true", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "dup\ndup\ndup"); + const r = await editTool({ path: "x.txt", old_string: "dup", new_string: "X", replace_all: true }, ctx); + expect(r.ok).toBe(true); + expect(r.audit.replacements).toBe(3); + expect(await fs.readFile(path.join(tmpDir, "x.txt"), "utf8")).toBe("X\nX\nX"); + }); + + it("refuses no-op (old_string === new_string)", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "abc"); + const r = await editTool({ path: "x.txt", old_string: "abc", new_string: "abc" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("noop"); + }); + + it("returns not_found when old_string is absent", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "abc"); + const r = await editTool({ path: "x.txt", old_string: "ZZZ", new_string: "YYY" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("not_found"); + }); + + it("rejects empty old_string with bad_args", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "abc"); + const r = await editTool({ path: "x.txt", old_string: "", new_string: "abc" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); + + it("rejects jail escape", async () => { + const r = await editTool({ path: "../e.txt", old_string: "a", new_string: "b" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("lexical_escape"); + }); + + it("returns ENOENT when file missing", async () => { + const r = await editTool({ path: "nope.txt", old_string: "a", new_string: "b" }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("ENOENT"); + }); + + it("can delete content by replacing with empty string", async () => { + await fs.writeFile(path.join(tmpDir, "x.txt"), "before-DELETEME-after"); + const r = await editTool({ path: "x.txt", old_string: "-DELETEME-", new_string: "" }, ctx); + expect(r.ok).toBe(true); + expect(await fs.readFile(path.join(tmpDir, "x.txt"), "utf8")).toBe("beforeafter"); + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/ipBlocklist.test.ts b/packages/jimmy/src/engines/tools/__tests__/ipBlocklist.test.ts new file mode 100644 index 00000000..5cf7cc7d --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/ipBlocklist.test.ts @@ -0,0 +1,132 @@ +import { describe, it, expect } from "vitest"; +import { checkIPv4, checkIPv6, parseIpLiteral } from "../ipBlocklist.js"; + +describe("ipBlocklist — IPv4", () => { + it("returns null for routable public addresses", () => { + expect(checkIPv4("1.1.1.1")).toBeNull(); + expect(checkIPv4("8.8.8.8")).toBeNull(); + expect(checkIPv4("142.250.190.78")).toBeNull(); // google + }); + + it("blocks 0.0.0.0/8 as unspecified", () => { + expect(checkIPv4("0.0.0.0")).toBe("blocked_unspecified"); + expect(checkIPv4("0.1.2.3")).toBe("blocked_unspecified"); + }); + + it("blocks loopback 127.0.0.0/8", () => { + expect(checkIPv4("127.0.0.1")).toBe("blocked_loopback"); + expect(checkIPv4("127.255.255.255")).toBe("blocked_loopback"); + }); + + it("blocks RFC1918 ranges", () => { + expect(checkIPv4("10.0.0.1")).toBe("blocked_private"); + expect(checkIPv4("10.255.255.255")).toBe("blocked_private"); + expect(checkIPv4("172.16.0.1")).toBe("blocked_private"); + expect(checkIPv4("172.31.255.255")).toBe("blocked_private"); + expect(checkIPv4("172.15.0.1")).toBeNull(); // just outside + expect(checkIPv4("172.32.0.1")).toBeNull(); + expect(checkIPv4("192.168.0.1")).toBe("blocked_private"); + expect(checkIPv4("192.168.255.255")).toBe("blocked_private"); + }); + + it("blocks CGNAT 100.64.0.0/10", () => { + expect(checkIPv4("100.64.0.1")).toBe("blocked_cgnat"); + expect(checkIPv4("100.127.255.255")).toBe("blocked_cgnat"); + expect(checkIPv4("100.63.255.255")).toBeNull(); // just outside + expect(checkIPv4("100.128.0.1")).toBeNull(); + }); + + it("blocks link-local 169.254.0.0/16", () => { + expect(checkIPv4("169.254.1.1")).toBe("blocked_link_local"); + expect(checkIPv4("169.254.255.255")).toBe("blocked_link_local"); + expect(checkIPv4("169.253.255.255")).toBeNull(); + }); + + it("blocks multicast 224.0.0.0/4", () => { + expect(checkIPv4("224.0.0.1")).toBe("blocked_multicast"); + expect(checkIPv4("239.255.255.255")).toBe("blocked_multicast"); + expect(checkIPv4("223.255.255.255")).toBeNull(); + }); + + it("blocks broadcast 255.255.255.255", () => { + expect(checkIPv4("255.255.255.255")).toBe("blocked_broadcast"); + }); + + it("returns null for non-IPv4 input", () => { + expect(checkIPv4("not-an-ip")).toBeNull(); + expect(checkIPv4("::1")).toBeNull(); // IPv6 not handled here + }); +}); + +describe("ipBlocklist — IPv6", () => { + it("returns null for routable public addresses", () => { + expect(checkIPv6("2001:4860:4860::8888")).toBeNull(); // google + expect(checkIPv6("2606:4700:4700::1111")).toBeNull(); // cloudflare + }); + + it("blocks :: (unspecified)", () => { + expect(checkIPv6("::")).toBe("blocked_unspecified"); + }); + + it("blocks ::1 (loopback)", () => { + expect(checkIPv6("::1")).toBe("blocked_loopback"); + }); + + it("blocks fc00::/7 (unique local)", () => { + expect(checkIPv6("fc00::1")).toBe("blocked_unique_local"); + expect(checkIPv6("fd12:3456:789a:bcde::1")).toBe("blocked_unique_local"); + expect(checkIPv6("fdff::1")).toBe("blocked_unique_local"); + }); + + it("blocks fe80::/10 (link-local)", () => { + expect(checkIPv6("fe80::1")).toBe("blocked_link_local"); + expect(checkIPv6("febf::1")).toBe("blocked_link_local"); + expect(checkIPv6("fec0::1")).toBeNull(); // just outside fe80::/10 (fec0/10 is site-local, deprecated; not blocked here) + }); + + it("blocks ff00::/8 (multicast)", () => { + expect(checkIPv6("ff02::1")).toBe("blocked_multicast"); + expect(checkIPv6("ff05::1:3")).toBe("blocked_multicast"); + }); + + it("blocks IPv4-mapped IPv6 addresses by their embedded v4", () => { + expect(checkIPv6("::ffff:127.0.0.1")).toBe("blocked_loopback"); + expect(checkIPv6("::ffff:10.0.0.1")).toBe("blocked_private"); + expect(checkIPv6("::ffff:169.254.1.1")).toBe("blocked_link_local"); + expect(checkIPv6("::ffff:8.8.8.8")).toBeNull(); + }); + + it("handles zone-id suffix in fe80::%iface form", () => { + expect(checkIPv6("fe80::1%eth0")).toBe("blocked_link_local"); + }); + + it("returns null for non-IPv6 input", () => { + expect(checkIPv6("not-an-ip")).toBeNull(); + expect(checkIPv6("127.0.0.1")).toBeNull(); + }); +}); + +describe("ipBlocklist — parseIpLiteral", () => { + it("recognizes bare IPv4", () => { + expect(parseIpLiteral("127.0.0.1")).toEqual({ family: 4, address: "127.0.0.1" }); + }); + + it("recognizes bracketed IPv6 (URL style)", () => { + expect(parseIpLiteral("[::1]")).toEqual({ family: 6, address: "::1" }); + expect(parseIpLiteral("[2001:db8::1]")).toEqual({ family: 6, address: "2001:db8::1" }); + }); + + it("recognizes unbracketed IPv6", () => { + expect(parseIpLiteral("::1")).toEqual({ family: 6, address: "::1" }); + }); + + it("returns null for hostnames", () => { + expect(parseIpLiteral("example.com")).toBeNull(); + expect(parseIpLiteral("localhost")).toBeNull(); // is a hostname, not a literal + }); + + it("returns null for empty/invalid input", () => { + expect(parseIpLiteral("")).toBeNull(); + expect(parseIpLiteral("not.an.ip")).toBeNull(); + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/registry.test.ts b/packages/jimmy/src/engines/tools/__tests__/registry.test.ts new file mode 100644 index 00000000..47c04bb8 --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/registry.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect } from "vitest"; +import { buildToolRegistry, KNOWN_TOOL_NAMES } from "../index.js"; + +describe("buildToolRegistry", () => { + it("returns an empty registry when toolsConfig is undefined (text-only)", () => { + const r = buildToolRegistry(undefined); + expect(r.executors.size).toBe(0); + expect(r.schemas).toEqual([]); + expect(r.unknownRequested).toEqual([]); + }); + + it("returns an empty registry when enabled is an empty array", () => { + const r = buildToolRegistry({ enabled: [] }); + expect(r.executors.size).toBe(0); + expect(r.schemas).toEqual([]); + }); + + it("returns only the requested tools", () => { + const r = buildToolRegistry({ enabled: ["read", "write"] }); + expect([...r.executors.keys()].sort()).toEqual(["read", "write"]); + expect(r.schemas.map((s) => s.name).sort()).toEqual(["read", "write"]); + }); + + it("preserves order of `enabled` in schemas array", () => { + const r = buildToolRegistry({ enabled: ["webfetch", "read"] }); + expect(r.schemas.map((s) => s.name)).toEqual(["webfetch", "read"]); + }); + + it("ignores duplicates in enabled", () => { + const r = buildToolRegistry({ enabled: ["read", "read", "read"] }); + expect(r.executors.size).toBe(1); + expect(r.schemas).toHaveLength(1); + }); + + it("collects unknown tool names without throwing", () => { + const r = buildToolRegistry({ enabled: ["read", "nonexistent_tool", "write"] }); + expect([...r.executors.keys()].sort()).toEqual(["read", "write"]); + expect(r.unknownRequested).toEqual(["nonexistent_tool"]); + }); + + it("exposes the full known-tool set", () => { + const r = buildToolRegistry({ enabled: [...KNOWN_TOOL_NAMES] }); + expect(r.executors.size).toBe(5); + expect(r.schemas).toHaveLength(5); + }); + + it("each schema has required JSON-schema fields", () => { + const r = buildToolRegistry({ enabled: [...KNOWN_TOOL_NAMES] }); + for (const s of r.schemas) { + expect(typeof s.name).toBe("string"); + expect(typeof s.description).toBe("string"); + expect(s.parameters).toBeTruthy(); + expect((s.parameters as Record).type).toBe("object"); + } + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/runCommand.test.ts b/packages/jimmy/src/engines/tools/__tests__/runCommand.test.ts new file mode 100644 index 00000000..996888e2 --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/runCommand.test.ts @@ -0,0 +1,418 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { runCommandTool } from "../runCommand.js"; +import type { ToolExecutionContext } from "../types.js"; +import type { JsonObject, JsonValue } from "../../../shared/types.js"; + +let jail: string; +let ctx: ToolExecutionContext; + +beforeEach(async () => { + jail = await fs.mkdtemp(path.join(os.tmpdir(), "runcmd-test-")); + ctx = { cwd: jail }; +}); + +afterEach(async () => { + await fs.rm(jail, { recursive: true, force: true }); +}); + +function withBashOpts(overrides: Record): ToolExecutionContext { + return { cwd: jail, toolOpts: { bash: overrides as JsonObject } }; +} + +// ─── Disabled / allowlist ──────────────────────────────────────────── + +describe("runCommand: tool disabled", () => { + it("returns disabled when no toolOpts at all", async () => { + const r = await runCommandTool({ command: "echo", args: ["hi"] }, ctx); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("disabled"); + }); + + it("returns disabled when bashAllowlist is empty", async () => { + const r = await runCommandTool( + { command: "echo", args: ["hi"] }, + withBashOpts({ allowlist: [] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("disabled"); + }); +}); + +describe("runCommand: allowlist gate", () => { + it("rejects argv[0] not in allowlist", async () => { + const r = await runCommandTool( + { command: "rm", args: ["-rf", "/"] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("not_in_allowlist"); + }); + + it("accepts argv[0] in allowlist (basename matches)", async () => { + const r = await runCommandTool( + { command: "echo", args: ["hello"] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(true); + expect(r.content).toContain("hello"); + expect(r.audit.exit_code).toBe(0); + }); + + it("matches by basename when command is an absolute path", async () => { + const r = await runCommandTool( + { command: "/bin/echo", args: ["abs"] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(true); + expect(r.content).toContain("abs"); + }); +}); + +// ─── Hardcoded NEVER-LIST ──────────────────────────────────────────── + +describe("runCommand: shell + bypass blocklist (never overridable)", () => { + it.each([ + "sh", "bash", "zsh", "fish", "ksh", "csh", "tcsh", "dash", "ash", + "env", "xargs", "eval", "exec", "source", + ])("blocks %s even when explicitly allowlisted", async (name) => { + const r = await runCommandTool( + { command: name, args: [] }, + withBashOpts({ allowlist: [name] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("shell_blocked"); + }); + + it("blocks /bin/bash by basename, not full path", async () => { + const r = await runCommandTool( + { command: "/bin/bash", args: ["-c", "echo pwned"] }, + withBashOpts({ allowlist: ["bash"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("shell_blocked"); + }); + + it("blocks env even though env+arg looks innocuous", async () => { + const r = await runCommandTool( + { command: "env", args: ["PATH=/bin", "echo", "hi"] }, + withBashOpts({ allowlist: ["env", "echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("shell_blocked"); + }); +}); + +// ─── Metacharacter rejection ──────────────────────────────────────── + +describe("runCommand: shell metacharacters rejected upfront", () => { + const metas = [";", "|", "&", "`", "$", ">", "<", "\n", "\r", "\t", "*", "?", "~", "(", ")", "{", "}", "\\", "[", "]", "!"]; + it.each(metas)("rejects %j in args[0]", async (ch) => { + const r = await runCommandTool( + { command: "echo", args: [`hi${ch}there`] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("metacharacter"); + }); + + it("rejects NUL byte in command", async () => { + const r = await runCommandTool( + { command: "echo\0", args: [] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("metacharacter"); + }); + + it("rejects shell-injection attempt via args", async () => { + const r = await runCommandTool( + { command: "echo", args: ["hi; rm -rf /"] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("metacharacter"); + }); +}); + +// ─── Python wrapper restrictions ───────────────────────────────────── + +describe("runCommand: python3 restrictions", () => { + it("rejects python3 -c (inline code execution)", async () => { + // Use a metachar-free code string so the python3-flag check fires + // before the metachar check, isolating this assertion to the -c gate. + const r = await runCommandTool( + { command: "python3", args: ["-c", "pass"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("python3_unsafe_args"); + }); + + it("rejects python3 -c even when code contains metacharacters (either gate is acceptable)", async () => { + const r = await runCommandTool( + { command: "python3", args: ["-c", "print('x')"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(["python3_unsafe_args", "metacharacter"]).toContain(r.audit.error); + }); + + it("rejects python3 -m ", async () => { + const r = await runCommandTool( + { command: "python3", args: ["-m", "http.server"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("python3_unsafe_args"); + }); + + it("rejects python3 with stdin (bare -)", async () => { + const r = await runCommandTool( + { command: "python3", args: ["-"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("python3_unsafe_args"); + }); + + it("rejects python3 with no positional script", async () => { + const r = await runCommandTool( + { command: "python3", args: [] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("python3_no_script"); + }); + + it("rejects python3 with a script path that escapes the cwd jail", async () => { + const outside = await fs.mkdtemp(path.join(os.tmpdir(), "runcmd-outside-")); + try { + const scriptPath = path.join(outside, "evil.py"); + await fs.writeFile(scriptPath, "print('pwned')"); + const r = await runCommandTool( + { command: "python3", args: [scriptPath] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("lexical_escape"); + } finally { + await fs.rm(outside, { recursive: true, force: true }); + } + }); + + it("rejects python3 with a script that doesn't exist", async () => { + const r = await runCommandTool( + { command: "python3", args: ["does-not-exist.py"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("python3_script_missing"); + }); + + it("accepts python3 with a script that exists under cwd", async () => { + await fs.writeFile(path.join(jail, "ok.py"), "print('done')"); + const r = await runCommandTool( + { command: "python3", args: ["ok.py"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(true); + expect(r.content).toContain("done"); + }); + + it("allows non-banned flags before the script (e.g. -O)", async () => { + await fs.writeFile(path.join(jail, "ok.py"), "print('opt')"); + const r = await runCommandTool( + { command: "python3", args: ["-O", "ok.py"] }, + withBashOpts({ allowlist: ["python3"] }), + ); + expect(r.ok).toBe(true); + expect(r.content).toContain("opt"); + }); +}); + +// ─── Exit code propagation ─────────────────────────────────────────── + +describe("runCommand: exit codes", () => { + it("ok=false on nonzero exit", async () => { + // `false` exits 1; needs to be on allowlist. + const r = await runCommandTool( + { command: "false", args: [] }, + withBashOpts({ allowlist: ["false"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.exit_code).toBe(1); + expect(r.audit.error).toBe("nonzero_exit"); + expect(r.audit.timeout).toBe(false); + }); + + it("ok=true on exit 0", async () => { + const r = await runCommandTool( + { command: "true", args: [] }, + withBashOpts({ allowlist: ["true"] }), + ); + expect(r.ok).toBe(true); + expect(r.audit.exit_code).toBe(0); + expect(r.audit.error).toBeUndefined(); + }); + + it("ok=false on spawn ENOENT (command not found on PATH)", async () => { + const r = await runCommandTool( + { command: "definitely-not-a-real-binary-xyz", args: [] }, + withBashOpts({ allowlist: ["definitely-not-a-real-binary-xyz"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("ENOENT"); + }); +}); + +// ─── Truncation (separate stdout/stderr) ───────────────────────────── + +describe("runCommand: truncation", () => { + it("truncates stdout independently of stderr", async () => { + await fs.writeFile( + path.join(jail, "noisy.py"), + "import sys; sys.stdout.write('o' * 50000); sys.stderr.write('e' * 5000); sys.exit(0)", + ); + const r = await runCommandTool( + { command: "python3", args: ["noisy.py"] }, + withBashOpts({ allowlist: ["python3"], maxStdout: 32000, maxStderr: 16000 }), + ); + expect(r.ok).toBe(true); + expect(r.audit.truncated_stdout).toBe(true); + expect(r.audit.truncated_stderr).toBe(false); + expect(r.audit.truncated).toBe(true); // OR + expect(r.audit.original_stdout_bytes).toBe(50000); + expect(r.audit.original_stderr_bytes).toBe(5000); + expect(r.content).toMatch(/stdout capped at 32000 of 50000 bytes/); + }); + + it("truncates stderr independently when stderr is the noisy one", async () => { + await fs.writeFile( + path.join(jail, "noisy.py"), + "import sys; sys.stdout.write('o' * 1000); sys.stderr.write('e' * 30000); sys.exit(0)", + ); + const r = await runCommandTool( + { command: "python3", args: ["noisy.py"] }, + withBashOpts({ allowlist: ["python3"], maxStdout: 32000, maxStderr: 16000 }), + ); + expect(r.audit.truncated_stdout).toBe(false); + expect(r.audit.truncated_stderr).toBe(true); + expect(r.audit.truncated).toBe(true); + expect(r.content).toMatch(/stderr capped at 16000 of 30000 bytes/); + }); + + it("no truncation flags when output fits under caps", async () => { + const r = await runCommandTool( + { command: "echo", args: ["short"] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.audit.truncated).toBe(false); + expect(r.audit.truncated_stdout).toBe(false); + expect(r.audit.truncated_stderr).toBe(false); + }); +}); + +// ─── Timeout + kill ────────────────────────────────────────────────── + +describe("runCommand: timeout + SIGTERM/SIGKILL", () => { + it("times out at perCallTimeoutMs and reports timeout=true", async () => { + await fs.writeFile( + path.join(jail, "loop.py"), + "import time\nwhile True: time.sleep(1)\n", + ); + const r = await runCommandTool( + { command: "python3", args: ["loop.py"] }, + withBashOpts({ + allowlist: ["python3"], + perCallTimeoutMs: 300, + killGraceMs: 100, + }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("timeout"); + expect(r.audit.timeout).toBe(true); + expect(r.audit.signal === "SIGTERM" || r.audit.signal === "SIGKILL").toBe(true); + expect(r.content).toMatch(/timed out after 300ms/); + }); + + it("escalates SIGTERM → SIGKILL when the process ignores SIGTERM", async () => { + // Trap SIGTERM and keep running; only SIGKILL can stop it. + await fs.writeFile( + path.join(jail, "stubborn.py"), + [ + "import signal, time", + "signal.signal(signal.SIGTERM, signal.SIG_IGN)", + "while True:", + " time.sleep(0.1)", + ].join("\n") + "\n", + ); + const r = await runCommandTool( + { command: "python3", args: ["stubborn.py"] }, + withBashOpts({ + allowlist: ["python3"], + perCallTimeoutMs: 200, + killGraceMs: 200, + }), + ); + expect(r.audit.timeout).toBe(true); + expect(r.audit.signal).toBe("SIGKILL"); + }); +}); + +// ─── Audit row shape ───────────────────────────────────────────────── + +describe("runCommand: audit row shape", () => { + it("includes command + summarized args + duration", async () => { + const r = await runCommandTool( + { command: "echo", args: ["one", "two"] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.audit.command).toBe("echo"); + expect(r.audit.args).toEqual(["one", "two"]); + expect(typeof r.audit.duration_ms).toBe("number"); + expect(r.audit.exit_code).toBe(0); + expect(r.audit.signal).toBeNull(); + }); + + it("truncates very long args in the audit row", async () => { + const long = "x".repeat(500); + const r = await runCommandTool( + { command: "echo", args: [long] }, + withBashOpts({ allowlist: ["echo"] }), + ); + const args = r.audit.args as string[]; + expect(args[0]!.length).toBeLessThan(300); + expect(args[0]).toMatch(/more chars/); + }); +}); + +// ─── Bad arg shapes ────────────────────────────────────────────────── + +describe("runCommand: bad arg shapes", () => { + it("rejects missing command", async () => { + const r = await runCommandTool({} as JsonObject, withBashOpts({ allowlist: ["echo"] })); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); + + it("rejects non-array args", async () => { + const r = await runCommandTool( + { command: "echo", args: "hi" } as unknown as JsonObject, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); + + it("rejects non-string elements inside args", async () => { + const r = await runCommandTool( + { command: "echo", args: ["ok", 42 as unknown as string] }, + withBashOpts({ allowlist: ["echo"] }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); +}); diff --git a/packages/jimmy/src/engines/tools/__tests__/webfetch.test.ts b/packages/jimmy/src/engines/tools/__tests__/webfetch.test.ts new file mode 100644 index 00000000..4364810c --- /dev/null +++ b/packages/jimmy/src/engines/tools/__tests__/webfetch.test.ts @@ -0,0 +1,340 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { webfetchTool } from "../webfetch.js"; +import type { ToolExecutionContext } from "../types.js"; +import type { JsonObject, JsonValue } from "../../../shared/types.js"; + +// ─── Fixture HTTP server ───────────────────────────────────────────── + +let server: http.Server; +let baseUrl: string; +let port: number; + +beforeAll(async () => { + server = http.createServer((req, res) => { + const url = new URL(req.url ?? "/", `http://127.0.0.1`); + const route = url.pathname; + + if (route === "/text") { + res.writeHead(200, { "Content-Type": "text/plain; charset=utf-8" }); + res.end("hello, fixture\n"); + return; + } + if (route === "/json") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true, n: 42 })); + return; + } + if (route === "/html") { + res.writeHead(200, { "Content-Type": "text/html" }); + res.end("

fix

"); + return; + } + if (route === "/binary") { + res.writeHead(200, { "Content-Type": "application/octet-stream" }); + res.end(Buffer.from([0, 1, 2, 3, 4, 5])); + return; + } + if (route === "/big") { + res.writeHead(200, { "Content-Type": "text/plain" }); + // Send 3 MB in 64k chunks — should overflow the 2 MB raw cap. + let sent = 0; + const chunk = Buffer.alloc(64 * 1024, 0x78); // 'x' + const tick = () => { + if (sent >= 3 * 1024 * 1024) { + res.end(); + return; + } + sent += chunk.length; + if (res.write(chunk)) { + setImmediate(tick); + } else { + res.once("drain", tick); + } + }; + tick(); + return; + } + if (route === "/slow") { + // Send headers, then hang. + res.writeHead(200, { "Content-Type": "text/plain" }); + res.write("starting...\n"); + // Don't end; just hold. The deadline in the tool should fire. + return; + } + if (route === "/redirect-once") { + res.writeHead(302, { Location: "/text" }); + res.end(); + return; + } + if (route === "/redirect-loop") { + // Each hop bumps a counter; loops indefinitely. + const n = parseInt(url.searchParams.get("n") ?? "0", 10) || 0; + res.writeHead(302, { Location: `/redirect-loop?n=${n + 1}` }); + res.end(); + return; + } + if (route === "/redirect-to-private") { + // Send the model to 10.0.0.1 — should be blocked even though our + // fixture itself is on loopback. + res.writeHead(302, { Location: "http://10.0.0.1/" }); + res.end(); + return; + } + if (route === "/redirect-to-https") { + res.writeHead(302, { Location: "https://example.com/" }); + res.end(); + return; + } + if (route === "/redirect-to-file") { + res.writeHead(302, { Location: "file:///etc/passwd" }); + res.end(); + return; + } + res.writeHead(404, { "Content-Type": "text/plain" }); + res.end("not found"); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve()); + }); + const addr = server.address() as AddressInfo; + port = addr.port; + baseUrl = `http://127.0.0.1:${port}`; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); +}); + +function ctxAllow(extra: Record = {}): ToolExecutionContext { + return { + cwd: process.cwd(), + toolOpts: { webfetch: { allowPrivate: true, ...extra } as JsonObject }, + }; +} +function ctxStrict(extra: Record = {}): ToolExecutionContext { + return { + cwd: process.cwd(), + toolOpts: { webfetch: { allowPrivate: false, ...extra } as JsonObject }, + }; +} + +// ─── Happy path (allowPrivate so we can hit the local fixture) ─────── + +describe("webfetch — happy path", () => { + it("fetches a small text response", async () => { + const r = await webfetchTool({ url: `${baseUrl}/text` }, ctxAllow()); + expect(r.ok).toBe(true); + expect(r.content).toContain("hello, fixture"); + expect(r.audit.http_status).toBe(200); + expect(r.audit.truncated).toBe(false); + expect(r.audit.hops).toBe(0); + }); + + it("fetches JSON", async () => { + const r = await webfetchTool({ url: `${baseUrl}/json` }, ctxAllow()); + expect(r.ok).toBe(true); + expect(JSON.parse(r.content).n).toBe(42); + expect(r.audit.content_type).toMatch(/json/); + }); + + it("fetches HTML", async () => { + const r = await webfetchTool({ url: `${baseUrl}/html` }, ctxAllow()); + expect(r.ok).toBe(true); + expect(r.content).toContain("

fix

"); + }); +}); + +// ─── Scheme + content-type gates ───────────────────────────────────── + +describe("webfetch — scheme rejection", () => { + it("rejects file://", async () => { + const r = await webfetchTool({ url: "file:///etc/passwd" }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("disallowed_scheme"); + }); + + it("rejects ftp://", async () => { + const r = await webfetchTool({ url: "ftp://example.com/" }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("disallowed_scheme"); + }); + + it("rejects gopher://", async () => { + const r = await webfetchTool({ url: "gopher://example.com/" }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("disallowed_scheme"); + }); +}); + +describe("webfetch — content-type gating", () => { + it("rejects application/octet-stream", async () => { + const r = await webfetchTool({ url: `${baseUrl}/binary` }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("non_text_content"); + }); +}); + +// ─── Network controls (private/IP blocks) ──────────────────────────── + +describe("webfetch — IP literal / DNS blocks (allowPrivate=false default)", () => { + it("blocks http://127.0.0.1/", async () => { + const r = await webfetchTool({ url: `http://127.0.0.1:${port}/text` }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_loopback"); + }); + + it("blocks http://localhost/ via DNS pre-resolve", async () => { + const r = await webfetchTool({ url: `http://localhost:${port}/text` }, ctxStrict()); + expect(r.ok).toBe(false); + // localhost may resolve to 127.0.0.1 (loopback) or ::1 (also loopback). + expect(r.audit.error).toBe("blocked_loopback"); + }); + + it("blocks http://10.0.0.1/ (RFC1918)", async () => { + const r = await webfetchTool({ url: "http://10.0.0.1/" }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_private"); + }); + + it("blocks http://192.168.1.1/", async () => { + const r = await webfetchTool({ url: "http://192.168.1.1/" }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_private"); + }); + + it("blocks http://169.254.169.254/ (link-local, AWS metadata endpoint)", async () => { + const r = await webfetchTool({ url: "http://169.254.169.254/latest/meta-data/" }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_link_local"); + }); + + it("blocks http://100.64.0.1/ (CGNAT)", async () => { + const r = await webfetchTool({ url: "http://100.64.0.1/" }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_cgnat"); + }); + + it("blocks http://[::1]/", async () => { + const r = await webfetchTool({ url: `http://[::1]:${port}/text` }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_loopback"); + }); + + it("blocks http://[fe80::1]/", async () => { + const r = await webfetchTool({ url: "http://[fe80::1]/" }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_link_local"); + }); + + it("blocks an IPv4-mapped IPv6 loopback", async () => { + const r = await webfetchTool({ url: "http://[::ffff:127.0.0.1]/" }, ctxStrict()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("blocked_loopback"); + }); +}); + +// ─── Redirects ─────────────────────────────────────────────────────── + +describe("webfetch — redirects", () => { + it("follows a single safe redirect within allowPrivate scope", async () => { + const r = await webfetchTool({ url: `${baseUrl}/redirect-once` }, ctxAllow()); + expect(r.ok).toBe(true); + expect(r.content).toContain("hello, fixture"); + expect(r.audit.hops).toBe(1); + const chain = r.audit.redirect_chain as string[]; + expect(chain).toHaveLength(2); + }); + + it("captures both URLs in the redirect chain when redirecting to a private target", async () => { + // With allowPrivate=true throughout, the next-hop pre-resolve passes + // (10.0.0.1 is a literal IP) but the actual TCP dial either fails + // fast or hits the timeout. We just verify the chain captured both + // URLs and the audit error is a connection/timeout-class code + // (not silently absorbed). + const r = await webfetchTool( + { url: `${baseUrl}/redirect-to-private` }, + ctxAllow({ perCallTimeoutMs: 400 }), + ); + expect(r.ok).toBe(false); + const chain = r.audit.redirect_chain as string[]; + expect(chain.length).toBeGreaterThanOrEqual(2); + expect(chain[1]).toMatch(/10\.0\.0\.1/); + expect( + ["timeout", "EHOSTUNREACH", "ENETUNREACH", "ECONNREFUSED", "EADDRNOTAVAIL", "ETIMEDOUT"], + ).toContain(r.audit.error); + }, 3_000); + + it("refuses a redirect that changes scheme (http → https)", async () => { + const r = await webfetchTool({ url: `${baseUrl}/redirect-to-https` }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("scheme_change_redirect"); + }); + + it("refuses a redirect to a disallowed scheme (file://)", async () => { + const r = await webfetchTool({ url: `${baseUrl}/redirect-to-file` }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("scheme_change_redirect"); + }); + + it("exhausts the redirect limit at 5 with a loop", async () => { + const r = await webfetchTool({ url: `${baseUrl}/redirect-loop` }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("redirect_limit"); + expect(r.audit.hops).toBe(6); // we attempted one beyond the limit + }); +}); + +// ─── Byte cap stops reading (no full-body buffering) ───────────────── + +describe("webfetch — byte cap aborts the request", () => { + it("caps the body at maxRawBytes, audit.truncated=true", async () => { + const r = await webfetchTool( + { url: `${baseUrl}/big` }, + ctxAllow({ maxRawBytes: 256 * 1024, maxChars: 64_000 }), + ); + expect(r.ok).toBe(true); + expect(r.audit.truncated).toBe(true); + expect(r.audit.original_bytes as number).toBeGreaterThanOrEqual(256 * 1024); + // model-output truncation also kicks in + expect(r.content.length).toBeLessThan(70_000); + expect(r.content).toMatch(/truncated/); + }); +}); + +// ─── Total timeout ─────────────────────────────────────────────────── + +describe("webfetch — total timeout", () => { + it("aborts when the server holds the connection past the deadline", async () => { + const r = await webfetchTool( + { url: `${baseUrl}/slow` }, + ctxAllow({ perCallTimeoutMs: 300 }), + ); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("timeout"); + }, 5_000); +}); + +// ─── Bad args ──────────────────────────────────────────────────────── + +describe("webfetch — bad inputs", () => { + it("rejects missing url", async () => { + const r = await webfetchTool({} as JsonObject, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); + + it("rejects a malformed URL string", async () => { + const r = await webfetchTool({ url: "http://" }, ctxAllow()); + expect(r.ok).toBe(false); + // Either bad_url upstream or a downstream parse failure + expect(["bad_url", "disallowed_scheme"]).toContain(r.audit.error); + }); + + it("rejects a non-string url", async () => { + const r = await webfetchTool({ url: 42 as unknown as string }, ctxAllow()); + expect(r.ok).toBe(false); + expect(r.audit.error).toBe("bad_args"); + }); +}); diff --git a/packages/jimmy/src/engines/tools/cwdJail.ts b/packages/jimmy/src/engines/tools/cwdJail.ts new file mode 100644 index 00000000..21ff7c92 --- /dev/null +++ b/packages/jimmy/src/engines/tools/cwdJail.ts @@ -0,0 +1,119 @@ +/** + * Cwd jail — bound filesystem access for the read/write/edit tools. + * + * Two-stage check on every resolve: + * + * 1. Lexical: path.resolve(cwd, requested) must not escape cwd via ".." + * or absolute paths to elsewhere. + * 2. Realpath: after walking up to the deepest existing ancestor and + * resolving symlinks via fs.realpath, the canonical path must still + * be under realpath(cwd). This catches symlink escapes both at the + * leaf and at any parent directory. + * + * Tools can additionally request `rejectSymlinkLeaf: true` (write/edit) to + * refuse operating when the final path component exists and is itself a + * symbolic link, even if its target lies inside the jail. This keeps + * write/edit semantics straightforward: "you are modifying the file at + * this exact path, no indirection." + * + * The lexical helper is intentionally NOT exported so future code cannot + * accidentally pick the unsafe-on-its-own variant. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; + +export type JailReason = "lexical_escape" | "realpath_escape" | "symlink_leaf"; + +export class JailViolation extends Error { + constructor( + public readonly requestedPath: string, + public readonly cwd: string, + public readonly reason: JailReason, + ) { + super(`path "${requestedPath}" violates jail "${cwd}" (${reason})`); + this.name = "JailViolation"; + } +} + +interface ResolveOpts { + /** When true and the leaf exists as a symlink, reject (write/edit). */ + rejectSymlinkLeaf?: boolean; +} + +/** + * Resolve `requested` against `cwd` to a canonical absolute path under + * the jail, or throw. Performs both lexical and realpath checks. + */ +export async function resolveInJail( + cwd: string, + requested: string, + opts: ResolveOpts = {}, +): Promise { + const lexResolved = lexicalResolve(cwd, requested); + const realCwd = await fs.realpath(cwd); + + // Walk up the lexical path to find the deepest existing ancestor. + // Accumulate trailing segments (those that don't yet exist on disk). + const trailing: string[] = []; + let ancestor = lexResolved; + while (true) { + let stat; + try { + stat = await fs.lstat(ancestor); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err; + // Doesn't exist — climb up. + const parent = path.dirname(ancestor); + if (parent === ancestor) { + // Walked to root without finding an existing ancestor. Shouldn't + // happen if cwd exists; treat as a realpath escape defensively. + throw new JailViolation(requested, cwd, "realpath_escape"); + } + trailing.unshift(path.basename(ancestor)); + ancestor = parent; + continue; + } + + // Leaf-symlink check: if we're at the lexResolved leaf itself, is it + // a symlink? Only enforced when caller asks (write/edit). + if (opts.rejectSymlinkLeaf && ancestor === lexResolved && stat.isSymbolicLink()) { + throw new JailViolation(requested, cwd, "symlink_leaf"); + } + break; + } + + // Canonicalize the existing ancestor (resolves any symlinks along the + // path). Re-attach any trailing segments that don't yet exist. + const realAncestor = await fs.realpath(ancestor); + const canonical = + trailing.length === 0 ? realAncestor : path.join(realAncestor, ...trailing); + + // Final jail check against the realpathed cwd. + const rel = path.relative(realCwd, canonical); + if (rel.startsWith("..") || path.isAbsolute(rel)) { + throw new JailViolation(requested, cwd, "realpath_escape"); + } + + return canonical; +} + +/** Lexical-only resolution. Private — exported helpers run realpath too. */ +function lexicalResolve(cwd: string, requested: string): string { + if (typeof requested !== "string") { + throw new Error(`path must be a string, got ${typeof requested}`); + } + if (requested.length === 0) { + throw new Error("path must be a non-empty string"); + } + if (requested.includes("\0")) { + throw new Error("path must not contain NUL bytes"); + } + const baseAbs = path.resolve(cwd); + const resolved = path.resolve(baseAbs, requested); + const rel = path.relative(baseAbs, resolved); + if (rel.startsWith("..") || path.isAbsolute(rel)) { + throw new JailViolation(requested, cwd, "lexical_escape"); + } + return resolved; +} diff --git a/packages/jimmy/src/engines/tools/edit.ts b/packages/jimmy/src/engines/tools/edit.ts new file mode 100644 index 00000000..e316c276 --- /dev/null +++ b/packages/jimmy/src/engines/tools/edit.ts @@ -0,0 +1,170 @@ +/** + * `edit` tool — exact string replacement in a file under the cwd jail. + * + * Args: + * path: string (must resolve under ctx.cwd) + * old_string: string (must already exist in the file) + * new_string: string (replacement; may be empty to delete) + * replace_all: optional boolean. Default false. + * + * Behavior: + * - If `old_string` is not found → error + * - If `old_string === new_string` → error (no-op) + * - If multiple matches and `replace_all=false` → error (refuses to + * guess which match the model meant) + * - If `replace_all=true` → replaces every occurrence + * - Otherwise replaces the single match + * + * Mirrors the semantics of Claude Code's Edit tool so prompts written for + * Claude can be reused. + */ + +import fs from "node:fs/promises"; +import type { JsonObject } from "../../shared/types.js"; +import { JailViolation, resolveInJail } from "./cwdJail.js"; +import type { ToolExecutionContext, ToolResult } from "./types.js"; + +const MAX_FILE_BYTES = 5 * 1024 * 1024; + +interface EditArgs { + path: string; + old_string: string; + new_string: string; + replace_all: boolean; +} + +function parseArgs(raw: JsonObject): { ok: true; args: EditArgs } | { ok: false; reason: string } { + if (typeof raw.path !== "string" || raw.path.length === 0) { + return { ok: false, reason: "edit: 'path' is required and must be a non-empty string" }; + } + if (typeof raw.old_string !== "string") { + return { ok: false, reason: "edit: 'old_string' is required and must be a string" }; + } + if (raw.old_string.length === 0) { + return { ok: false, reason: "edit: 'old_string' must be non-empty" }; + } + if (typeof raw.new_string !== "string") { + return { ok: false, reason: "edit: 'new_string' is required and must be a string" }; + } + let replace_all = false; + if (raw.replace_all !== undefined) { + if (typeof raw.replace_all !== "boolean") { + return { ok: false, reason: "edit: 'replace_all' must be a boolean" }; + } + replace_all = raw.replace_all; + } + return { ok: true, args: { path: raw.path, old_string: raw.old_string, new_string: raw.new_string, replace_all } }; +} + +export async function editTool(raw: JsonObject, ctx: ToolExecutionContext): Promise { + const parsed = parseArgs(raw); + if (!parsed.ok) { + return { ok: false, content: parsed.reason, audit: { truncated: false, error: "bad_args" } }; + } + const { path: requestedPath, old_string, new_string, replace_all } = parsed.args; + + if (old_string === new_string) { + return { + ok: false, + content: "edit: 'old_string' and 'new_string' are identical — no-op refused", + audit: { truncated: false, error: "noop" }, + }; + } + + let abs: string; + try { + abs = await resolveInJail(ctx.cwd, requestedPath, { rejectSymlinkLeaf: true }); + } catch (err) { + return { + ok: false, + content: `edit: ${(err as Error).message}`, + audit: { truncated: false, error: err instanceof JailViolation ? err.reason : "bad_path" }, + }; + } + + // Size cap before readFile. + try { + const st = await fs.stat(abs); + if (st.size > MAX_FILE_BYTES) { + return { + ok: false, + content: `edit: "${requestedPath}" is ${st.size} bytes; max is ${MAX_FILE_BYTES} (${Math.floor(MAX_FILE_BYTES / 1024 / 1024)}MB)`, + audit: { truncated: false, error: "too_large", file_bytes: st.size }, + }; + } + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + content: `edit: cannot stat "${requestedPath}" (${code})`, + audit: { truncated: false, error: code }, + }; + } + + let content: string; + try { + content = await fs.readFile(abs, "utf8"); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + content: `edit: cannot read "${requestedPath}" (${code})`, + audit: { truncated: false, error: code }, + }; + } + + const occurrences = countOccurrences(content, old_string); + if (occurrences === 0) { + return { + ok: false, + content: `edit: 'old_string' not found in "${requestedPath}"`, + audit: { truncated: false, error: "not_found" }, + }; + } + if (occurrences > 1 && !replace_all) { + return { + ok: false, + content: + `edit: 'old_string' matches ${occurrences} locations in "${requestedPath}"; ` + + `provide more context to make it unique, or set replace_all=true`, + audit: { truncated: false, error: "ambiguous", matches: occurrences }, + }; + } + + const updated = replace_all + ? content.split(old_string).join(new_string) + : content.replace(old_string, new_string); + + try { + await fs.writeFile(abs, updated, "utf8"); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + content: `edit: cannot write "${requestedPath}" (${code})`, + audit: { truncated: false, error: code }, + }; + } + + return { + ok: true, + content: replace_all + ? `edited ${requestedPath} (replaced all ${occurrences} occurrences)` + : `edited ${requestedPath} (1 replacement)`, + audit: { + truncated: false, + replacements: replace_all ? occurrences : 1, + }, + }; +} + +function countOccurrences(haystack: string, needle: string): number { + if (needle.length === 0) return 0; + let count = 0; + let idx = 0; + while ((idx = haystack.indexOf(needle, idx)) !== -1) { + count++; + idx += needle.length; + } + return count; +} diff --git a/packages/jimmy/src/engines/tools/index.ts b/packages/jimmy/src/engines/tools/index.ts new file mode 100644 index 00000000..dc80ba28 --- /dev/null +++ b/packages/jimmy/src/engines/tools/index.ts @@ -0,0 +1,75 @@ +/** + * Tool registry — maps tool names to executors, filtered per engine. + * + * The agent loop never sees a tool the engine hasn't enabled. An engine + * with `tools.enabled: []` (or no tools block at all) operates in + * text-only mode and the schemas array is empty. + */ + +import type { EngineToolsConfig, JsonObject } from "../../shared/types.js"; +import type { ProviderToolDef } from "../providers/types.js"; +import { readTool } from "./read.js"; +import { writeTool } from "./write.js"; +import { editTool } from "./edit.js"; +import { runCommandTool } from "./runCommand.js"; +import { webfetchTool } from "./webfetch.js"; +import { ALL_SCHEMAS } from "./schemas.js"; +import type { ToolExecutionContext, ToolResult } from "./types.js"; + +export type ToolExecutor = ( + args: JsonObject, + ctx: ToolExecutionContext, +) => Promise; + +interface InternalEntry { + executor: ToolExecutor; + schema: ProviderToolDef; +} + +const ALL_TOOLS: Record = { + read: { executor: readTool, schema: ALL_SCHEMAS.read }, + write: { executor: writeTool, schema: ALL_SCHEMAS.write }, + edit: { executor: editTool, schema: ALL_SCHEMAS.edit }, + bash: { executor: runCommandTool, schema: ALL_SCHEMAS.bash }, + webfetch: { executor: webfetchTool, schema: ALL_SCHEMAS.webfetch }, +}; + +export interface ToolRegistry { + /** Tool name → executor. Empty in text-only configs. */ + executors: Map; + /** Schemas matching `executors`, in declaration order from config.enabled. */ + schemas: ProviderToolDef[]; + /** Names that were requested in config but don't correspond to a known tool. */ + unknownRequested: string[]; +} + +const KNOWN_TOOL_NAMES = Object.freeze(Object.keys(ALL_TOOLS)); + +/** + * Build a tool registry for one engine instance from its config. + * + * - `undefined` or missing `enabled` → text-only mode (empty registry). + * - Unknown names are reported via `unknownRequested` but do not throw — + * the engine wrapper logs a warning at construction time. This lets + * forward-compat configs name tools that don't exist yet without + * breaking the gateway. + */ +export function buildToolRegistry(toolsConfig?: EngineToolsConfig): ToolRegistry { + const enabled = toolsConfig?.enabled ?? []; + const executors = new Map(); + const schemas: ProviderToolDef[] = []; + const unknownRequested: string[] = []; + for (const name of enabled) { + const entry = ALL_TOOLS[name]; + if (!entry) { + unknownRequested.push(name); + continue; + } + if (executors.has(name)) continue; // dedupe + executors.set(name, entry.executor); + schemas.push(entry.schema); + } + return { executors, schemas, unknownRequested }; +} + +export { KNOWN_TOOL_NAMES }; diff --git a/packages/jimmy/src/engines/tools/ipBlocklist.ts b/packages/jimmy/src/engines/tools/ipBlocklist.ts new file mode 100644 index 00000000..e5008498 --- /dev/null +++ b/packages/jimmy/src/engines/tools/ipBlocklist.ts @@ -0,0 +1,168 @@ +/** + * IP-address blocklist for the webfetch tool. + * + * Pure functions, no I/O — exported so the agent loop, tests, and the + * webfetch tool itself can share one canonical "is this address safe to + * connect to" decision. Reason codes flow through to ToolResult.audit.error. + */ + +import net from "node:net"; + +export type BlockReason = + | "blocked_unspecified" + | "blocked_loopback" + | "blocked_private" + | "blocked_cgnat" + | "blocked_link_local" + | "blocked_unique_local" + | "blocked_multicast" + | "blocked_broadcast"; + +interface IPv4Range { + base: number; + mask: number; + reason: BlockReason; +} + +function ipv4ToInt(addr: string): number { + const parts = addr.split("."); + if (parts.length !== 4) return NaN; + let n = 0; + for (const p of parts) { + const v = Number(p); + if (!Number.isInteger(v) || v < 0 || v > 255) return NaN; + n = (n << 8) | v; + } + return n >>> 0; +} + +const IPV4_RANGES: IPv4Range[] = [ + { base: 0x00000000, mask: 0xff000000, reason: "blocked_unspecified" }, // 0.0.0.0/8 + { base: 0x0a000000, mask: 0xff000000, reason: "blocked_private" }, // 10.0.0.0/8 + { base: 0x64400000, mask: 0xffc00000, reason: "blocked_cgnat" }, // 100.64.0.0/10 + { base: 0x7f000000, mask: 0xff000000, reason: "blocked_loopback" }, // 127.0.0.0/8 + { base: 0xa9fe0000, mask: 0xffff0000, reason: "blocked_link_local" }, // 169.254.0.0/16 + { base: 0xac100000, mask: 0xfff00000, reason: "blocked_private" }, // 172.16.0.0/12 + { base: 0xc0a80000, mask: 0xffff0000, reason: "blocked_private" }, // 192.168.0.0/16 + { base: 0xe0000000, mask: 0xf0000000, reason: "blocked_multicast" }, // 224.0.0.0/4 + { base: 0xffffffff, mask: 0xffffffff, reason: "blocked_broadcast" }, // 255.255.255.255/32 +]; + +export function checkIPv4(addr: string): BlockReason | null { + if (!net.isIPv4(addr)) return null; + const n = ipv4ToInt(addr); + if (Number.isNaN(n)) return null; + for (const r of IPV4_RANGES) { + // `>>> 0` re-unsigns the AND result — JS bitwise ops produce signed + // int32, and ranges like 0xff000000 round-trip as negative without this. + if (((n & r.mask) >>> 0) === r.base) return r.reason; + } + return null; +} + +/** + * Expand an IPv6 address (which may include "::" compression) into its 8 + * 16-bit hextet integers. Returns null if the address is not parseable. + */ +function parseIPv6(addr: string): number[] | null { + if (!net.isIPv6(addr)) return null; + + // Strip a possible zone-id (e.g. "fe80::1%eth0"); the canonical address is + // everything before "%". + const at = addr.indexOf("%"); + const naked = at === -1 ? addr : addr.slice(0, at); + + // Handle IPv4-embedded suffix (e.g. ::ffff:1.2.3.4) — convert that v4 + // portion into two hextets. + const lastColon = naked.lastIndexOf(":"); + let head = naked; + let tail4: number[] | null = null; + if (lastColon !== -1) { + const maybeV4 = naked.slice(lastColon + 1); + if (net.isIPv4(maybeV4)) { + const n = ipv4ToInt(maybeV4); + if (!Number.isNaN(n)) { + tail4 = [(n >>> 16) & 0xffff, n & 0xffff]; + head = naked.slice(0, lastColon); + } + } + } + + const hextets: number[] = []; + const doubleColon = head.indexOf("::"); + if (doubleColon === -1) { + const segs = head.split(":"); + for (const s of segs) hextets.push(parseInt(s, 16) | 0); + } else { + const left = head.slice(0, doubleColon); + const right = head.slice(doubleColon + 2); + const leftSegs = left === "" ? [] : left.split(":"); + const rightSegs = right === "" ? [] : right.split(":"); + const totalNeeded = (tail4 ? 6 : 8) - leftSegs.length - rightSegs.length; + for (const s of leftSegs) hextets.push(parseInt(s, 16) | 0); + for (let i = 0; i < totalNeeded; i++) hextets.push(0); + for (const s of rightSegs) hextets.push(parseInt(s, 16) | 0); + } + if (tail4) hextets.push(...tail4); + if (hextets.length !== 8) return null; + for (const h of hextets) { + if (h < 0 || h > 0xffff || Number.isNaN(h)) return null; + } + return hextets; +} + +export function checkIPv6(addr: string): BlockReason | null { + const hextets = parseIPv6(addr); + if (!hextets) return null; + const allZero = hextets.every((h) => h === 0); + if (allZero) return "blocked_unspecified"; // :: + if ( + hextets[0] === 0 && hextets[1] === 0 && hextets[2] === 0 && + hextets[3] === 0 && hextets[4] === 0 && hextets[5] === 0 && + hextets[6] === 0 && hextets[7] === 1 + ) { + return "blocked_loopback"; // ::1 + } + + // ::ffff:0:0/96 — IPv4-mapped IPv6. Validate the embedded v4. + if ( + hextets[0] === 0 && hextets[1] === 0 && hextets[2] === 0 && + hextets[3] === 0 && hextets[4] === 0 && hextets[5] === 0xffff + ) { + const v4 = `${(hextets[6]! >>> 8) & 0xff}.${hextets[6]! & 0xff}.${(hextets[7]! >>> 8) & 0xff}.${hextets[7]! & 0xff}`; + const reason = checkIPv4(v4); + if (reason) return reason; + return null; + } + + // fe80::/10 — first 10 bits = 1111111010 + if ((hextets[0]! & 0xffc0) === 0xfe80) return "blocked_link_local"; + // fc00::/7 — first 7 bits = 1111110 + if ((hextets[0]! & 0xfe00) === 0xfc00) return "blocked_unique_local"; + // ff00::/8 + if ((hextets[0]! & 0xff00) === 0xff00) return "blocked_multicast"; + + return null; +} + +/** Convenience: dispatch by family. */ +export function checkAddress(family: number, addr: string): BlockReason | null { + if (family === 4) return checkIPv4(addr); + if (family === 6) return checkIPv6(addr); + return null; +} + +/** + * Return the (family, address) tuple if `host` is an IP literal — including + * bracketed IPv6 notation as seen in URLs. Returns null for hostnames. + */ +export function parseIpLiteral(host: string): { family: 4 | 6; address: string } | null { + if (!host || typeof host !== "string") return null; + if (net.isIPv4(host)) return { family: 4, address: host }; + let bare = host; + if (host.startsWith("[") && host.endsWith("]")) { + bare = host.slice(1, -1); + } + if (net.isIPv6(bare)) return { family: 6, address: bare }; + return null; +} diff --git a/packages/jimmy/src/engines/tools/read.ts b/packages/jimmy/src/engines/tools/read.ts new file mode 100644 index 00000000..22575085 --- /dev/null +++ b/packages/jimmy/src/engines/tools/read.ts @@ -0,0 +1,138 @@ +/** + * `read` tool — read a text file under the cwd jail. + * + * Args: + * path: string (relative or absolute, must resolve under ctx.cwd) + * offset: optional 1-indexed line number to start from. Default 1. + * limit: optional max line count to return. Default 2000. + * + * Truncation: + * The slice (after offset/limit) is further capped at `maxChars` chars + * (default 64_000). If truncated, the returned content ends with + * `\n[truncated: NN of MM total characters]\n`. + * + * Returns a ToolResult. Failures (file not found, jail violation, bad arg + * shape) return `{ok:false, ...}` rather than throwing so the agent loop + * can feed the error to the model. + */ + +import fs from "node:fs/promises"; +import type { JsonObject } from "../../shared/types.js"; +import { JailViolation, resolveInJail } from "./cwdJail.js"; +import type { ToolExecutionContext, ToolResult } from "./types.js"; + +const DEFAULT_LINE_LIMIT = 2000; +const DEFAULT_MAX_CHARS = 64_000; +const MAX_FILE_BYTES = 5 * 1024 * 1024; + +interface ReadArgs { + path: string; + offset: number; + limit: number; +} + +function parseArgs(raw: JsonObject): { ok: true; args: ReadArgs } | { ok: false; reason: string } { + if (typeof raw.path !== "string" || raw.path.length === 0) { + return { ok: false, reason: "read: 'path' is required and must be a non-empty string" }; + } + let offset = 1; + if (raw.offset !== undefined) { + if (typeof raw.offset !== "number" || !Number.isInteger(raw.offset) || raw.offset < 1) { + return { ok: false, reason: "read: 'offset' must be a positive integer (1-indexed line number)" }; + } + offset = raw.offset; + } + let limit = DEFAULT_LINE_LIMIT; + if (raw.limit !== undefined) { + if (typeof raw.limit !== "number" || !Number.isInteger(raw.limit) || raw.limit < 1) { + return { ok: false, reason: "read: 'limit' must be a positive integer" }; + } + limit = raw.limit; + } + return { ok: true, args: { path: raw.path, offset, limit } }; +} + +export async function readTool(raw: JsonObject, ctx: ToolExecutionContext): Promise { + const parsed = parseArgs(raw); + if (!parsed.ok) { + return { ok: false, content: parsed.reason, audit: { truncated: false, error: "bad_args" } }; + } + const { path: requestedPath, offset, limit } = parsed.args; + + let abs: string; + try { + abs = await resolveInJail(ctx.cwd, requestedPath); + } catch (err) { + const message = (err as Error).message; + return { + ok: false, + content: `read: ${message}`, + audit: { truncated: false, error: err instanceof JailViolation ? err.reason : "bad_path" }, + }; + } + + // Size cap: stat before readFile so huge files don't blow up memory. + try { + const st = await fs.stat(abs); + if (st.size > MAX_FILE_BYTES) { + return { + ok: false, + content: `read: "${requestedPath}" is ${st.size} bytes; max is ${MAX_FILE_BYTES} (${Math.floor(MAX_FILE_BYTES / 1024 / 1024)}MB)`, + audit: { truncated: false, error: "too_large", file_bytes: st.size }, + }; + } + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + content: `read: cannot stat "${requestedPath}" (${code})`, + audit: { truncated: false, error: code }, + }; + } + + let raw_content: string; + try { + raw_content = await fs.readFile(abs, "utf8"); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + content: `read: cannot read "${requestedPath}" (${code})`, + audit: { truncated: false, error: code }, + }; + } + + // Line slicing (1-indexed offset like cat -n). + const lines = raw_content.split("\n"); + const totalLines = lines.length; + const startIdx = Math.min(offset - 1, totalLines); + const endIdx = Math.min(startIdx + limit, totalLines); + const sliced = lines.slice(startIdx, endIdx).join("\n"); + + const maxChars = readMaxChars(ctx); + let content = sliced; + let truncated = false; + if (content.length > maxChars) { + truncated = true; + content = content.slice(0, maxChars) + `\n[truncated: ${maxChars} of ${sliced.length} characters]\n`; + } + + return { + ok: true, + content, + audit: { + truncated, + originalBytes: sliced.length, + total_lines: totalLines, + returned_lines: endIdx - startIdx, + }, + }; +} + +function readMaxChars(ctx: ToolExecutionContext): number { + const override = ctx.toolOpts?.read; + if (override && typeof override.maxChars === "number" && override.maxChars > 0) { + return override.maxChars; + } + return DEFAULT_MAX_CHARS; +} diff --git a/packages/jimmy/src/engines/tools/runCommand.ts b/packages/jimmy/src/engines/tools/runCommand.ts new file mode 100644 index 00000000..cfc97393 --- /dev/null +++ b/packages/jimmy/src/engines/tools/runCommand.ts @@ -0,0 +1,400 @@ +/** + * `runCommand` tool — argv-only command execution under the cwd jail. + * + * Hardening posture (V1): + * - Spawns via node:child_process spawn() with shell:false, so the OS + * never sees a shell expansion of the model's input. + * - The model's argv is rejected upfront if any element contains a + * shell-metacharacter or NUL byte (belt-and-suspenders even though + * shell:false doesn't expand them). + * - An allowlist of argv[0] basenames must be configured on the engine. + * Missing or empty allowlist → tool is disabled with a clear error. + * - A hardcoded NEVER-LIST overrides the allowlist: shell executables + * (sh/bash/zsh/fish/...) and shell-like bypasses (env/xargs/eval/exec) + * are refused even if the user adds them to bashAllowlist by mistake. + * - python3 has extra restrictions: no -c / -m / stdin / interactive, + * and the first positional arg must be an existing file under cwd. + * + * Truncation: + * - stdout capped at 32 KB by default, stderr at 16 KB. Independently + * tracked; audit row has `truncated_stdout`, `truncated_stderr`, + * `original_stdout_bytes`, `original_stderr_bytes`. + * - Top-level `audit.truncated` is the OR of the two. + * + * Timeout / kill: + * - Per-call wall-clock timeout (default 60s, configurable). On hit: + * SIGTERM, wait `killGraceMs` (default 3s), then SIGKILL. + * - `audit.timeout: true` indicates the process was killed by us. + * - `audit.signal: "SIGTERM" | "SIGKILL" | null` carries the signal + * the OS reported back. + */ + +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { JsonObject, JsonValue } from "../../shared/types.js"; +import { JailViolation, resolveInJail } from "./cwdJail.js"; +import type { ToolExecutionContext, ToolResult } from "./types.js"; + +const DEFAULT_MAX_STDOUT = 32_000; +const DEFAULT_MAX_STDERR = 16_000; +const DEFAULT_TIMEOUT_MS = 60_000; +const DEFAULT_KILL_GRACE_MS = 3_000; + +/** + * Characters that have meaning in a shell context. argv-only execution + * doesn't expand them, but we reject upfront so a misconfigured downstream + * (re-shelling, copy/paste of an argv into a shell command line, future + * code change) can't turn a benign-looking command into a code execution + * surface. + */ +const METACHARS_RE = /[;|&`$<>\n\r\t*?~(){}\\\[\]!]/; + +/** argv[0] basenames refused regardless of allowlist. Lowercase. */ +const NEVER_LIST = new Set([ + "sh", "bash", "zsh", "fish", "ksh", "csh", "tcsh", "dash", "ash", + "env", "xargs", "eval", "exec", "source", +]); + +/** + * Python flags that bypass the "must run a script file under cwd" intent. + * Reject if present in args. + */ +const PYTHON_BANNED_FLAGS = new Set([ + "-c", "--command", "-m", "--module", "-i", "--interactive", "-", +]); + +interface RunCommandArgs { + command: string; + args: string[]; +} + +interface BashOpts { + allowlist: string[]; + maxStdout: number; + maxStderr: number; + perCallTimeoutMs: number; + killGraceMs: number; +} + +function readBashOpts(ctx: ToolExecutionContext): BashOpts { + const raw = (ctx.toolOpts?.bash ?? {}) as Record; + const rawAllowlist = raw.allowlist; + const allowlist = + Array.isArray(rawAllowlist) && rawAllowlist.every((x) => typeof x === "string") + ? (rawAllowlist as string[]) + : []; + return { + allowlist, + maxStdout: typeof raw.maxStdout === "number" ? raw.maxStdout : DEFAULT_MAX_STDOUT, + maxStderr: typeof raw.maxStderr === "number" ? raw.maxStderr : DEFAULT_MAX_STDERR, + perCallTimeoutMs: + typeof raw.perCallTimeoutMs === "number" ? raw.perCallTimeoutMs : DEFAULT_TIMEOUT_MS, + killGraceMs: typeof raw.killGraceMs === "number" ? raw.killGraceMs : DEFAULT_KILL_GRACE_MS, + }; +} + +function parseArgs(raw: JsonObject): { ok: true; args: RunCommandArgs } | { ok: false; reason: string } { + if (typeof raw.command !== "string" || raw.command.length === 0) { + return { ok: false, reason: "bash: 'command' is required and must be a non-empty string" }; + } + if (raw.args === undefined) { + return { ok: true, args: { command: raw.command, args: [] } }; + } + if (!Array.isArray(raw.args)) { + return { ok: false, reason: "bash: 'args' must be an array of strings" }; + } + for (let i = 0; i < raw.args.length; i++) { + if (typeof raw.args[i] !== "string") { + return { ok: false, reason: `bash: args[${i}] must be a string` }; + } + } + return { ok: true, args: { command: raw.command, args: raw.args as string[] } }; +} + +function metacharCheck(value: string, label: string): string | null { + if (value.includes("\0")) return `${label} contains NUL byte`; + const m = METACHARS_RE.exec(value); + if (m) return `${label} contains shell metacharacter ${JSON.stringify(m[0])}`; + return null; +} + +/** Truncate a single arg to keep the audit row bounded. */ +function summarizeArg(s: string): string { + if (s.length <= 200) return s; + return s.slice(0, 200) + `…[${s.length - 200} more chars]`; +} + +async function validatePython(args: string[], ctx: ToolExecutionContext): Promise<{ ok: true } | { ok: false; reason: string; code: string }> { + for (const a of args) { + if (PYTHON_BANNED_FLAGS.has(a)) { + return { + ok: false, + reason: `bash: python3 invocation must not use ${a} (no inline code or stdin execution)`, + code: "python3_unsafe_args", + }; + } + } + const positional = args.find((a) => !a.startsWith("-")); + if (!positional) { + return { + ok: false, + reason: "bash: python3 invocation must include a script path as a positional argument", + code: "python3_no_script", + }; + } + try { + const abs = await resolveInJail(ctx.cwd, positional); + const stat = await fs.stat(abs); + if (!stat.isFile()) { + return { ok: false, reason: `bash: python3 script "${positional}" is not a regular file`, code: "python3_script_not_file" }; + } + } catch (err) { + if (err instanceof JailViolation) { + return { + ok: false, + reason: `bash: python3 script "${positional}" ${err.reason}`, + code: err.reason, + }; + } + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + reason: `bash: python3 script "${positional}" not accessible (${code})`, + code: code === "ENOENT" ? "python3_script_missing" : code, + }; + } + return { ok: true }; +} + +interface CollectorState { + parts: string[]; + byteCount: number; + truncated: boolean; +} + +function appendBounded(state: CollectorState, chunk: Buffer | string, maxBytes: number): void { + const buf = typeof chunk === "string" ? Buffer.from(chunk, "utf8") : chunk; + const before = state.byteCount; + state.byteCount = before + buf.length; + if (before >= maxBytes) { + state.truncated = true; + return; + } + if (before + buf.length <= maxBytes) { + state.parts.push(buf.toString("utf8")); + } else { + state.parts.push(buf.subarray(0, maxBytes - before).toString("utf8")); + state.truncated = true; + } +} + +function finalizeCollector(state: CollectorState, maxBytes: number, label: string): string { + const joined = state.parts.join(""); + if (!state.truncated) return joined; + return joined + `\n[truncated: ${label} capped at ${maxBytes} of ${state.byteCount} bytes]`; +} + +export async function runCommandTool(raw: JsonObject, ctx: ToolExecutionContext): Promise { + const parsed = parseArgs(raw); + if (!parsed.ok) { + return { ok: false, content: parsed.reason, audit: { truncated: false, error: "bad_args" } }; + } + const { command, args } = parsed.args; + + const cmdMetaErr = metacharCheck(command, "command"); + if (cmdMetaErr) { + return { + ok: false, + content: `bash: ${cmdMetaErr}`, + audit: { truncated: false, error: "metacharacter", command: summarizeArg(command) }, + }; + } + for (let i = 0; i < args.length; i++) { + const argMetaErr = metacharCheck(args[i]!, `args[${i}]`); + if (argMetaErr) { + return { + ok: false, + content: `bash: ${argMetaErr}`, + audit: { + truncated: false, + error: "metacharacter", + command: summarizeArg(command), + args: args.map(summarizeArg), + }, + }; + } + } + + const basename = path.basename(command).toLowerCase(); + if (NEVER_LIST.has(basename)) { + return { + ok: false, + content: `bash: "${basename}" is a shell or shell-like bypass and is never permitted`, + audit: { + truncated: false, + error: "shell_blocked", + command: summarizeArg(command), + args: args.map(summarizeArg), + }, + }; + } + + const opts = readBashOpts(ctx); + if (opts.allowlist.length === 0) { + return { + ok: false, + content: `bash: tool is disabled (no allowlist configured for this engine)`, + audit: { + truncated: false, + error: "disabled", + command: summarizeArg(command), + }, + }; + } + if (!opts.allowlist.includes(basename)) { + return { + ok: false, + content: `bash: "${basename}" is not in the configured allowlist (${opts.allowlist.join(", ")})`, + audit: { + truncated: false, + error: "not_in_allowlist", + command: summarizeArg(command), + args: args.map(summarizeArg), + }, + }; + } + + if (basename === "python3" || basename === "python") { + const py = await validatePython(args, ctx); + if (!py.ok) { + return { + ok: false, + content: py.reason, + audit: { + truncated: false, + error: py.code, + command: summarizeArg(command), + args: args.map(summarizeArg), + }, + }; + } + } + + const start = Date.now(); + const stdoutState: CollectorState = { parts: [], byteCount: 0, truncated: false }; + const stderrState: CollectorState = { parts: [], byteCount: 0, truncated: false }; + + let exitCode: number | null = null; + let signal: NodeJS.Signals | null = null; + let timedOut = false; + let spawnError: NodeJS.ErrnoException | null = null; + + await new Promise((resolve) => { + const proc = spawn(command, args, { + cwd: ctx.cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + + let settled = false; + const settle = () => { + if (settled) return; + settled = true; + resolve(); + }; + + proc.on("error", (err) => { + spawnError = err as NodeJS.ErrnoException; + settle(); + }); + + proc.stdout?.on("data", (d: Buffer) => appendBounded(stdoutState, d, opts.maxStdout)); + proc.stderr?.on("data", (d: Buffer) => appendBounded(stderrState, d, opts.maxStderr)); + + let killer: NodeJS.Timeout | null = null; + const termTimer = setTimeout(() => { + timedOut = true; + try { + proc.kill("SIGTERM"); + } catch { + // proc already exited + } + killer = setTimeout(() => { + try { + proc.kill("SIGKILL"); + } catch { + // already gone + } + }, opts.killGraceMs); + }, opts.perCallTimeoutMs); + + proc.on("close", (code, sig) => { + exitCode = code; + signal = sig; + clearTimeout(termTimer); + if (killer) clearTimeout(killer); + settle(); + }); + }); + + const durationMs = Date.now() - start; + + // Cast through `as` because TS narrows the let-binding through the + // closure to `never` in the truthy branch even though we assign in the + // spawn 'error' handler. + const errorMaybe = spawnError as NodeJS.ErrnoException | null; + if (errorMaybe !== null) { + return { + ok: false, + content: `bash: spawn failed (${errorMaybe.code ?? "unknown"}): ${errorMaybe.message}`, + audit: { + truncated: false, + error: errorMaybe.code ?? "spawn_failed", + command: summarizeArg(command), + args: args.map(summarizeArg), + duration_ms: durationMs, + }, + }; + } + + const stdout = finalizeCollector(stdoutState, opts.maxStdout, "stdout"); + const stderr = finalizeCollector(stderrState, opts.maxStderr, "stderr"); + + const lines: string[] = []; + if (timedOut) { + lines.push(`[timed out after ${opts.perCallTimeoutMs}ms — killed]`); + } else { + lines.push(`[exit ${exitCode}${signal ? ` signal=${signal}` : ""}]`); + } + if (stdout.length > 0) { + lines.push("--- stdout ---"); + lines.push(stdout); + } + if (stderr.length > 0) { + lines.push("--- stderr ---"); + lines.push(stderr); + } + const content = lines.join("\n"); + + const truncatedAny = stdoutState.truncated || stderrState.truncated; + const ok = !timedOut && exitCode === 0; + return { + ok, + content, + audit: { + truncated: truncatedAny, + truncated_stdout: stdoutState.truncated, + truncated_stderr: stderrState.truncated, + original_stdout_bytes: stdoutState.byteCount, + original_stderr_bytes: stderrState.byteCount, + exit_code: exitCode, + signal, + timeout: timedOut, + duration_ms: durationMs, + command: summarizeArg(command), + args: args.map(summarizeArg), + error: timedOut ? "timeout" : ok ? undefined : "nonzero_exit", + }, + }; +} diff --git a/packages/jimmy/src/engines/tools/schemas.ts b/packages/jimmy/src/engines/tools/schemas.ts new file mode 100644 index 00000000..58ef205f --- /dev/null +++ b/packages/jimmy/src/engines/tools/schemas.ts @@ -0,0 +1,105 @@ +/** + * JSON-schema tool definitions presented to the model via the + * provider tools[] array. Kept in OpenAI function-calling format + * (which Ollama also accepts). + * + * Keep descriptions terse — every token here is in every prompt. + */ + +import type { ProviderToolDef } from "../providers/types.js"; + +export const READ_TOOL_SCHEMA: ProviderToolDef = { + name: "read", + description: + "Read a text file under the working directory. Returns the file contents as a string. " + + "Use offset/limit for large files. Path may be relative or absolute; must resolve under the working directory.", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "File path." }, + offset: { type: "integer", minimum: 1, description: "1-indexed line number to start at." }, + limit: { type: "integer", minimum: 1, description: "Max lines to return (default 2000)." }, + }, + required: ["path"], + }, +}; + +export const WRITE_TOOL_SCHEMA: ProviderToolDef = { + name: "write", + description: + "Write text content to a file under the working directory. Overwrites existing files. " + + "Creates parent directories as needed. Refuses symbolic links and paths outside the working directory.", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "File path." }, + content: { type: "string", description: "File contents (UTF-8)." }, + }, + required: ["path", "content"], + }, +}; + +export const EDIT_TOOL_SCHEMA: ProviderToolDef = { + name: "edit", + description: + "Replace an exact substring in a file with another string. " + + "Fails if old_string is not found, or matches multiple times unless replace_all is true. " + + "Refuses symbolic links and paths outside the working directory.", + parameters: { + type: "object", + properties: { + path: { type: "string", description: "File path." }, + old_string: { type: "string", description: "Exact substring to find. Must be non-empty." }, + new_string: { type: "string", description: "Replacement text. Pass empty string to delete." }, + replace_all: { + type: "boolean", + description: "Replace every occurrence. Default false (requires unique match).", + }, + }, + required: ["path", "old_string", "new_string"], + }, +}; + +export const BASH_TOOL_SCHEMA: ProviderToolDef = { + name: "bash", + description: + "Execute a command in argv form (no shell). " + + "Shell metacharacters in arguments are rejected. " + + "Available executables are limited to a per-engine allowlist; shell binaries (sh, bash, etc.) are never permitted. " + + "python3 must be invoked with a script path argument (no -c, -m, or stdin).", + parameters: { + type: "object", + properties: { + command: { type: "string", description: "Executable basename or path." }, + args: { + type: "array", + items: { type: "string" }, + description: "Argument vector; each element passed as a separate argv slot.", + }, + }, + required: ["command"], + }, +}; + +export const WEBFETCH_TOOL_SCHEMA: ProviderToolDef = { + name: "webfetch", + description: + "GET an http or https URL. Returns the response body decoded as text. " + + "Private-network targets, redirects to other schemes, and non-text content types are refused. " + + "Follows up to 5 same-scheme redirects.", + parameters: { + type: "object", + properties: { + url: { type: "string", description: "Absolute http:// or https:// URL." }, + }, + required: ["url"], + }, +}; + +export const ALL_SCHEMAS = { + read: READ_TOOL_SCHEMA, + write: WRITE_TOOL_SCHEMA, + edit: EDIT_TOOL_SCHEMA, + bash: BASH_TOOL_SCHEMA, + webfetch: WEBFETCH_TOOL_SCHEMA, +} as const; diff --git a/packages/jimmy/src/engines/tools/types.ts b/packages/jimmy/src/engines/tools/types.ts new file mode 100644 index 00000000..2cc1c60c --- /dev/null +++ b/packages/jimmy/src/engines/tools/types.ts @@ -0,0 +1,47 @@ +/** + * Shared types for tool implementations consumed by the agent loop. + * + * Every tool exports an executor with signature + * (args: JsonObject, ctx: ToolExecutionContext) => Promise + * + * The executor MUST NOT throw on user-input errors (bad arg shape, file not + * found, jail violation, etc.). It returns `{ ok: false, content: , audit: { error } }` so the agent loop can feed the error back + * to the model as a `tool` role message and let the model recover. + * + * The executor MAY throw on programmer errors (bad config, missing helper). + * The loop catches and converts these into engine errors. + */ + +import type { JsonObject, JsonValue } from "../../shared/types.js"; + +export interface ToolExecutionContext { + /** Absolute path that bounds filesystem access for jailed tools. */ + cwd: string; + /** Per-tool overrides from EngineToolsConfig (truncation caps, etc.). */ + toolOpts?: Record; + /** Jin session id (used by the audit-log writer in Phase 6). */ + sessionId?: string; + /** Engine name (audit log + error context). */ + engineName?: string; +} + +/** + * What every tool returns. The `content` string is what gets fed back to + * the model verbatim as the `tool` role message body. `audit` is consumed + * by the audit-log writer and never reaches the model. + */ +export interface ToolResult { + ok: boolean; + /** Plain string the model sees. May already include truncation markers. */ + content: string; + audit: { + truncated: boolean; + /** Pre-truncation byte/char count if known. */ + originalBytes?: number; + /** Set when ok=false. Short reason. */ + error?: string; + /** Free-form per-tool extras (exit_code for bash, http_status for webfetch, etc.). */ + [key: string]: JsonValue | undefined; + }; +} diff --git a/packages/jimmy/src/engines/tools/webfetch.ts b/packages/jimmy/src/engines/tools/webfetch.ts new file mode 100644 index 00000000..10eb4b49 --- /dev/null +++ b/packages/jimmy/src/engines/tools/webfetch.ts @@ -0,0 +1,498 @@ +/** + * `webfetch` tool — HTTP/HTTPS fetch with strict network controls. + * + * Why not bare `fetch`? + * We need to validate the IP address actually used by the socket (DNS + * rebinding mitigation). Bare fetch in Node doesn't let you intercept + * the connect-time DNS lookup. node:http / node:https `request` accept + * a `lookup` option that lets us return only validated addresses. + * + * Hardening posture (V1): + * - Scheme: http: and https: only. file:, ftp:, gopher:, etc. rejected. + * - IP literals in the URL hostname are validated against ipBlocklist. + * - For hostnames: dns.resolve4 + dns.resolve6 BEFORE connect. If any + * resolved address is on the blocklist, refuse the whole request. + * - At connect time, a custom net.LookupFunction runs again as defense + * in depth — even if the cached pre-resolved set was clean, the + * address handed to the socket gets re-validated. DNS rebinding + * attempts hit this gate. + * - Redirects: same-scheme only, hard limit of 5, every redirect target + * re-validated from scratch. + * - Total wall-clock deadline (default 15s) covers the whole call, + * including all redirects. + * - Response body capped at 2 MB raw; the socket is destroyed once + * the cap is exceeded (we do NOT buffer the whole response and then + * truncate). + * - Content-Type whitelist: text/* and a small set of application/* + * (json, xml, atom, rss, yaml). Other types → non_text_content. + * - Caller can opt into private-network destinations with + * toolOpts.webfetch.allowPrivate = true. Default false. + * - Never throws to caller. All failure modes return {ok:false, audit: + * {error: , ...}}. + */ + +import http from "node:http"; +import https from "node:https"; +import dns from "node:dns/promises"; +import { URL } from "node:url"; +import type { LookupAddress, LookupOptions } from "node:dns"; +import type { LookupFunction } from "node:net"; +import type { JsonObject, JsonValue } from "../../shared/types.js"; +import { + checkAddress, + checkIPv4, + checkIPv6, + parseIpLiteral, +} from "./ipBlocklist.js"; +import type { ToolExecutionContext, ToolResult } from "./types.js"; + +const DEFAULT_MAX_CHARS = 64_000; +const DEFAULT_MAX_RAW_BYTES = 2 * 1024 * 1024; +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_REDIRECTS = 5; + +const ALLOWED_CONTENT_TYPES = [ + "text/", + "application/json", + "application/xml", + "application/xhtml+xml", + "application/atom+xml", + "application/rss+xml", + "application/yaml", + "application/x-yaml", + "application/ld+json", +]; + +interface WebfetchOpts { + maxChars: number; + maxRawBytes: number; + perCallTimeoutMs: number; + allowPrivate: boolean; +} + +function readOpts(ctx: ToolExecutionContext): WebfetchOpts { + const raw = (ctx.toolOpts?.webfetch ?? {}) as Record; + return { + maxChars: typeof raw.maxChars === "number" ? raw.maxChars : DEFAULT_MAX_CHARS, + maxRawBytes: typeof raw.maxRawBytes === "number" ? raw.maxRawBytes : DEFAULT_MAX_RAW_BYTES, + perCallTimeoutMs: + typeof raw.perCallTimeoutMs === "number" ? raw.perCallTimeoutMs : DEFAULT_TIMEOUT_MS, + allowPrivate: raw.allowPrivate === true, + }; +} + +interface WebfetchArgs { + url: string; +} + +function parseArgs(raw: JsonObject): { ok: true; args: WebfetchArgs } | { ok: false; reason: string } { + if (typeof raw.url !== "string" || raw.url.length === 0) { + return { ok: false, reason: "webfetch: 'url' is required and must be a non-empty string" }; + } + return { ok: true, args: { url: raw.url } }; +} + +function contentTypeAllowed(headerValue: string | undefined): boolean { + if (!headerValue) return false; + const main = headerValue.split(";")[0]!.trim().toLowerCase(); + return ALLOWED_CONTENT_TYPES.some((p) => main === p || main.startsWith(p)); +} + +/** + * Validate a hostname by resolving via dns.lookup({all:true}) and checking + * every returned address. We use lookup() not resolve4/6 because: + * - lookup() consults /etc/hosts and the OS resolver, matching what the + * socket-time custom LookupFunction will see. resolve4/6 are + * DNS-protocol-only and miss /etc/hosts entries (e.g. localhost). + * - Returns both A and AAAA in one call. + */ +async function preResolve( + host: string, + allowPrivate: boolean, +): Promise<{ ok: true } | { ok: false; reason: string }> { + const literal = parseIpLiteral(host); + if (literal) { + if (allowPrivate) return { ok: true }; + const r = checkAddress(literal.family, literal.address); + return r ? { ok: false, reason: r } : { ok: true }; + } + let all: LookupAddress[]; + try { + all = await dns.lookup(host, { all: true }); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "dns_error"; + return { + ok: false, + reason: code === "ENOTFOUND" || code === "ENODATA" ? "dns_no_addresses" : code, + }; + } + if (all.length === 0) { + return { ok: false, reason: "dns_no_addresses" }; + } + if (allowPrivate) return { ok: true }; + for (const a of all) { + const r = a.family === 4 ? checkIPv4(a.address) : checkIPv6(a.address); + if (r) return { ok: false, reason: r }; + } + return { ok: true }; +} + +/** + * Construct a net.LookupFunction that validates the resolved address before + * the socket connects. This is the DNS-rebinding mitigation — even if the + * pre-resolve check passed against a clean address set, the actual address + * handed to the socket is verified here. + * + * Exported so unit tests can exercise the validator path directly without + * setting up real sockets. + */ +export function buildLookup(allowPrivate: boolean): LookupFunction { + // Cast through `as` because node:net's LookupFunction signature uses an + // overloaded shape that's awkward to type from TS strictly. + return ((hostname: string, optsOrCallback: unknown, maybeCallback?: unknown): void => { + const callback = (typeof optsOrCallback === "function" ? optsOrCallback : maybeCallback) as ( + err: NodeJS.ErrnoException | null, + address?: string, + family?: number, + ) => void; + const options = (typeof optsOrCallback === "object" && optsOrCallback !== null ? optsOrCallback : {}) as LookupOptions; + dns + .lookup(hostname, options) + .then((result) => { + const single = result as LookupAddress; + if (!allowPrivate) { + const reason = checkAddress(single.family, single.address); + if (reason) { + const err: NodeJS.ErrnoException = Object.assign(new Error(reason), { code: reason }); + callback(err); + return; + } + } + callback(null, single.address, single.family); + }) + .catch((err) => callback(err as NodeJS.ErrnoException)); + }) as unknown as LookupFunction; +} + +interface FetchedOnce { + status: number; + headers: Record; + body: string; + bodyTruncated: boolean; + originalBytes: number; + redirectTo: string | null; +} + +interface FetchOnceFail { + failure: string; + detail?: string; +} + +async function fetchOnce( + rawUrl: string, + opts: WebfetchOpts, + deadline: number, + lookup: LookupFunction, +): Promise<{ ok: true; res: FetchedOnce } | { ok: false; fail: FetchOnceFail }> { + let url: URL; + try { + url = new URL(rawUrl); + } catch { + return { ok: false, fail: { failure: "bad_url", detail: rawUrl } }; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + return { ok: false, fail: { failure: "disallowed_scheme", detail: url.protocol } }; + } + + const pre = await preResolve(url.hostname, opts.allowPrivate); + if (!pre.ok) return { ok: false, fail: { failure: pre.reason } }; + + const remaining = deadline - Date.now(); + if (remaining <= 0) return { ok: false, fail: { failure: "timeout" } }; + + const transport = url.protocol === "https:" ? https : http; + + return await new Promise((resolve) => { + let settled = false; + let aborted = false; + let receivedBytes = 0; + const chunks: Buffer[] = []; + + const settle = (out: { ok: true; res: FetchedOnce } | { ok: false; fail: FetchOnceFail }) => { + if (settled) return; + settled = true; + resolve(out); + }; + + const req = transport.request( + { + hostname: url.hostname, + port: url.port || (url.protocol === "https:" ? 443 : 80), + path: `${url.pathname}${url.search}`, + method: "GET", + headers: { + "User-Agent": "Mozilla/5.0 (jin-webfetch/0.1)", + Accept: "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.5", + }, + lookup, + }, + (res) => { + const status = res.statusCode ?? 0; + const headers = res.headers; + + // Redirect: capture Location, drain (briefly) and resolve. + // HTTP spec says Location is single-valued, but node's + // IncomingHttpHeaders types it as `string | string[]`. Defensively + // unwrap so an unusual server can't poison the URL with comma-joins. + const locRaw = headers.location; + const locStr = Array.isArray(locRaw) ? locRaw[0] : locRaw; + if (status >= 300 && status < 400 && locStr) { + // Don't accumulate body bytes for redirect responses. + res.resume(); + res.on("end", () => { + settle({ + ok: true, + res: { + status, + headers, + body: "", + bodyTruncated: false, + originalBytes: 0, + redirectTo: locStr, + }, + }); + }); + res.on("error", () => { + settle({ + ok: true, + res: { + status, + headers, + body: "", + bodyTruncated: false, + originalBytes: 0, + redirectTo: locStr, + }, + }); + }); + return; + } + + // Non-redirect: check content-type before streaming. + const ct = headers["content-type"]; + if (!contentTypeAllowed(typeof ct === "string" ? ct : undefined)) { + req.destroy(); + settle({ + ok: false, + fail: { failure: "non_text_content", detail: typeof ct === "string" ? ct : "missing" }, + }); + return; + } + + res.on("data", (chunk: Buffer) => { + if (aborted) return; + receivedBytes += chunk.length; + if (receivedBytes > opts.maxRawBytes) { + aborted = true; + const overshoot = receivedBytes - opts.maxRawBytes; + const usable = chunk.length - overshoot; + if (usable > 0) chunks.push(chunk.subarray(0, usable)); + req.destroy(); + settle({ + ok: true, + res: { + status, + headers, + body: Buffer.concat(chunks).toString("utf8"), + bodyTruncated: true, + originalBytes: receivedBytes, + redirectTo: null, + }, + }); + } else { + chunks.push(chunk); + } + }); + res.on("end", () => { + if (aborted) return; + settle({ + ok: true, + res: { + status, + headers, + body: Buffer.concat(chunks).toString("utf8"), + bodyTruncated: false, + originalBytes: receivedBytes, + redirectTo: null, + }, + }); + }); + res.on("error", (err) => { + if (settled) return; + settle({ ok: false, fail: { failure: "stream_error", detail: err.message } }); + }); + }, + ); + + req.on("error", (err) => { + // Custom-lookup errors flow through here as well. + const code = (err as NodeJS.ErrnoException).code ?? "request_error"; + settle({ ok: false, fail: { failure: code, detail: err.message } }); + }); + + // Wall-clock deadline (covers DNS + connect + TLS + body). + const timer = setTimeout(() => { + aborted = true; + try { + req.destroy(); + } catch { + // ignore + } + settle({ ok: false, fail: { failure: "timeout" } }); + }, remaining); + req.on("close", () => clearTimeout(timer)); + + req.end(); + }); +} + +function modelTruncate(body: string, maxChars: number, totalBytes: number, alreadyTruncated: boolean): { + text: string; + truncated: boolean; +} { + if (body.length <= maxChars && !alreadyTruncated) { + return { text: body, truncated: false }; + } + if (body.length <= maxChars && alreadyTruncated) { + return { + text: body + `\n[truncated: server returned > ${totalBytes} bytes; raw cap hit]`, + truncated: true, + }; + } + return { + text: + body.slice(0, maxChars) + + `\n[truncated: ${maxChars} of ${body.length} characters returned to model${alreadyTruncated ? `; underlying body capped at ${totalBytes} bytes` : ""}]`, + truncated: true, + }; +} + +export async function webfetchTool(raw: JsonObject, ctx: ToolExecutionContext): Promise { + const parsed = parseArgs(raw); + if (!parsed.ok) { + return { ok: false, content: parsed.reason, audit: { truncated: false, error: "bad_args" } }; + } + const opts = readOpts(ctx); + const lookup = buildLookup(opts.allowPrivate); + const deadline = Date.now() + opts.perCallTimeoutMs; + + let currentUrl = parsed.args.url; + const redirectChain: string[] = [currentUrl]; + let currentScheme = ""; + + try { + currentScheme = new URL(currentUrl).protocol; + } catch { + return { ok: false, content: `webfetch: bad URL "${currentUrl}"`, audit: { truncated: false, error: "bad_url" } }; + } + if (currentScheme !== "http:" && currentScheme !== "https:") { + return { + ok: false, + content: `webfetch: scheme "${currentScheme}" not allowed (http/https only)`, + audit: { truncated: false, error: "disallowed_scheme", scheme: currentScheme }, + }; + } + + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + if (Date.now() >= deadline) { + return { + ok: false, + content: `webfetch: total timeout after ${opts.perCallTimeoutMs}ms`, + audit: { + truncated: false, + error: "timeout", + redirect_chain: redirectChain, + hops: hop, + }, + }; + } + const single = await fetchOnce(currentUrl, opts, deadline, lookup); + if (!single.ok) { + return { + ok: false, + content: `webfetch: ${single.fail.failure}${single.fail.detail ? ` — ${single.fail.detail}` : ""}`, + audit: { + truncated: false, + error: single.fail.failure, + redirect_chain: redirectChain, + hops: hop, + }, + }; + } + const r = single.res; + if (r.redirectTo !== null) { + // Resolve relative redirects against the current URL. + let nextUrl: URL; + try { + nextUrl = new URL(r.redirectTo, currentUrl); + } catch { + return { + ok: false, + content: `webfetch: bad redirect Location "${r.redirectTo}"`, + audit: { truncated: false, error: "bad_redirect", redirect_chain: redirectChain, hops: hop }, + }; + } + if (nextUrl.protocol !== currentScheme) { + return { + ok: false, + content: `webfetch: redirect changes scheme (${currentScheme} → ${nextUrl.protocol}); same-scheme redirects only`, + audit: { + truncated: false, + error: "scheme_change_redirect", + redirect_chain: [...redirectChain, nextUrl.toString()], + hops: hop, + }, + }; + } + if (hop >= MAX_REDIRECTS) { + return { + ok: false, + content: `webfetch: redirect limit (${MAX_REDIRECTS}) exhausted`, + audit: { + truncated: false, + error: "redirect_limit", + redirect_chain: [...redirectChain, nextUrl.toString()], + hops: hop + 1, + }, + }; + } + currentUrl = nextUrl.toString(); + redirectChain.push(currentUrl); + continue; + } + + // Terminal response. + const { text, truncated } = modelTruncate(r.body, opts.maxChars, r.originalBytes, r.bodyTruncated); + const ok = r.status >= 200 && r.status < 300; + const ctHeader = r.headers["content-type"]; + return { + ok, + content: text, + audit: { + truncated, + original_bytes: r.originalBytes, + http_status: r.status, + content_type: typeof ctHeader === "string" ? ctHeader : null, + redirect_chain: redirectChain, + hops: hop, + error: ok ? undefined : "http_status", + }, + }; + } + + return { + ok: false, + content: `webfetch: redirect limit (${MAX_REDIRECTS}) exhausted`, + audit: { truncated: false, error: "redirect_limit", redirect_chain: redirectChain, hops: MAX_REDIRECTS }, + }; +} diff --git a/packages/jimmy/src/engines/tools/write.ts b/packages/jimmy/src/engines/tools/write.ts new file mode 100644 index 00000000..0822a406 --- /dev/null +++ b/packages/jimmy/src/engines/tools/write.ts @@ -0,0 +1,84 @@ +/** + * `write` tool — overwrite a text file under the cwd jail. + * + * Args: + * path: string (relative or absolute, must resolve under ctx.cwd) + * content: string (the new file contents; UTF-8 written) + * + * Creates parent directories implicitly via fs.mkdir(..., recursive: true) + * before writing — this matches the practical expectation when an agent + * asks to write `subdir/new.json` under cwd. The recursive mkdir cannot + * escape the jail because the resolved path is already jail-checked. + * + * No truncation policy (write is input-side; the model decides the content). + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import type { JsonObject } from "../../shared/types.js"; +import { JailViolation, resolveInJail } from "./cwdJail.js"; +import type { ToolExecutionContext, ToolResult } from "./types.js"; + +interface WriteArgs { + path: string; + content: string; +} + +function parseArgs(raw: JsonObject): { ok: true; args: WriteArgs } | { ok: false; reason: string } { + if (typeof raw.path !== "string" || raw.path.length === 0) { + return { ok: false, reason: "write: 'path' is required and must be a non-empty string" }; + } + if (typeof raw.content !== "string") { + return { ok: false, reason: "write: 'content' is required and must be a string" }; + } + return { ok: true, args: { path: raw.path, content: raw.content } }; +} + +export async function writeTool(raw: JsonObject, ctx: ToolExecutionContext): Promise { + const parsed = parseArgs(raw); + if (!parsed.ok) { + return { ok: false, content: parsed.reason, audit: { truncated: false, error: "bad_args" } }; + } + + let abs: string; + try { + abs = await resolveInJail(ctx.cwd, parsed.args.path, { rejectSymlinkLeaf: true }); + } catch (err) { + return { + ok: false, + content: `write: ${(err as Error).message}`, + audit: { truncated: false, error: err instanceof JailViolation ? err.reason : "bad_path" }, + }; + } + + // Refuse to write to the cwd itself (would EISDIR after mkdir-on-parent-of-cwd). + const realCwd = await fs.realpath(ctx.cwd); + if (abs === realCwd) { + return { + ok: false, + content: `write: refusing to write to cwd directory itself`, + audit: { truncated: false, error: "is_cwd_dir" }, + }; + } + + try { + await fs.mkdir(path.dirname(abs), { recursive: true }); + await fs.writeFile(abs, parsed.args.content, "utf8"); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code ?? "unknown"; + return { + ok: false, + content: `write: cannot write "${parsed.args.path}" (${code})`, + audit: { truncated: false, error: code }, + }; + } + + return { + ok: true, + content: `wrote ${parsed.args.content.length} chars to ${parsed.args.path}`, + audit: { + truncated: false, + bytes_written: Buffer.byteLength(parsed.args.content, "utf8"), + }, + }; +} diff --git a/packages/jimmy/src/gateway/api.ts b/packages/jimmy/src/gateway/api.ts index a7ef8621..a45e8261 100644 --- a/packages/jimmy/src/gateway/api.ts +++ b/packages/jimmy/src/gateway/api.ts @@ -81,6 +81,13 @@ export interface ApiContext { getConfig: () => JinnConfig; emit: (event: string, payload: unknown) => void; connectors: Map; + /** + * Registered engines, keyed by name. Used by /api/status to report + * what's actually live — config.engines.* may be present but + * registration may have been declined (e.g. opt-in HTTP engines whose + * config is malformed and construction throws). + */ + engines: Map; reloadConnectorInstances?: () => Promise<{ started: string[]; stopped: string[]; errors: string[] }>; } @@ -419,16 +426,21 @@ export async function handleApiRequest( if (v === "sideband" || v === "stdout") return v; return "off"; })(); + // Iterate the live engines Map so /api/status reflects what's + // actually registered, not just what's declared in config. An + // opt-in HTTP engine whose construction was declined (missing + // apiKey, malformed config) won't appear here — operators + // use the gap to detect mis-registered engines. + const enginesOut: Record = { default: config.engines.default }; + for (const name of context.engines.keys()) { + const cfgBlock = (config.engines as unknown as Record)[name]; + enginesOut[name] = { model: cfgBlock?.model, available: true }; + } return json(res, { status: "ok", uptime: Math.floor((Date.now() - context.startTime) / 1000), port: config.gateway.port || 7777, - engines: { - default: config.engines.default, - claude: { model: config.engines.claude.model, available: true }, - codex: { model: config.engines.codex.model, available: true }, - ...(config.engines.gemini ? { gemini: { model: config.engines.gemini.model, available: true } } : {}), - }, + engines: enginesOut, sessions: { total: sessions.length, running, active: running }, connectors, features: { diff --git a/packages/jimmy/src/gateway/server.ts b/packages/jimmy/src/gateway/server.ts index 2bf64012..e44891cb 100644 --- a/packages/jimmy/src/gateway/server.ts +++ b/packages/jimmy/src/gateway/server.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { randomUUID } from "node:crypto"; import { WebSocketServer, type WebSocket } from "ws"; -import type { JinnConfig, Connector, Employee } from "../shared/types.js"; +import type { JinnConfig, Connector, Employee, Engine } from "../shared/types.js"; import { loadConfig } from "../shared/config.js"; import { configureLogger, logger } from "../shared/logger.js"; import { initDb, recoverStaleSessions, recoverStaleQueueItems, getInterruptedSessions, listSessions, updateSession } from "../sessions/registry.js"; @@ -133,11 +133,47 @@ export async function startGateway( const claudeEngine = new ClaudeEngine(); const codexEngine = new CodexEngine(); const geminiEngine = new GeminiEngine(); - const engines = new Map | InstanceType | InstanceType>(); + // Widen to Engine so HTTP-loop engines (ollama/openai) can co-exist with + // the CLI-spawning engines. Process-killing paths still use the original + // typed references (claudeEngine, codexEngine) below. + const engines = new Map(); engines.set("claude", claudeEngine); engines.set("codex", codexEngine); engines.set("gemini", geminiEngine); + // HTTP-loop engines: register ONLY when configured, so /api/status and + // route-resolution errors reflect what's actually available. + // + // Construction-time failure policy: declaring engines. in config is + // a commitment to run it. If the engine can't construct (missing url, + // missing apiKey env var, malformed shape), the gateway FAILS LOUDLY at + // boot rather than silently disabling — the operator either fixes the + // problem or removes the config block to opt out. + if (config.engines.ollama) { + try { + const { OllamaEngine } = await import("../engines/ollama.js"); + engines.set("ollama", new OllamaEngine(config.engines.ollama)); + logger.info("engine registered: ollama"); + } catch (err) { + throw new Error( + `engines.ollama is declared but engine construction failed: ${(err as Error).message}\n` + + `→ Fix the config OR remove the engines.ollama block to opt out.`, + ); + } + } + if (config.engines.openai) { + try { + const { OpenAIEngine } = await import("../engines/openai.js"); + engines.set("openai", new OpenAIEngine(config.engines.openai)); + logger.info("engine registered: openai"); + } catch (err) { + throw new Error( + `engines.openai is declared but engine construction failed: ${(err as Error).message}\n` + + `→ Set the API key env var OR remove the engines.openai block to opt out.`, + ); + } + } + // Derive connector names from config const connectorNames: string[] = []; if (config.connectors?.slack?.appToken && config.connectors?.slack?.botToken) { @@ -567,6 +603,7 @@ export async function startGateway( getConfig: () => currentConfig, emit, connectors: connectorMap, + engines, reloadConnectorInstances, }; diff --git a/packages/jimmy/src/sessions/manager.ts b/packages/jimmy/src/sessions/manager.ts index cf519c1a..ebc600de 100644 --- a/packages/jimmy/src/sessions/manager.ts +++ b/packages/jimmy/src/sessions/manager.ts @@ -1,5 +1,6 @@ import fs from "node:fs"; import type { + BuiltInEngineName, Connector, Employee, Engine, @@ -8,6 +9,7 @@ import type { Session, Target, } from "../shared/types.js"; +import { modelFor } from "../shared/types.js"; import { startSessionTimeout } from "../shared/timeout.js"; import { accumulateSessionCost, @@ -934,7 +936,7 @@ export class SessionManager { `Session: ${session.id}`, `Engine: ${session.engine}`, `Connector: ${session.connector || session.source}`, - `Model: ${session.model || this.config.engines[session.engine as "claude" | "codex" | "gemini"]?.model || "default"}`, + `Model: ${session.model || modelFor(this.config.engines, session.engine as BuiltInEngineName) || "default"}`, `State: ${transportState}`, `Queue depth: ${queueDepth}`, `Created: ${session.createdAt}`, diff --git a/packages/jimmy/src/shared/types.ts b/packages/jimmy/src/shared/types.ts index e1a8c386..34147852 100644 --- a/packages/jimmy/src/shared/types.ts +++ b/packages/jimmy/src/shared/types.ts @@ -382,15 +382,88 @@ export interface PortalConfig { onboarded?: boolean; } +/** + * BuiltInEngineName lists the engines a user can configure via `config.yaml` + * or select via `cron job.engine`. `mock` is registered in the engines Map + * for tests but is not user-configurable — see EngineName below. + */ +export type BuiltInEngineName = "claude" | "codex" | "gemini" | "ollama" | "openai"; + +/** + * EngineName widens BuiltInEngineName with `mock` for the test surface. + * Anywhere user input may flow (cron jobs, config defaults, session + * `engine` field), use BuiltInEngineName. Anywhere the engines registry + * Map is keyed, use EngineName. + */ +export type EngineName = BuiltInEngineName | "mock"; + +export interface EngineConfigBase { + /** Default model the engine uses when a job doesn't specify one. */ + model?: string; + /** Engine binary (only meaningful for CLI-spawning engines: claude/codex/gemini). */ + bin?: string; + /** Reasoning-effort hint passed through to the CLI engine where supported. */ + effortLevel?: string; + /** Effort override applied only when this engine runs as a child session. */ + childEffortOverride?: string; + /** Max turns the agent loop will run before returning. Default per engine. */ + maxTurns?: number; + /** Wall-clock timeout for the whole engine run, in ms. Default per engine. */ + timeoutMs?: number; +} + +/** Per-tool enable + per-tool config knobs for HTTP-loop engines (ollama, openai). */ +export interface EngineToolsConfig { + /** Subset of {"read","write","edit","bash","webfetch"} the engine may use. Empty = no tools. */ + enabled?: string[]; + /** Bash tool allowlist (argv[0] tokens). Empty array = bash disabled regardless of `enabled`. */ + bashAllowlist?: string[]; + /** Per-tool overrides for truncation caps + webfetch network policy. */ + bash?: { maxStdout?: number; maxStderr?: number; perCallTimeoutMs?: number }; + read?: { maxChars?: number }; + webfetch?: { maxChars?: number; perCallTimeoutMs?: number; allowPrivate?: boolean }; +} + +export interface OllamaConfig extends EngineConfigBase { + /** Base URL for the Ollama HTTP API (e.g. https://ollama.aga.my). */ + url: string; + /** Optional bearer token; read from process.env.OLLAMA_TOKEN if unset here. */ + authTokenEnvVar?: string; + tools?: EngineToolsConfig; + /** HTTP per-call timeout (ms). Default 60000. */ + providerTimeoutMs?: number; +} + +export interface OpenAIConfig extends EngineConfigBase { + /** Defaults to https://api.openai.com/v1. */ + baseUrl?: string; + /** Env var holding the API key. Default OPENAI_API_KEY. */ + apiKeyEnvVar?: string; + tools?: EngineToolsConfig; + /** HTTP per-call timeout (ms). Default 60000. */ + providerTimeoutMs?: number; +} + +export interface EnginesConfig { + /** Default engine for sessions that don't specify one. Cannot default to `mock`. */ + default: BuiltInEngineName; + claude: EngineConfigBase & { bin: string; model: string }; + codex: EngineConfigBase & { bin: string; model: string }; + gemini?: EngineConfigBase & { bin: string; model: string }; + ollama?: OllamaConfig; + openai?: OpenAIConfig; +} + +/** Resolve the configured model for an engine, returning undefined if the engine isn't configured. */ +export function modelFor(config: EnginesConfig, engine: BuiltInEngineName): string | undefined { + const e = config[engine]; + return e?.model; +} + export interface JinnConfig { jinn?: { version?: string }; gateway: { port: number; host: string; streaming?: boolean }; - engines: { - default: "claude" | "codex" | "gemini"; - claude: { bin: string; model: string; effortLevel?: string; childEffortOverride?: string }; - codex: { bin: string; model: string; effortLevel?: string; childEffortOverride?: string }; - gemini?: { bin: string; model: string; effortLevel?: string; childEffortOverride?: string }; - }; + engines: EnginesConfig; connectors: Record & { web?: WebConnectorConfig; slack?: SlackConnectorConfig;