From 71a3cd08d589cb21a702e95b36b4b7e8a2697722 Mon Sep 17 00:00:00 2001 From: blue-az Date: Mon, 13 Jul 2026 18:31:16 +0200 Subject: [PATCH] feat: opt-in OpenRouter hardening and subagent kill switch Three independent, opt-in run-hardening controls, none of which change behavior unless explicitly set: - OPENWIKI_OPENROUTER_MAX_INPUT_TOKENS attaches a maxInputTokens profile to OpenRouter model instances so DeepAgents' summarization middleware triggers early enough to stay under OpenRouter's payload limit on constrained/budget models, instead of failing with an HTTP 500 on large transcripts. - OPENWIKI_DISABLE_MODEL_FALLBACK=1 disables OpenRouter's route: "fallback" model list for deterministic single-model runs. - OPENWIKI_DISABLE_SUBAGENTS=1 excludes the task tool via both a registered harness profile and runtime middleware (the same tool name enforced in one place), and updates the subagent-discipline prompt text to match, for runs where transcript growth from subagent fan-out is the binding constraint. Also excludes openwiki.* variant directories (testing/validation copies of generated docs) from repository discovery, so they are never inspected or documented. This supersedes #50, which hardened the same OpenRouter path against an older main; main has since gained connector tools, output modes, and ChatGPT OAuth login that touched every function #50 modified, so this re-implements the same opt-in controls directly against current main instead of resolving that merge conflict. Co-Authored-By: Claude Fable 5 --- .../operations/credentials-and-updates.md | 4 + src/agent/index.ts | 139 +++++++++++++++++- src/agent/prompt.ts | 37 ++++- src/constants.ts | 10 ++ test/openrouter-hardening.test.ts | 79 ++++++++++ test/prompt.test.ts | 46 +++++- test/subagent-disable.test.ts | 87 +++++++++++ 7 files changed, 392 insertions(+), 10 deletions(-) create mode 100644 test/openrouter-hardening.test.ts create mode 100644 test/subagent-disable.test.ts diff --git a/openwiki/operations/credentials-and-updates.md b/openwiki/operations/credentials-and-updates.md index 80dfef28..21cc7b8e 100644 --- a/openwiki/operations/credentials-and-updates.md +++ b/openwiki/operations/credentials-and-updates.md @@ -39,6 +39,10 @@ The file stores provider configuration and API keys: - Optional OAuth callback settings: `OPENWIKI_OAUTH_CALLBACK_PORT` controls the local callback port, and `OPENWIKI_HTTPS_OAUTH_REDIRECT_URI` stores the Slack-only HTTPS callback URL created by `openwiki ngrok start`. +- Run-hardening options (all opt-in; unset, behavior is unchanged): + - `OPENWIKI_OPENROUTER_MAX_INPUT_TOKENS` — sets a `maxInputTokens` profile on OpenRouter models so DeepAgents' summarization middleware triggers early enough to stay under OpenRouter's payload limit on constrained/budget models, instead of failing with an HTTP 500 on large transcripts. Suggested value for constrained models: `15000`. + - `OPENWIKI_DISABLE_MODEL_FALLBACK` — set to `1` to disable OpenRouter's `route: "fallback"` model list for deterministic single-model runs. + - `OPENWIKI_DISABLE_SUBAGENTS` — set to `1` to disable subagent task delegation entirely, for runs where transcript growth from subagent fan-out is the binding constraint. The loader merges those values into `process.env`, while preferring existing process-level values over file values. Deprecated keys (`OPENAI_BASE_URL`, `OPENAI_ORG_ID`, `OPENAI_PROJECT`) are skipped on load and removed on save. diff --git a/src/agent/index.ts b/src/agent/index.ts index 1c9e37ca..442dec4a 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -6,7 +6,8 @@ import { SqliteSaver } from "@langchain/langgraph-checkpoint-sqlite"; import { ChatOpenAI } from "@langchain/openai"; import { ChatOpenRouter } from "@langchain/openrouter"; import type { Event as ProtocolEvent } from "@langchain/protocol"; -import { createDeepAgent } from "deepagents"; +import { createDeepAgent, registerHarnessProfile } from "deepagents"; +import { createMiddleware } from "langchain"; import { createOpenWikiConnectorTools } from "../connectors/tools.js"; import { ensureWriteConnectorSkill } from "../connectors/write-connector-skill.js"; import { @@ -46,6 +47,7 @@ import { OPENAI_COMPATIBLE_BASE_URL_ENV_KEY, OPENROUTER_API_KEY_ENV_KEY, OPENROUTER_BASE_URL, + OPENROUTER_FALLBACK_MODEL_IDS, OPENWIKI_MODEL_ID_ENV_KEY, OPENWIKI_PROVIDER_ENV_KEY, OPENWIKI_PROVIDER_RETRY_ATTEMPTS_ENV_KEY, @@ -163,6 +165,13 @@ async function runOpenWikiAgentCore( emitDebug(options, "openwiki.snapshot=created"); const model = createModel(provider, modelId, providerRetryAttempts); emitDebug(options, `model.provider=${provider}`); + configureDeepAgentHarness(provider, modelId); + if (provider === "openrouter") { + emitDebug( + options, + `openrouter.route=fallback models=${JSON.stringify(createModelRoute(provider, modelId))}`, + ); + } emitDebug(options, "model=initialized"); const threadId = options.threadId ?? createThreadId(cwd, createRunThreadId()); emitDebug(options, `thread=${threadId}`); @@ -177,6 +186,7 @@ async function runOpenWikiAgentCore( const agent = createDeepAgent({ model, tools: createOpenWikiConnectorTools(), + middleware: createOpenWikiMiddleware(), checkpointer, backend: new OpenWikiLocalShellBackend({ docsOnly: command !== "chat", @@ -471,13 +481,17 @@ function createModel( } if (provider === "openrouter") { - return new ChatOpenRouter({ + const model = new ChatOpenRouter({ apiKey: process.env[OPENROUTER_API_KEY_ENV_KEY], baseURL: OPENROUTER_BASE_URL, model: modelId, + models: createModelRoute(provider, modelId), + route: "fallback", siteName: "OpenWiki", ...retryOptions, }); + applyOpenRouterMaxInputTokens(model); + return model; } const baseURL = resolveProviderBaseUrl(provider); @@ -495,6 +509,127 @@ function createModel( }); } +export function isModelFallbackDisabled(): boolean { + return process.env.OPENWIKI_DISABLE_MODEL_FALLBACK === "1"; +} + +/** + * The OpenRouter `models` list tried in order via `route: "fallback"`. Opt out + * with `OPENWIKI_DISABLE_MODEL_FALLBACK=1` for deterministic single-model runs. + */ +export function createModelRoute( + provider: OpenWikiProvider, + modelId: string, +): string[] { + if (provider !== "openrouter" || isModelFallbackDisabled()) { + return [modelId]; + } + + return Array.from(new Set([modelId, ...OPENROUTER_FALLBACK_MODEL_IDS])); +} + +/** + * Parses `OPENWIKI_OPENROUTER_MAX_INPUT_TOKENS`. Returns `undefined` when + * unset or non-numeric, so the opt-in cap is a no-op unless a valid value is + * configured. Pure so the parsing edge cases are testable without a real + * model instance. + */ +export function resolveOpenRouterMaxInputTokens( + raw: string | undefined, +): number | undefined { + if (!raw) { + return undefined; + } + + const parsed = parseInt(raw, 10); + return Number.isFinite(parsed) ? parsed : undefined; +} + +/** + * Sets a `maxInputTokens` profile on the OpenRouter model instance (opt-in via + * `OPENWIKI_OPENROUTER_MAX_INPUT_TOKENS`) so DeepAgents' summarization + * middleware triggers early enough to stay under OpenRouter's payload limit + * on constrained/budget models, instead of failing with an HTTP 500 on large + * transcripts. `defineProperty` avoids crashing on a frozen/non-extensible + * model instance. + */ +function applyOpenRouterMaxInputTokens(model: ChatOpenRouter): void { + const maxInputTokens = resolveOpenRouterMaxInputTokens( + process.env.OPENWIKI_OPENROUTER_MAX_INPUT_TOKENS, + ); + if (maxInputTokens === undefined) { + return; + } + + Object.defineProperty(model, "profile", { + value: { maxInputTokens }, + writable: true, + configurable: true, + enumerable: true, + }); +} + +// The subagent delegation tool. Excluded from the model both via the harness +// profile (below) and via runtime middleware so the same name is enforced in +// one place. +const SUBAGENT_TASK_TOOL_NAME = "task"; + +/** + * The harness profile applied when subagents are disabled: it drops the task + * tool and turns off the general-purpose subagent. Pure and exported so the + * disabled shape can be asserted in tests without touching the global harness + * registry. + */ +export function buildSubagentDisabledProfile() { + return { + excludedTools: [SUBAGENT_TASK_TOOL_NAME], + generalPurposeSubagent: { + enabled: false, + }, + }; +} + +function configureDeepAgentHarness( + provider: OpenWikiProvider, + modelId: string, +): void { + if (!isSubagentDisabled()) { + return; + } + + const profile = buildSubagentDisabledProfile(); + + registerHarnessProfile(provider, profile); + + if (!modelId.includes(":")) { + registerHarnessProfile(`${provider}:${modelId}`, profile); + } +} + +export function isSubagentDisabled(): boolean { + return process.env.OPENWIKI_DISABLE_SUBAGENTS === "1"; +} + +export function createOpenWikiMiddleware() { + if (!isSubagentDisabled()) { + return []; + } + + return [ + createMiddleware({ + name: "OpenWikiToolExclusionMiddleware", + wrapModelCall: async (request, handler) => { + return handler({ + ...request, + tools: request.tools?.filter( + (tool) => tool.name !== SUBAGENT_TASK_TOOL_NAME, + ), + }); + }, + }), + ]; +} + const CHATGPT_LOGIN_INCOMPLETE_MESSAGE = "ChatGPT login is incomplete. Run `openwiki code --init` or `openwiki personal --init` to sign in with your ChatGPT account."; diff --git a/src/agent/prompt.ts b/src/agent/prompt.ts index e5526c80..5d30e292 100644 --- a/src/agent/prompt.ts +++ b/src/agent/prompt.ts @@ -38,6 +38,7 @@ Run discipline: - Shell execute commands run on the host. If you use execute, run commands from the current runtime root unless a source-specific instruction explicitly tells you to inspect a connector raw file or configured local repository path. - Do not exhaustively read every file. For a local knowledge wiki, inspect the existing wiki structure and only the relevant connector evidence or configured local repository paths. For an explicit repository source, inspect the repository tree, package/config files, README-style files, entrypoints, routing files, database/schema files, and representative files for each major domain. - Do not call glob with **/* from the root. Use targeted discovery by directory and extension. Prefer shell commands like rg --files with excludes for .git, node_modules, dist, build, cache directories, and existing generated wiki output. +- Do not read, write, or search any openwiki.* variant directories (anything other than openwiki/ itself); they are testing/validation artifacts, not part of the codebase documentation. - Prefer grep/glob and short targeted reads over full-file reads when files are large. - Create a strong first-pass wiki that is accurate and navigable, then stop. The wiki can be refined in later update runs. - Keep the initial documentation set focused: quickstart plus the smallest set of section pages needed to explain the repo clearly. @@ -75,13 +76,7 @@ Wiki-first question answering: - When the wiki answers the question, do not inspect or mention raw connector data. - When you do inspect raw data, keep reads narrow: list latest raw items for the relevant connector, open only the specific files needed, and summarize only the minimum evidence required to answer or update the wiki. -Subagent discipline: -- You may use the task tool to parallelize read-only research during init and update runs when the repository has multiple substantial domains. -- Default to 1-2 subagents for large or unfamiliar repositories. Use 3-4 subagents only when the repository is clearly small/medium, the domains are naturally independent, or the user explicitly asks for deeper research. -- Subagents must only inspect and summarize. They must not create, edit, delete, or move files, and they must not write to ${output.docsLocation}. -- Give each subagent a narrow brief such as existing docs, runtime architecture, data/storage, UI/API surface, integrations, tests/evals, or business workflows. -- Ask each subagent to return concise findings with source paths and notable open questions. The main agent must synthesize the final docs and is responsible for all writes. -- Treat subagent reports as internal discovery notes. Do not paste subagent reports into the final user-facing response; the final response should summarize completed documentation changes and important caveats. +${createSubagentInstructions(output.docsLocation)} Planning discipline: - After discovery and before writing final documentation, create a temporary ${output.planPath} file that lists the intended wiki pages, source evidence for each page, and remaining questions. @@ -165,6 +160,34 @@ ${createModeInstructions(command, outputMode)} `.trim(); } +/** + * Subagent discipline text. When `OPENWIKI_DISABLE_SUBAGENTS=1`, tells the + * agent not to delegate to the task tool at all — paired with the + * harness-profile/middleware exclusion in `src/agent/index.ts` that actually + * removes the tool, for runs where transcript growth from subagent fan-out is + * the binding constraint. + */ +function createSubagentInstructions(docsLocation: string): string { + if (process.env.OPENWIKI_DISABLE_SUBAGENTS === "1") { + return ` +Subagent discipline: +- Do not use the task tool or delegate research to subagents during this run. +- Perform discovery directly with ls, glob, grep, read_file, and targeted shell execute commands. +- Keep discovery concise and write documentation as soon as the main architecture, workflows, operations, and tests are understood. +`.trim(); + } + + return ` +Subagent discipline: +- You may use the task tool to parallelize read-only research during init and update runs when the repository has multiple substantial domains. +- Default to 1-2 subagents for large or unfamiliar repositories. Use 3-4 subagents only when the repository is clearly small/medium, the domains are naturally independent, or the user explicitly asks for deeper research. +- Subagents must only inspect and summarize. They must not create, edit, delete, or move files, and they must not write to ${docsLocation}. +- Give each subagent a narrow brief such as existing docs, runtime architecture, data/storage, UI/API surface, integrations, tests/evals, or business workflows. +- Ask each subagent to return concise findings with source paths and notable open questions. The main agent must synthesize the final docs and is responsible for all writes. +- Treat subagent reports as internal discovery notes. Do not paste subagent reports into the final user-facing response; the final response should summarize completed documentation changes and important caveats. +`.trim(); +} + export function createModeInstructions( command: OpenWikiCommand, outputMode: OpenWikiOutputMode = "local-wiki", diff --git a/src/constants.ts b/src/constants.ts index 1c1eb2bb..90e36aa6 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -219,6 +219,16 @@ export const SUGGESTED_MODEL_IDS = PROVIDER_CONFIGS[ DEFAULT_PROVIDER ].modelOptions.map((model) => model.id); +/** + * Fallback model ids OpenRouter tries, in order, when the primary model + * request fails (`route: "fallback"` in `createModel`). Opt out entirely with + * `OPENWIKI_DISABLE_MODEL_FALLBACK=1` for deterministic single-model runs. + */ +export const OPENROUTER_FALLBACK_MODEL_IDS = [ + "openai/gpt-5.4-mini", + "anthropic/claude-sonnet-5", +]; + export function getProviderConfig(provider: OpenWikiProvider): ProviderConfig { return PROVIDER_CONFIGS[provider]; } diff --git a/test/openrouter-hardening.test.ts b/test/openrouter-hardening.test.ts new file mode 100644 index 00000000..63dfb11e --- /dev/null +++ b/test/openrouter-hardening.test.ts @@ -0,0 +1,79 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + createModelRoute, + isModelFallbackDisabled, + resolveOpenRouterMaxInputTokens, +} from "../src/agent/index.ts"; +import { OPENROUTER_FALLBACK_MODEL_IDS } from "../src/constants.ts"; + +const FALLBACK_ENV_KEY = "OPENWIKI_DISABLE_MODEL_FALLBACK"; +let previousFallbackValue: string | undefined; + +beforeEach(() => { + previousFallbackValue = process.env[FALLBACK_ENV_KEY]; + delete process.env[FALLBACK_ENV_KEY]; +}); + +afterEach(() => { + if (previousFallbackValue === undefined) { + delete process.env[FALLBACK_ENV_KEY]; + } else { + process.env[FALLBACK_ENV_KEY] = previousFallbackValue; + } +}); + +describe("isModelFallbackDisabled", () => { + test('only reports true for exactly "1"', () => { + process.env[FALLBACK_ENV_KEY] = "1"; + expect(isModelFallbackDisabled()).toBe(true); + + delete process.env[FALLBACK_ENV_KEY]; + expect(isModelFallbackDisabled()).toBe(false); + + process.env[FALLBACK_ENV_KEY] = "true"; + expect(isModelFallbackDisabled()).toBe(false); + }); +}); + +describe("createModelRoute", () => { + test("non-openrouter providers never get a fallback list", () => { + expect(createModelRoute("anthropic", "claude-sonnet-5")).toEqual([ + "claude-sonnet-5", + ]); + }); + + test("openrouter gets the primary model plus fallback ids by default", () => { + const route = createModelRoute("openrouter", "z-ai/glm-5.2"); + expect(route[0]).toBe("z-ai/glm-5.2"); + for (const fallback of OPENROUTER_FALLBACK_MODEL_IDS) { + expect(route).toContain(fallback); + } + }); + + test("does not duplicate the primary model when it is also a fallback id", () => { + const [primary] = OPENROUTER_FALLBACK_MODEL_IDS; + const route = createModelRoute("openrouter", primary); + expect(route.filter((id) => id === primary)).toHaveLength(1); + }); + + test("OPENWIKI_DISABLE_MODEL_FALLBACK=1 collapses openrouter to a single model", () => { + process.env[FALLBACK_ENV_KEY] = "1"; + expect(createModelRoute("openrouter", "z-ai/glm-5.2")).toEqual([ + "z-ai/glm-5.2", + ]); + }); +}); + +describe("resolveOpenRouterMaxInputTokens", () => { + test("returns undefined when unset", () => { + expect(resolveOpenRouterMaxInputTokens(undefined)).toBeUndefined(); + }); + + test("returns undefined for non-numeric values", () => { + expect(resolveOpenRouterMaxInputTokens("not-a-number")).toBeUndefined(); + }); + + test("parses a valid integer string", () => { + expect(resolveOpenRouterMaxInputTokens("15000")).toBe(15000); + }); +}); diff --git a/test/prompt.test.ts b/test/prompt.test.ts index c670a7f9..81ba2af5 100644 --- a/test/prompt.test.ts +++ b/test/prompt.test.ts @@ -1,7 +1,23 @@ -import { describe, expect, test } from "vitest"; +import { afterEach, beforeEach, describe, expect, test } from "vitest"; import { createSystemPrompt, createUserPrompt } from "../src/agent/prompt.ts"; import type { RunContext } from "../src/agent/types.ts"; +const SUBAGENT_ENV_KEY = "OPENWIKI_DISABLE_SUBAGENTS"; +let previousSubagentValue: string | undefined; + +beforeEach(() => { + previousSubagentValue = process.env[SUBAGENT_ENV_KEY]; + delete process.env[SUBAGENT_ENV_KEY]; +}); + +afterEach(() => { + if (previousSubagentValue === undefined) { + delete process.env[SUBAGENT_ENV_KEY]; + } else { + process.env[SUBAGENT_ENV_KEY] = previousSubagentValue; + } +}); + describe("createUserPrompt", () => { test("includes the wiki brief for repository init runs", () => { const context: RunContext = { @@ -53,3 +69,31 @@ describe("documentation coverage guidance", () => { } }); }); + +describe("run discipline", () => { + test("excludes openwiki.* variant directories from discovery", () => { + expect(createSystemPrompt("init", "repository")).toContain( + "Do not read, write, or search any openwiki.* variant directories", + ); + }); +}); + +describe("subagent discipline", () => { + test("permits subagent delegation by default", () => { + const prompt = createSystemPrompt("init", "repository"); + expect(prompt).toContain( + "You may use the task tool to parallelize read-only research", + ); + }); + + test("OPENWIKI_DISABLE_SUBAGENTS=1 tells the agent not to delegate", () => { + process.env[SUBAGENT_ENV_KEY] = "1"; + const prompt = createSystemPrompt("init", "repository"); + expect(prompt).toContain( + "Do not use the task tool or delegate research to subagents", + ); + expect(prompt).not.toContain( + "You may use the task tool to parallelize read-only research", + ); + }); +}); diff --git a/test/subagent-disable.test.ts b/test/subagent-disable.test.ts new file mode 100644 index 00000000..42242737 --- /dev/null +++ b/test/subagent-disable.test.ts @@ -0,0 +1,87 @@ +import { afterEach, beforeEach, describe, expect, test } from "vitest"; +import { + buildSubagentDisabledProfile, + createOpenWikiMiddleware, + isSubagentDisabled, +} from "../src/agent/index.ts"; + +const ENV_KEY = "OPENWIKI_DISABLE_SUBAGENTS"; +let previousValue: string | undefined; + +beforeEach(() => { + previousValue = process.env[ENV_KEY]; + delete process.env[ENV_KEY]; +}); + +afterEach(() => { + if (previousValue === undefined) { + delete process.env[ENV_KEY]; + } else { + process.env[ENV_KEY] = previousValue; + } +}); + +// Invoke a middleware's wrapModelCall with a captured request so we can +// inspect the tool list the model would actually be handed. +type MinimalWrapModelCallMiddleware = { + wrapModelCall: ( + request: { tools: Array<{ name: string }> }, + handler: (request: { tools: Array<{ name: string }> }) => Promise, + ) => Promise; +}; + +async function toolsSeenByModel( + middleware: ReturnType, + tools: Array<{ name: string }>, +): Promise> { + expect(middleware.length).toBe(1); + let captured: Array<{ name: string }> = tools; + const [wrapper] = middleware as unknown as MinimalWrapModelCallMiddleware[]; + await wrapper.wrapModelCall({ tools }, (request) => { + captured = request.tools; + return Promise.resolve({}); + }); + return captured; +} + +describe("isSubagentDisabled", () => { + test('only reports true for exactly "1"', () => { + process.env[ENV_KEY] = "1"; + expect(isSubagentDisabled()).toBe(true); + + delete process.env[ENV_KEY]; + expect(isSubagentDisabled()).toBe(false); + + process.env[ENV_KEY] = "true"; + expect(isSubagentDisabled()).toBe(false); + }); +}); + +describe("buildSubagentDisabledProfile", () => { + test("drops the task tool and turns off the general-purpose subagent", () => { + const profile = buildSubagentDisabledProfile(); + expect(profile.excludedTools).toEqual(["task"]); + expect(profile.generalPurposeSubagent.enabled).toBe(false); + }); +}); + +describe("createOpenWikiMiddleware", () => { + test("default behavior is unchanged: no middleware when unset", () => { + const middleware = createOpenWikiMiddleware(); + expect(middleware).toEqual([]); + }); + + test("when disabled, the task tool is excluded from the model call", async () => { + process.env[ENV_KEY] = "1"; + const middleware = createOpenWikiMiddleware(); + const tools = [ + { name: "read_file" }, + { name: "task" }, + { name: "write_file" }, + ]; + + const seen = await toolsSeenByModel(middleware, tools); + + expect(seen.map((tool) => tool.name)).toEqual(["read_file", "write_file"]); + }); +});