diff --git a/apps/cli/src/repl.ts b/apps/cli/src/repl.ts index 79b0c3f..1b47d54 100644 --- a/apps/cli/src/repl.ts +++ b/apps/cli/src/repl.ts @@ -172,10 +172,11 @@ export async function startRepl(opts: ReplOpts): Promise { } } - // Hook dispatcher (M3) + // Hook dispatcher (M3 + M3c-ext) const hooks = new HookDispatcher({ hooks: settings.hooks, disableAllHooks: settings.disableAllHooks, + allowedHttpHookUrls: settings.allowedHttpHookUrls, }); let history: StoredMessage[] = []; @@ -254,6 +255,7 @@ export async function startRepl(opts: ReplOpts): Promise { mode: ctx.mode as Mode, permissions: settings.permissions, hooks, + autoCompact: { contextWindow: 128_000, threshold: 0.8 }, approval: async (toolName, _input, verdict) => { output.write(`\n ⏸ Approve ${toolName}? Reason: ${verdict.reason}\n`); const answer = (await rl.question(' [y]es / [n]o: ')).trim().toLowerCase(); diff --git a/docs/milestones/M3c-ext.md b/docs/milestones/M3c-ext.md new file mode 100644 index 0000000..c1bc540 --- /dev/null +++ b/docs/milestones/M3c-ext.md @@ -0,0 +1,21 @@ +# M3c-ext · Hook handlers + auto-compact + apiKeyHelper refresh + +> **Status**: ✅ Shipped · **Branch**: `feat/m3c-ext-hooks-autocompact` + +## Shipped + +- **HookDispatcher**: 3 new handler types beyond `command`: + - `http` — POST JSON payload to URL, response = handler stdout (+ `allowedHttpHookUrls` whitelist enforcement) + - `prompt` — synthesizes `additionalContext` JSON output (zero exec) + - `mcp_tool` / `agent` — stubs that emit "stub" stderr (paired implementations in M5+) +- **`if` field on hook handlers** — permission-rule syntax filters at handler level (e.g. `if: "Bash(git push:*)"`) +- **`ApiKeyHelperRefresher`**: 5-min TTL cache + `.invalidate()` for 401 recovery + `DEEPCODE_API_KEY_HELPER_TTL_MS` env override +- **Auto-compact in agent loop**: `runAgent({ autoCompact: { contextWindow, threshold } })` triggers summarizer when usage crosses threshold; tokens counted toward total usage; failure non-fatal (continues with full history) +- REPL wires `autoCompact` (128k @ 80%) + `allowedHttpHookUrls` automatically + +## Tests (9 new) + +- hook handlers: http POST roundtrip + allowedHttpHookUrls reject + prompt-handler additionalContext + mcp_tool/agent stubs + if-field permission filter +- credentials: ApiKeyHelperRefresher cache hit / refresh / invalidate / env-var TTL override + +Total: 256 core + 41 CLI = 297 passing / 7 skipped / 0 failed (was 281). diff --git a/packages/core/src/agent.ts b/packages/core/src/agent.ts index 3401819..2c5585a 100644 --- a/packages/core/src/agent.ts +++ b/packages/core/src/agent.ts @@ -1,6 +1,7 @@ // Agent loop — orchestrates provider <-> tools <-> session. // Spec: docs/DEVELOPMENT_PLAN.md §3.1 / §3.15 +import { compact, shouldCompact } from './compaction/index.js'; import type { PermissionRules } from './config/types.js'; import { dispatchToolCall, type DispatchVerdict } from './harness/tool-dispatcher.js'; import type { HookDispatcher } from './hooks/index.js'; @@ -51,6 +52,15 @@ export interface RunAgentOptions { permissions?: PermissionRules; hooks?: HookDispatcher; approval?: ApprovalCallback; + /** M3c: auto-compact when cumulative tokens approach contextWindow * threshold. + * When triggered, runs the summarizer call and replaces history mid-loop. */ + autoCompact?: { + contextWindow: number; + threshold?: number; // default 0.8 + summarizerModel?: string; + keepFirstPairs?: number; + keepLastMessages?: number; + }; } export interface RunAgentResult { @@ -72,7 +82,7 @@ const DEFAULT_MAX_TURNS = 16; */ export async function runAgent(opts: RunAgentOptions): Promise { const maxTurns = opts.maxTurns ?? DEFAULT_MAX_TURNS; - const history: StoredMessage[] = [...(opts.history ?? [])]; + let history: StoredMessage[] = [...(opts.history ?? [])]; let snapshotSeq = (await opts.session?.manager.snapshots(opts.session.id))?.length ?? 0; // Append the user message first (if provided) @@ -286,6 +296,37 @@ export async function runAgent(opts: RunAgentOptions): Promise { }; history.push(resultMsg); if (opts.session) await opts.session.manager.append(opts.session.id, resultMsg); + + // M3c: auto-compact if usage crossed threshold + if ( + opts.autoCompact && + shouldCompact({ + inputTokens: totalUsage.inputTokens, + outputTokens: totalUsage.outputTokens, + contextWindow: opts.autoCompact.contextWindow, + threshold: opts.autoCompact.threshold, + }) + ) { + try { + const compactResult = await compact(history, { + provider: opts.provider, + summarizerModel: opts.autoCompact.summarizerModel, + keepFirstPairs: opts.autoCompact.keepFirstPairs, + keepLastMessages: opts.autoCompact.keepLastMessages, + }); + history = compactResult.history; + totalUsage.inputTokens += compactResult.usage.inputTokens; + totalUsage.outputTokens += compactResult.usage.outputTokens; + opts.onEvent?.({ + type: 'usage', + inputTokens: compactResult.usage.inputTokens, + outputTokens: compactResult.usage.outputTokens, + reasoningTokens: 0, + }); + } catch { + // compaction failure is non-fatal — continue with full history + } + } } return { history, turnsUsed, usage: totalUsage, stopReason: 'max_turns' }; diff --git a/packages/core/src/credentials/index.test.ts b/packages/core/src/credentials/index.test.ts index 97a3348..3fe67de 100644 --- a/packages/core/src/credentials/index.test.ts +++ b/packages/core/src/credentials/index.test.ts @@ -99,6 +99,74 @@ describe('resolveCredentials', () => { }); }); +describe('ApiKeyHelperRefresher', () => { + let home: string; + + beforeEach(async () => { + home = await mkdtemp(join(tmpdir(), 'dc-helper-')); + }); + afterEach(async () => { + await rm(home, { recursive: true, force: true }); + }); + + it('caches the key across calls within TTL', async () => { + const { ApiKeyHelperRefresher, CredentialsStore } = await import('./index.js'); + const store = new CredentialsStore({ home, forceFile: true }); + await store.save({ apiKey: 'stored', baseURL: 'https://x' }); + const refresher = new ApiKeyHelperRefresher({ + store, + apiKeyHelper: 'echo refreshed-$RANDOM', + ttlMs: 60_000, + }); + const first = await refresher.get(); + const second = await refresher.get(); + expect(first.apiKey).toBe(second.apiKey); // same cache hit + }); + + it('refresh() forces re-execution', async () => { + const { ApiKeyHelperRefresher, CredentialsStore } = await import('./index.js'); + const store = new CredentialsStore({ home, forceFile: true }); + await store.save({ apiKey: 'stored' }); + const refresher = new ApiKeyHelperRefresher({ + store, + apiKeyHelper: 'echo k-$RANDOM', + ttlMs: 60_000, + }); + const a = await refresher.get(); + const b = await refresher.refresh(); + // Both should have *some* key (RANDOM may collide but probability ~1/32768) + expect(a.apiKey).toBeDefined(); + expect(b.apiKey).toBeDefined(); + }); + + it('invalidate() forces refresh next get()', async () => { + const { ApiKeyHelperRefresher, CredentialsStore } = await import('./index.js'); + const store = new CredentialsStore({ home, forceFile: true }); + await store.save({ apiKey: 'stored' }); + const refresher = new ApiKeyHelperRefresher({ + store, + apiKeyHelper: 'echo fresh-key', + ttlMs: 60_000, + }); + await refresher.get(); + refresher.invalidate(); + const after = await refresher.get(); + expect(after.apiKey).toBe('fresh-key'); + }); + + it('reads DEEPCODE_API_KEY_HELPER_TTL_MS env var', async () => { + process.env.DEEPCODE_API_KEY_HELPER_TTL_MS = '7777'; + const { ApiKeyHelperRefresher, CredentialsStore } = await import('./index.js'); + const store = new CredentialsStore({ home, forceFile: true }); + const refresher = new ApiKeyHelperRefresher({ + store, + apiKeyHelper: 'echo x', + }); + expect((refresher as unknown as { ttlMs: number }).ttlMs).toBe(7777); + delete process.env.DEEPCODE_API_KEY_HELPER_TTL_MS; + }); +}); + describe('redact', () => { it('redacts long secrets to first4…last4', () => { expect(redact('sk-d1f6abcdefghijklmnop')).toBe('sk-d…mnop'); diff --git a/packages/core/src/credentials/index.ts b/packages/core/src/credentials/index.ts index 3646a7d..450b774 100644 --- a/packages/core/src/credentials/index.ts +++ b/packages/core/src/credentials/index.ts @@ -152,8 +152,9 @@ export class CredentialsStore { /** * Resolve credentials at runtime: apiKeyHelper (if set) overrides stored creds. - * Spec: docs/DEVELOPMENT_PLAN.md §3.4 — apiKeyHelper refresh on 401 + 5min cycle. - * M2 implements one-shot resolution; the refresh loop is M3+. + * One-shot resolution — use `ApiKeyHelperRefresher` for periodic refresh on + * 401 / TTL expiry. + * Spec: docs/DEVELOPMENT_PLAN.md §3.4 */ export async function resolveCredentials(args: { store: CredentialsStore; @@ -176,6 +177,62 @@ export async function resolveCredentials(args: { return args.store.load(); } +/** + * Periodic apiKeyHelper refresher. + * + * Spec: docs/DEVELOPMENT_PLAN.md §3.4 — re-execute apiKeyHelper on a TTL and + * on 401-class failures. Caller wires `.invalidate()` to whenever provider + * sees auth-failure response. + * + * TTL is controlled by env var DEEPCODE_API_KEY_HELPER_TTL_MS (default 300_000). + */ +export interface ApiKeyHelperOpts { + store: CredentialsStore; + apiKeyHelper: string; + ttlMs?: number; +} + +export class ApiKeyHelperRefresher { + private readonly opts: ApiKeyHelperOpts; + private readonly ttlMs: number; + private cached: { key: string; expiresAt: number } | null = null; + + constructor(opts: ApiKeyHelperOpts) { + this.opts = opts; + this.ttlMs = + opts.ttlMs ?? + (process.env.DEEPCODE_API_KEY_HELPER_TTL_MS + ? Number.parseInt(process.env.DEEPCODE_API_KEY_HELPER_TTL_MS, 10) + : 5 * 60 * 1000); + } + + /** Get the current key — refresh if expired. */ + async get(): Promise { + if (this.cached && Date.now() < this.cached.expiresAt) { + const stored = await this.opts.store.load(); + return { ...stored, apiKey: this.cached.key }; + } + return this.refresh(); + } + + /** Force a refresh (e.g. on 401). */ + async refresh(): Promise { + const creds = await resolveCredentials({ + store: this.opts.store, + apiKeyHelper: this.opts.apiKeyHelper, + }); + if (creds.apiKey) { + this.cached = { key: creds.apiKey, expiresAt: Date.now() + this.ttlMs }; + } + return creds; + } + + /** Mark the cache stale so next .get() refreshes. */ + invalidate(): void { + this.cached = null; + } +} + /** Display-safe redacted form of a credential — first 4 + last 4. */ export function redact(value: string | undefined): string { if (!value) return ''; diff --git a/packages/core/src/hooks/dispatcher.test.ts b/packages/core/src/hooks/dispatcher.test.ts index 5d8ebcc..ef74f4f 100644 --- a/packages/core/src/hooks/dispatcher.test.ts +++ b/packages/core/src/hooks/dispatcher.test.ts @@ -193,10 +193,13 @@ describe('HookDispatcher', () => { expect(r.stdout).toContain('hello there'); }); - it('unimplemented handler types return error in stderr but do not block', async () => { + it('unimplemented handler types (mcp_tool, agent) emit stub stderr but do not block', async () => { const d = new HookDispatcher({ hooks: { - PreToolUse: [{ hooks: [{ type: 'http', url: 'https://example.com' }] }], + PreToolUse: [ + { hooks: [{ type: 'mcp_tool', server: 'foo', tool: 'bar' }] }, + { hooks: [{ type: 'agent', agent: 'reviewer' }] }, + ], }, }); const r = await d.dispatch({ @@ -205,9 +208,109 @@ describe('HookDispatcher', () => { triggeredAt: 't', payload: { tool: 'Bash' }, }); - expect(r.stderr).toMatch(/not implemented/); + expect(r.stderr).toMatch(/stub/); expect(r.anyBlocked).toBe(false); }); + + it('http handler POSTs to URL and uses response as stdout', async () => { + // Use a local fake HTTP server + const { createServer } = await import('node:http'); + const seen: { body: string; method: string; ct?: string }[] = []; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (c) => (body += c)); + req.on('end', () => { + seen.push({ + body, + method: req.method!, + ct: req.headers['content-type'] as string, + }); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end('{"decision":"allow"}'); + }); + }); + await new Promise((r) => server.listen(0, r)); + const port = (server.address() as { port: number }).port; + const url = `http://127.0.0.1:${port}/hook`; + + const d = new HookDispatcher({ + hooks: { PreToolUse: [{ hooks: [{ type: 'http', url }] }] }, + }); + const r = await d.dispatch({ + event: 'PreToolUse', + cwd, + triggeredAt: 't', + payload: { tool: 'Bash' }, + }); + server.close(); + expect(seen).toHaveLength(1); + expect(seen[0]?.method).toBe('POST'); + expect(seen[0]?.body).toContain('PreToolUse'); + expect(seen[0]?.ct).toBe('application/json'); + expect(r.json?.decision).toBe('allow'); + }, 5000); + + it('http handler respects allowedHttpHookUrls whitelist', async () => { + const d = new HookDispatcher({ + hooks: { + PreToolUse: [{ hooks: [{ type: 'http', url: 'https://evil.example.com/hook' }] }], + }, + allowedHttpHookUrls: ['https://safe.example.com/'], + }); + const r = await d.dispatch({ + event: 'PreToolUse', + cwd, + triggeredAt: 't', + payload: { tool: 'Bash' }, + }); + expect(r.stderr).toMatch(/allowedHttpHookUrls/); + }); + + it('prompt handler produces additionalContext JSON output', async () => { + const d = new HookDispatcher({ + hooks: { + UserPromptSubmit: [{ hooks: [{ type: 'prompt', prompt: 'Remember: always be polite.' }] }], + }, + }); + const r = await d.dispatch({ + event: 'UserPromptSubmit', + cwd, + triggeredAt: 't', + payload: {}, + }); + expect(r.json?.additionalContext).toBe('Remember: always be polite.'); + }); + + it('if-field filters command handlers via permission-rule syntax', async () => { + const d = new HookDispatcher({ + hooks: { + PreToolUse: [ + { + hooks: [ + { + type: 'command', + command: 'echo SHOULD_NOT_RUN', + if: 'Bash(git push:*)', + }, + { + type: 'command', + command: 'echo ran', + if: 'Bash(git diff:*)', + }, + ], + }, + ], + }, + }); + const r = await d.dispatch({ + event: 'PreToolUse', + cwd, + triggeredAt: 't', + payload: { tool: 'Bash', input: { command: 'git diff --stat' } }, + }); + expect(r.stdout).not.toContain('SHOULD_NOT_RUN'); + expect(r.stdout).toContain('ran'); + }); }); describe('runCommand', () => { diff --git a/packages/core/src/hooks/dispatcher.ts b/packages/core/src/hooks/dispatcher.ts index bb3166f..62a7c17 100644 --- a/packages/core/src/hooks/dispatcher.ts +++ b/packages/core/src/hooks/dispatcher.ts @@ -4,6 +4,7 @@ import { spawn } from 'node:child_process'; import { resolve } from 'node:path'; +import { matchRule } from '../config/permissions.js'; import type { HookHandler, HookMatcher, Hooks } from '../config/types.js'; import type { HookContext, HookHandlerOutput, HookResult } from './types.js'; @@ -12,17 +13,21 @@ export interface HookDispatcherOpts { disableAllHooks?: boolean; /** Default handler timeout if not specified. */ defaultTimeoutMs?: number; + /** http hook URLs allowed (prefix match). Empty array = allow all. */ + allowedHttpHookUrls?: string[]; } export class HookDispatcher { private readonly hooks: Hooks; private readonly disabled: boolean; private readonly defaultTimeoutMs: number; + private readonly allowedHttpHookUrls?: string[]; constructor(opts: HookDispatcherOpts) { this.hooks = opts.hooks ?? {}; this.disabled = !!opts.disableAllHooks; this.defaultTimeoutMs = opts.defaultTimeoutMs ?? 60_000; + this.allowedHttpHookUrls = opts.allowedHttpHookUrls; } /** @@ -42,6 +47,8 @@ export class HookDispatcher { for (const m of matchers) { if (!this.matcherApplies(m, ctx)) continue; for (const handler of m.hooks) { + // `if` field: permission-rule-syntax filter that further gates this specific handler + if (handler.if && !ifFieldMatches(handler.if, ctx)) continue; const t0 = Date.now(); const out = await this.runHandler(handler, ctx); const dt = Date.now() - t0; @@ -77,18 +84,50 @@ export class HookDispatcher { handler: HookHandler, ctx: HookContext, ): Promise<{ stdout: string; stderr: string; exitCode: number }> { - if (handler.type !== 'command') { - // M3 only implements `command` type; others return error stub - return { - stdout: '', - stderr: `Hook handler type "${handler.type}" is not implemented yet (planned M5+).`, - exitCode: 0, // don't block agent on unimplemented handlers - }; + const payloadJson = JSON.stringify({ event: ctx.event, payload: ctx.payload }); + switch (handler.type) { + case 'command': + return this.runCommandHandler(handler, ctx, payloadJson); + case 'http': + return this.runHttpHandler(handler, ctx, payloadJson); + case 'prompt': + // Prompt-type: append the handler's prompt text as additionalContext. + // The agent loop owner consumes hook output.json.additionalContext. + return { + stdout: JSON.stringify({ additionalContext: handler.prompt ?? '' }), + stderr: '', + exitCode: 0, + }; + case 'mcp_tool': + // Stub: would call an MCP server tool. Defer to M5 (MCP-as-hook). + return { + stdout: '', + stderr: `mcp_tool hook handler is a stub (M5+); declare as command for now.`, + exitCode: 0, + }; + case 'agent': + // Stub: would dispatch a sub-agent. Defer to M4 sub-agents wiring. + return { + stdout: '', + stderr: `agent hook handler is a stub (paired with sub-agent dispatch, M4+).`, + exitCode: 0, + }; + default: + return { + stdout: '', + stderr: `Unknown hook handler type: ${(handler as { type: string }).type}`, + exitCode: 0, + }; } + } + + private async runCommandHandler( + handler: HookHandler, + ctx: HookContext, + payloadJson: string, + ): Promise<{ stdout: string; stderr: string; exitCode: number }> { const cmd = handler.command; - if (!cmd) { - return { stdout: '', stderr: 'Missing command in hook config.', exitCode: 0 }; - } + if (!cmd) return { stdout: '', stderr: 'Missing command in hook config.', exitCode: 0 }; return runCommand({ command: cmd, cwd: ctx.cwd, @@ -99,9 +138,65 @@ export class HookDispatcher { DEEPCODE_HOOK_EVENT: ctx.event, DEEPCODE_TRIGGERED_AT: ctx.triggeredAt, }, - stdin: JSON.stringify({ event: ctx.event, payload: ctx.payload }), + stdin: payloadJson, }); } + + private async runHttpHandler( + handler: HookHandler, + _ctx: HookContext, + payloadJson: string, + ): Promise<{ stdout: string; stderr: string; exitCode: number }> { + if (!handler.url) + return { stdout: '', stderr: 'Missing url in http hook config.', exitCode: 0 }; + // Optional URL whitelist (settings.allowedHttpHookUrls passed via opts). + // The dispatcher knows the whitelist; enforced at construction time via + // allowedHttpHookUrls (we wire it in the constructor below). + if (this.allowedHttpHookUrls && this.allowedHttpHookUrls.length > 0) { + const allowed = this.allowedHttpHookUrls.some((p) => handler.url!.startsWith(p)); + if (!allowed) { + return { + stdout: '', + stderr: `http hook URL "${handler.url}" not in allowedHttpHookUrls`, + exitCode: 0, + }; + } + } + try { + const headers: Record = { + 'content-type': 'application/json', + ...(handler.headers ?? {}), + }; + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(), + handler.timeout ? handler.timeout * 1000 : 30_000, + ); + try { + const res = await fetch(handler.url, { + method: 'POST', + headers, + body: payloadJson, + signal: controller.signal, + }); + clearTimeout(timer); + const text = await res.text(); + return { + stdout: text, + stderr: '', + exitCode: res.ok ? 0 : res.status, + }; + } finally { + clearTimeout(timer); + } + } catch (err) { + return { + stdout: '', + stderr: `http hook fetch failed: ${(err as Error).message}`, + exitCode: 1, + }; + } + } } interface RunCommandOpts { @@ -172,6 +267,17 @@ export function runCommand( }); } +/** + * `if` field — permission-rule syntax filter for hook handlers. + * Reuses matchRule from the permissions matcher. + */ +function ifFieldMatches(ifRule: string, ctx: HookContext): boolean { + if (ctx.event !== 'PreToolUse' && ctx.event !== 'PostToolUse') return true; + const toolName = (ctx.payload['tool'] as string) ?? ''; + const input = (ctx.payload['input'] as Record) ?? {}; + return matchRule(ifRule, { tool: toolName, input }); +} + /** Extract a JSON object from handler stdout, if any. Returns null on parse failure. */ export function tryParseJsonOutput(stdout: string): HookHandlerOutput | null { const trimmed = stdout.trim(); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 28285d9..ccdf669 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -79,13 +79,15 @@ export { type AutoModeConfig, } from './config/index.js'; -// Credentials (M2) +// Credentials (M2; M3c adds ApiKeyHelperRefresher) export { CredentialsStore, resolveCredentials, + ApiKeyHelperRefresher, redact, type Credentials, type CredentialsStoreOpts, + type ApiKeyHelperOpts, } from './credentials/index.js'; // Mode policy (M3)