diff --git a/packages/jimmy/src/cron/__tests__/runner.test.ts b/packages/jimmy/src/cron/__tests__/runner.test.ts index 69759968..9b001e6b 100644 --- a/packages/jimmy/src/cron/__tests__/runner.test.ts +++ b/packages/jimmy/src/cron/__tests__/runner.test.ts @@ -30,6 +30,12 @@ vi.mock("../precheck.js", () => ({ runPrecheck: vi.fn() })); // Stub ops alert so a precheck_error doesn't try to reach Telegram. vi.mock("../../shared/ops-alert.js", () => ({ opsAlert: vi.fn().mockResolvedValue(undefined) })); +// Stub the session registry. Default returns undefined (no session record), +// which matches the prior behaviour of the success-path tests. Individual tests +// override getSession to inject a finished session whose status the runner +// inspects (e.g. status:"error" for the silent-spawn-failure regression). +vi.mock("../../sessions/registry.js", () => ({ getSession: vi.fn(() => undefined) })); + function makeJob(overrides: Partial = {}): CronJob { return { id: "test-job", @@ -275,3 +281,62 @@ describe("runCronJob — precheck gate", () => { expect(sessionManager.route).toHaveBeenCalledTimes(1); }); }); + +describe("runCronJob — session ended in error without route() throwing", () => { + // Regression for the Jun-2026 silent cron outage: a moved Claude binary made + // the engine reject ("Failed to spawn Claude CLI ... ENOENT"); the session + // manager caught it, set session.status="error", and route() STILL resolved + // with a sessionId. The runner used to log "completed"/"success" with no alert. + // ~400 fires went dark for days. The runner must now treat an errored finished + // session as a cron failure: status:"error" in the run-log + an ops-alert. + beforeEach(async () => { + const { opsAlert } = await import("../../shared/ops-alert.js"); + const { appendRunLog } = await import("../jobs.js"); + const { getSession } = await import("../../sessions/registry.js"); + (opsAlert as any).mockReset().mockResolvedValue(undefined); + (appendRunLog as any).mockClear(); + (getSession as any).mockReset().mockReturnValue(undefined); + }); + + it("logs status 'error' and fires an ops-alert when the session status is error", async () => { + const { appendRunLog } = await import("../jobs.js"); + const { opsAlert } = await import("../../shared/ops-alert.js"); + const { getSession } = await import("../../sessions/registry.js"); + const spawnErr = "Failed to spawn Claude CLI: spawn /bad/path/claude ENOENT"; + (getSession as any).mockReturnValue({ status: "error", lastError: spawnErr }); + + const sessionManager = makeMockSessionManager(0); // route resolves { sessionId: "sess-123" } + const connector = makeMockConnector(); + const connectors = new Map([["slack", connector]]); + + await runCronJob(makeJob(), sessionManager, makeConfig(), connectors); + + // run-log records the failure (not a false "success") + expect(appendRunLog).toHaveBeenCalledWith( + "test-job", + expect.objectContaining({ status: "error", error: spawnErr }), + ); + // and an ops-alert fires carrying the underlying error + expect(opsAlert).toHaveBeenCalledTimes(1); + expect((opsAlert as any).mock.calls[0][0]).toContain("FAILED"); + expect((opsAlert as any).mock.calls[0][0]).toContain("ENOENT"); + }); + + it("still logs 'success' for a normal finished session (no false positive)", async () => { + const { appendRunLog } = await import("../jobs.js"); + const { opsAlert } = await import("../../shared/ops-alert.js"); + const { getSession } = await import("../../sessions/registry.js"); + (getSession as any).mockReturnValue({ status: "idle", lastError: null }); + + const sessionManager = makeMockSessionManager(0); + const connectors = new Map([["slack", makeMockConnector()]]); + + await runCronJob(makeJob(), sessionManager, makeConfig(), connectors); + + expect(appendRunLog).toHaveBeenCalledWith( + "test-job", + expect.objectContaining({ status: "success" }), + ); + expect(opsAlert).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/jimmy/src/cron/runner.ts b/packages/jimmy/src/cron/runner.ts index bb77d0ad..14f7e2e9 100644 --- a/packages/jimmy/src/cron/runner.ts +++ b/packages/jimmy/src/cron/runner.ts @@ -166,13 +166,26 @@ export async function runCronJob( const durationMs = Date.now() - startTime; const finalSession = routeResult?.sessionId ? getSession(routeResult.sessionId) : undefined; const budgetStopped = !!finalSession?.lastError?.startsWith(SESSION_BUDGET_STOP_PREFIX); + // A session can end in ERROR without route() throwing: the engine rejects + // (e.g. "Failed to spawn Claude CLI" / "engine never started"), the session + // manager catches it, records session.status="error"+lastError, and route() + // still RESOLVES with a sessionId. Without this check those land as + // "success"/"completed" with no alert — the exact failure mode behind the + // Jun-2026 silent cron outage (a moved Claude binary ENOENT'd ~400 fires, + // every one logged "completed in 16ms"). Treat a non-budget session error as + // a cron failure: record it and fire an ops-alert. + const sessionErrored = !budgetStopped && finalSession?.status === "error"; appendRunLog(job.id, { timestamp: startedAt, sessionKey, sessionId: routeResult?.sessionId ?? null, - status: budgetStopped ? "session_budget_stop" : "success", + status: budgetStopped ? "session_budget_stop" : sessionErrored ? "error" : "success", durationMs, - error: budgetStopped ? finalSession?.lastError ?? null : null, + error: budgetStopped + ? finalSession?.lastError ?? null + : sessionErrored + ? finalSession?.lastError ?? "session ended in error" + : null, ...(budgetStopped ? { maxTurns: budget.maxTurns, actualTurns: finalSession?.totalTurns ?? null } : {}), ...catchUpFields, resultPreview: null, @@ -183,7 +196,17 @@ export async function runCronJob( `after ~${finalSession?.totalTurns ?? "?"} turns. This job is flagged sideEffects:true — check for partial external writes.`, ).catch(() => {}); } - logger.info(`Cron job "${job.name}" ${budgetStopped ? "stopped at turn budget" : "completed"} in ${durationMs}ms`); + if (sessionErrored) { + logger.error( + `Cron job "${job.name}" (${job.id}) session ended in error in ${durationMs}ms: ${finalSession?.lastError ?? "(no message)"}`, + ); + await opsAlert( + `Cron "${job.name}" (${job.id}) FAILED — session ended in error (engine never produced a result, no exception thrown). ` + + `${finalSession?.lastError?.slice(0, 300) ?? "(no error message)"}`, + ).catch(() => {}); + } else { + logger.info(`Cron job "${job.name}" ${budgetStopped ? "stopped at turn budget" : "completed"} in ${durationMs}ms`); + } // Latency alert: warn if job exceeded threshold const thresholdMs = config.cron?.alertThresholdMs;