Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions packages/jimmy/src/cron/__tests__/runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ vi.mock("../precheck.js", () => ({ runPrecheck: vi.fn() }));
// Stub ops alert so a precheck_error doesn't try to reach Telegram.
vi.mock("../../shared/ops-alert.js", () => ({ opsAlert: vi.fn().mockResolvedValue(undefined) }));

// Stub the session registry. Default returns undefined (no session record),
// which matches the prior behaviour of the success-path tests. Individual tests
// override getSession to inject a finished session whose status the runner
// inspects (e.g. status:"error" for the silent-spawn-failure regression).
vi.mock("../../sessions/registry.js", () => ({ getSession: vi.fn(() => undefined) }));

function makeJob(overrides: Partial<CronJob> = {}): CronJob {
return {
id: "test-job",
Expand Down Expand Up @@ -275,3 +281,62 @@ describe("runCronJob — precheck gate", () => {
expect(sessionManager.route).toHaveBeenCalledTimes(1);
});
});

describe("runCronJob — session ended in error without route() throwing", () => {
// Regression for the Jun-2026 silent cron outage: a moved Claude binary made
// the engine reject ("Failed to spawn Claude CLI ... ENOENT"); the session
// manager caught it, set session.status="error", and route() STILL resolved
// with a sessionId. The runner used to log "completed"/"success" with no alert.
// ~400 fires went dark for days. The runner must now treat an errored finished
// session as a cron failure: status:"error" in the run-log + an ops-alert.
beforeEach(async () => {
const { opsAlert } = await import("../../shared/ops-alert.js");
const { appendRunLog } = await import("../jobs.js");
const { getSession } = await import("../../sessions/registry.js");
(opsAlert as any).mockReset().mockResolvedValue(undefined);
(appendRunLog as any).mockClear();
(getSession as any).mockReset().mockReturnValue(undefined);
});

it("logs status 'error' and fires an ops-alert when the session status is error", async () => {
const { appendRunLog } = await import("../jobs.js");
const { opsAlert } = await import("../../shared/ops-alert.js");
const { getSession } = await import("../../sessions/registry.js");
const spawnErr = "Failed to spawn Claude CLI: spawn /bad/path/claude ENOENT";
(getSession as any).mockReturnValue({ status: "error", lastError: spawnErr });

const sessionManager = makeMockSessionManager(0); // route resolves { sessionId: "sess-123" }
const connector = makeMockConnector();
const connectors = new Map<string, Connector>([["slack", connector]]);

await runCronJob(makeJob(), sessionManager, makeConfig(), connectors);

// run-log records the failure (not a false "success")
expect(appendRunLog).toHaveBeenCalledWith(
"test-job",
expect.objectContaining({ status: "error", error: spawnErr }),
);
// and an ops-alert fires carrying the underlying error
expect(opsAlert).toHaveBeenCalledTimes(1);
expect((opsAlert as any).mock.calls[0][0]).toContain("FAILED");
expect((opsAlert as any).mock.calls[0][0]).toContain("ENOENT");
});

it("still logs 'success' for a normal finished session (no false positive)", async () => {
const { appendRunLog } = await import("../jobs.js");
const { opsAlert } = await import("../../shared/ops-alert.js");
const { getSession } = await import("../../sessions/registry.js");
(getSession as any).mockReturnValue({ status: "idle", lastError: null });

const sessionManager = makeMockSessionManager(0);
const connectors = new Map<string, Connector>([["slack", makeMockConnector()]]);

await runCronJob(makeJob(), sessionManager, makeConfig(), connectors);

expect(appendRunLog).toHaveBeenCalledWith(
"test-job",
expect.objectContaining({ status: "success" }),
);
expect(opsAlert).not.toHaveBeenCalled();
});
});
29 changes: 26 additions & 3 deletions packages/jimmy/src/cron/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,26 @@ export async function runCronJob(
const durationMs = Date.now() - startTime;
const finalSession = routeResult?.sessionId ? getSession(routeResult.sessionId) : undefined;
const budgetStopped = !!finalSession?.lastError?.startsWith(SESSION_BUDGET_STOP_PREFIX);
// A session can end in ERROR without route() throwing: the engine rejects
// (e.g. "Failed to spawn Claude CLI" / "engine never started"), the session
// manager catches it, records session.status="error"+lastError, and route()
// still RESOLVES with a sessionId. Without this check those land as
// "success"/"completed" with no alert — the exact failure mode behind the
// Jun-2026 silent cron outage (a moved Claude binary ENOENT'd ~400 fires,
// every one logged "completed in 16ms"). Treat a non-budget session error as
// a cron failure: record it and fire an ops-alert.
const sessionErrored = !budgetStopped && finalSession?.status === "error";
appendRunLog(job.id, {
timestamp: startedAt,
sessionKey,
sessionId: routeResult?.sessionId ?? null,
status: budgetStopped ? "session_budget_stop" : "success",
status: budgetStopped ? "session_budget_stop" : sessionErrored ? "error" : "success",
durationMs,
error: budgetStopped ? finalSession?.lastError ?? null : null,
error: budgetStopped
? finalSession?.lastError ?? null
: sessionErrored
? finalSession?.lastError ?? "session ended in error"
: null,
...(budgetStopped ? { maxTurns: budget.maxTurns, actualTurns: finalSession?.totalTurns ?? null } : {}),
...catchUpFields,
resultPreview: null,
Expand All @@ -183,7 +196,17 @@ export async function runCronJob(
`after ~${finalSession?.totalTurns ?? "?"} turns. This job is flagged sideEffects:true — check for partial external writes.`,
).catch(() => {});
}
logger.info(`Cron job "${job.name}" ${budgetStopped ? "stopped at turn budget" : "completed"} in ${durationMs}ms`);
if (sessionErrored) {
logger.error(
`Cron job "${job.name}" (${job.id}) session ended in error in ${durationMs}ms: ${finalSession?.lastError ?? "(no message)"}`,
);
await opsAlert(
`Cron "${job.name}" (${job.id}) FAILED — session ended in error (engine never produced a result, no exception thrown). ` +
`${finalSession?.lastError?.slice(0, 300) ?? "(no error message)"}`,
).catch(() => {});
} else {
logger.info(`Cron job "${job.name}" ${budgetStopped ? "stopped at turn budget" : "completed"} in ${durationMs}ms`);
}

// Latency alert: warn if job exceeded threshold
const thresholdMs = config.cron?.alertThresholdMs;
Expand Down
Loading