Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions packages/jimmy/src/cron/__tests__/catchup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
readCheckpoint,
writeCheckpoint,
lastRunAtFromDisk,
mostRecentRun,
} from "../catchup.js";
import type { CronJob } from "../../shared/types.js";

Expand Down Expand Up @@ -173,6 +174,48 @@ describe("checkpoint persistence", () => {
});
});

describe("mostRecentRun", () => {
it("returns the in-memory start when the disk log is empty", () => {
// An on-time fire still running: nothing on disk yet, but a start is tracked.
expect(mostRecentRun(null, ms("2026-06-03T01:00:00Z"))).toBe(
ms("2026-06-03T01:00:00Z"),
);
});

it("returns the disk run when there is no in-memory start", () => {
// Fresh process (restart): in-memory map empty, fall back to the run-log.
expect(mostRecentRun(ms("2026-06-03T01:00:00Z"), null)).toBe(
ms("2026-06-03T01:00:00Z"),
);
});

it("returns null when both are absent", () => {
expect(mostRecentRun(null, null)).toBeNull();
});

it("prefers the more recent of the two", () => {
const older = ms("2026-06-02T01:00:00Z");
const newer = ms("2026-06-03T01:00:00Z");
expect(mostRecentRun(older, newer)).toBe(newer);
expect(mostRecentRun(newer, older)).toBe(newer);
});

it("dedups a long on-time fire whose disk log is still yesterday's run", () => {
// The actual production bug: yesterday's completion on disk, today's fire
// started but not yet logged. The merged value must clear the dedup gate so
// computeMissedFires does NOT replay today's slot.
const yesterdayOnDisk = ms("2026-06-02T01:00:03Z");
const startedToday = ms("2026-06-03T01:00:01Z");
const merged = mostRecentRun(yesterdayOnDisk, startedToday);
const { replay } = computeMissedFires([job({})], {
...DEFAULTS,
lastCheck: ms("2026-06-03T00:30:00Z"),
lastRunAt: () => merged,
});
expect(replay).toHaveLength(0);
});
});

describe("lastRunAtFromDisk", () => {
it("returns the timestamp of the last run-log entry", () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "runs-"));
Expand Down
64 changes: 64 additions & 0 deletions packages/jimmy/src/cron/__tests__/precheck.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import { describe, it, expect } from "vitest";
import { runPrecheck, DEFAULT_PRECHECK_TIMEOUT_MS } from "../precheck.js";

// Pin cwd to a real dir: vitest sets JINN_HOME to an uncreated tmp path, and the
// default cwd would otherwise spawn-fail. Production cwd is the real ~/.jinn.
const run = (pc: Parameters<typeof runPrecheck>[0]) => runPrecheck(pc, { cwd: process.cwd() });

describe("runPrecheck — exit-code contract", () => {
it("decision=proceed when the command exits 0", async () => {
const r = await run({ command: "exit 0", skipExitCodes: [10] });
expect(r.decision).toBe("proceed");
expect(r.exitCode).toBe(0);
expect(r.timedOut).toBe(false);
});

it("decision=skip when the exit code is in skipExitCodes", async () => {
const r = await run({ command: "exit 10", skipExitCodes: [10, 20] });
expect(r.decision).toBe("skip");
expect(r.exitCode).toBe(10);
});

it("treats a second listed skip code (e.g. lock contention) as skip", async () => {
const r = await run({ command: "exit 20", skipExitCodes: [10, 20] });
expect(r.decision).toBe("skip");
expect(r.exitCode).toBe(20);
});

it("decision=error for a non-zero exit NOT in skipExitCodes (real failure not swallowed)", async () => {
// e.g. the jinn-watcher prefilter's 21 = wacli outage — must surface, not skip.
const r = await run({ command: "exit 21", skipExitCodes: [10, 20] });
expect(r.decision).toBe("error");
expect(r.exitCode).toBe(21);
});

it("with no skipExitCodes, ANY non-zero exit is an error (never skips blind)", async () => {
const r = await run({ command: "exit 1" });
expect(r.decision).toBe("error");
expect(r.exitCode).toBe(1);
});

it("decision=error and timedOut=true when the command exceeds timeoutMs", async () => {
const r = await run({ command: "sleep 5", skipExitCodes: [10], timeoutMs: 150 });
expect(r.decision).toBe("error");
expect(r.timedOut).toBe(true);
expect(r.exitCode).toBeNull();
});

it("decision=error on invalid config (missing/empty command)", async () => {
const r = await run({ command: "" });
expect(r.decision).toBe("error");
expect(r.exitCode).toBeNull();
expect(r.stderr).toMatch(/missing or empty command/);
});

it("captures stdout from the gate", async () => {
const r = await run({ command: "echo hello; exit 0" });
expect(r.decision).toBe("proceed");
expect(r.stdout).toMatch(/hello/);
});

it("default timeout constant is exported and sane", () => {
expect(DEFAULT_PRECHECK_TIMEOUT_MS).toBeGreaterThan(0);
});
});
95 changes: 95 additions & 0 deletions packages/jimmy/src/cron/__tests__/runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ vi.mock("../../shared/logger.js", () => ({
},
}));

// Stub the precheck gate (its own exit-code logic is unit-tested in precheck.test.ts);
// here we control its decision to assert how runCronJob branches.
vi.mock("../precheck.js", () => ({ runPrecheck: vi.fn() }));

// Stub ops alert so a precheck_error doesn't try to reach Telegram.
vi.mock("../../shared/ops-alert.js", () => ({ opsAlert: vi.fn().mockResolvedValue(undefined) }));

function makeJob(overrides: Partial<CronJob> = {}): CronJob {
return {
id: "test-job",
Expand Down Expand Up @@ -180,3 +187,91 @@ describe("runCronJob — latency alerting", () => {
expect(alertMsg).toContain("failed");
});
});

describe("runCronJob — precheck gate", () => {
beforeEach(async () => {
// Clear accumulated call history on the module-mocked fns and re-arm async ones.
const { runPrecheck } = await import("../precheck.js");
const { opsAlert } = await import("../../shared/ops-alert.js");
const { appendRunLog } = await import("../jobs.js");
(runPrecheck as any).mockReset();
(opsAlert as any).mockReset().mockResolvedValue(undefined);
(appendRunLog as any).mockClear();
});

const withPrecheck = (overrides = {}) =>
makeJob({ precheck: { command: "x", skipExitCodes: [10] }, ...overrides });

it("spawns the session when precheck decides proceed", async () => {
const { runPrecheck } = await import("../precheck.js");
(runPrecheck as any).mockResolvedValue({ decision: "proceed", exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", durationMs: 5 });
const sessionManager = makeMockSessionManager(0);
const connectors = new Map<string, Connector>([["slack", makeMockConnector()]]);

await runCronJob(withPrecheck(), sessionManager, makeConfig(), connectors);

expect(sessionManager.route).toHaveBeenCalledTimes(1);
});

it("does NOT spawn a session on skip, and logs gated-skip (no alert)", async () => {
const { appendRunLog } = await import("../jobs.js");
const { runPrecheck } = await import("../precheck.js");
const { opsAlert } = await import("../../shared/ops-alert.js");
(runPrecheck as any).mockResolvedValue({ decision: "skip", exitCode: 10, signal: null, timedOut: false, stdout: "", stderr: "", durationMs: 5 });
const sessionManager = makeMockSessionManager(0);
const connector = makeMockConnector();
const connectors = new Map<string, Connector>([["slack", connector]]);

await runCronJob(withPrecheck(), sessionManager, makeConfig(), connectors);

expect(sessionManager.route).not.toHaveBeenCalled();
expect(opsAlert).not.toHaveBeenCalled();
expect(connector.sendMessage).not.toHaveBeenCalled();
expect(appendRunLog).toHaveBeenCalledWith(
"test-job",
expect.objectContaining({ status: "gated-skip" }),
);
});

it("does NOT spawn a session on precheck error, logs precheck_error AND ops-alerts", async () => {
const { appendRunLog } = await import("../jobs.js");
const { runPrecheck } = await import("../precheck.js");
const { opsAlert } = await import("../../shared/ops-alert.js");
(runPrecheck as any).mockResolvedValue({ decision: "error", exitCode: 21, signal: null, timedOut: false, stdout: "", stderr: "wacli down", durationMs: 5 });
const sessionManager = makeMockSessionManager(0);
const connectors = new Map<string, Connector>([["slack", makeMockConnector()]]);

await runCronJob(withPrecheck(), sessionManager, makeConfig(), connectors);

expect(sessionManager.route).not.toHaveBeenCalled();
expect(opsAlert).toHaveBeenCalledTimes(1);
expect(appendRunLog).toHaveBeenCalledWith(
"test-job",
expect.objectContaining({ status: "precheck_error" }),
);
});

it("treats a precheck timeout as an error (no session)", async () => {
const { runPrecheck } = await import("../precheck.js");
const { opsAlert } = await import("../../shared/ops-alert.js");
(runPrecheck as any).mockResolvedValue({ decision: "error", exitCode: null, signal: "SIGTERM", timedOut: true, stdout: "", stderr: "", durationMs: 60000 });
const sessionManager = makeMockSessionManager(0);
const connectors = new Map<string, Connector>([["slack", makeMockConnector()]]);

await runCronJob(withPrecheck(), sessionManager, makeConfig(), connectors);

expect(sessionManager.route).not.toHaveBeenCalled();
expect(opsAlert).toHaveBeenCalledTimes(1);
});

it("ignores precheck entirely for jobs without a precheck field (backward compat)", async () => {
const { runPrecheck } = await import("../precheck.js");
const sessionManager = makeMockSessionManager(0);
const connectors = new Map<string, Connector>([["slack", makeMockConnector()]]);

await runCronJob(makeJob(), sessionManager, makeConfig(), connectors);

expect(runPrecheck).not.toHaveBeenCalled();
expect(sessionManager.route).toHaveBeenCalledTimes(1);
});
});
16 changes: 16 additions & 0 deletions packages/jimmy/src/cron/catchup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,22 @@ export function computeMissedFires(
return { replay, tooOld };
}

/**
* Combine a job's on-disk last-run with its in-memory last-start (either may be
* null), returning the more recent. The in-memory start covers an on-time fire
* that is still running and has not yet written its completion entry to the
* run-log — without it, catch-up would replay a slot the live scheduler already
* fired. See runner.ts `lastStartedAt` for why the start is tracked in memory.
*/
export function mostRecentRun(
disk: number | null,
started: number | null,
): number | null {
if (disk === null) return started;
if (started === null) return disk;
return Math.max(disk, started);
}

/** Read the persisted last-sweep checkpoint (ms epoch), or null. */
export function readCheckpoint(file: string): number | null {
try {
Expand Down
90 changes: 90 additions & 0 deletions packages/jimmy/src/cron/precheck.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { execFile } from "node:child_process";
import type { CronJob } from "../shared/types.js";
import { JINN_HOME } from "../shared/paths.js";

export type PrecheckDecision = "proceed" | "skip" | "error";

export interface PrecheckResult {
decision: PrecheckDecision;
/** Numeric exit code, or null if the process never exited cleanly (timeout / spawn error). */
exitCode: number | null;
/** Signal that terminated the process, if any (e.g. "SIGTERM" on timeout). */
signal: string | null;
timedOut: boolean;
stdout: string;
stderr: string;
durationMs: number;
}

export const DEFAULT_PRECHECK_TIMEOUT_MS = 60_000;
const MAX_OUTPUT_BYTES = 1024 * 1024;

/**
* Run a cron job's `precheck` gate and classify the outcome. NEVER rejects —
* every failure mode resolves to a result with decision "error" so the caller
* can branch deterministically.
*
* Contract (see CronJob.precheck): exit 0 → proceed; exit ∈ skipExitCodes →
* skip; any other non-zero (or timeout / spawn error / missing command) → error.
*/
export function runPrecheck(
precheck: NonNullable<CronJob["precheck"]>,
opts?: { cwd?: string; env?: NodeJS.ProcessEnv },
): Promise<PrecheckResult> {
const start = Date.now();
return new Promise<PrecheckResult>((resolve) => {
const command = precheck?.command;
if (typeof command !== "string" || command.trim() === "") {
resolve({
decision: "error",
exitCode: null,
signal: null,
timedOut: false,
stdout: "",
stderr: "precheck: missing or empty command",
durationMs: Date.now() - start,
});
return;
}
const timeoutMs = precheck.timeoutMs ?? DEFAULT_PRECHECK_TIMEOUT_MS;
const skipExitCodes = precheck.skipExitCodes ?? [];

execFile(
"/bin/bash",
["-c", command],
{
cwd: opts?.cwd ?? JINN_HOME,
env: opts?.env ?? process.env,
timeout: timeoutMs,
maxBuffer: MAX_OUTPUT_BYTES,
},
(err, stdout, stderr) => {
const durationMs = Date.now() - start;
const out = (stdout ?? "").toString();
const errOut = (stderr ?? "").toString();

if (!err) {
resolve({ decision: "proceed", exitCode: 0, signal: null, timedOut: false, stdout: out, stderr: errOut, durationMs });
return;
}

const e = err as NodeJS.ErrnoException & { code?: number | string; killed?: boolean; signal?: string };
const signal = e.signal ?? null;
// execFile sets killed=true and a termination signal when the timeout fires.
const timedOut = e.killed === true && (signal === "SIGTERM" || signal === "SIGKILL");
const exitCode = typeof e.code === "number" ? e.code : null;

if (timedOut) {
resolve({ decision: "error", exitCode: null, signal, timedOut: true, stdout: out, stderr: errOut, durationMs });
return;
}
if (exitCode !== null && skipExitCodes.includes(exitCode)) {
resolve({ decision: "skip", exitCode, signal: null, timedOut: false, stdout: out, stderr: errOut, durationMs });
return;
}
// Non-zero exit not on the skip list, or a spawn error (ENOENT etc.) → fail loud.
resolve({ decision: "error", exitCode, signal, timedOut: false, stdout: out, stderr: errOut, durationMs });
},
);
});
}
Loading
Loading