From 1dcb028cb453fd8042de663fbdfd95604a057fff Mon Sep 17 00:00:00 2001 From: Binlogo Date: Fri, 29 May 2026 18:37:48 +0800 Subject: [PATCH] Add system cron job observability debug endpoint (AXON-692) GET /api/debug/system-cron-jobs lists schedule_followup-created system cron jobs (filtered out of the user-facing /api/cron/jobs) with each job's recent RunRecord history. Supports thread_id and since filters and a configurable runs_limit. POST .../{id}/run is a system-only wrapper around CronService::run_now (non-system or missing jobs return 404). Routes live under the protected router so enforce_gateway_auth gates them: loopback passes, otherwise a valid gateway token is required. No separate debug-token surface is introduced. Existing /api/cron/jobs and /api/cron/runs behavior is unchanged (additive, read-only). Docs: new docs/schedule-followup-observability.md, cross-linked from schedule-followup.md. --- docs/schedule-followup-observability.md | 135 +++++++++++++ docs/schedule-followup.md | 8 + garyx-gateway/src/api.rs | 230 +++++++++++++++++++++++ garyx-gateway/src/api/tests.rs | 239 ++++++++++++++++++++++++ garyx-gateway/src/route_graph.rs | 11 ++ 5 files changed, 623 insertions(+) create mode 100644 docs/schedule-followup-observability.md diff --git a/docs/schedule-followup-observability.md b/docs/schedule-followup-observability.md new file mode 100644 index 00000000..d91b199a --- /dev/null +++ b/docs/schedule-followup-observability.md @@ -0,0 +1,135 @@ +# `schedule_followup` observability + +`schedule_followup` (see [schedule-followup.md](./schedule-followup.md)) creates +its cron jobs with `system: true`, which keeps them out of the user-facing +automation list (`GET /api/cron/jobs` filters `system` jobs out). That is the +right default — these are internal self-wake schedules, not automations a user +manages — but it leaves a blind spot during incidents like *"the agent promised +a followup and it never came back."* + +The debug endpoint `GET /api/debug/system-cron-jobs` closes that gap: it lists +the system cron jobs (including `schedule_followup`-created followups) together +with each job's recent `RunRecord` history, and offers a system-only manual +fire. + +## Authorization + +The debug routes live under the gateway's **protected** router, so they inherit +`enforce_gateway_auth`: + +- Requests from **loopback** (`127.0.0.1` / `::1`) pass without a token. On the + gateway host, `curl http://127.0.0.1:/api/debug/system-cron-jobs` just + works. +- Any **non-loopback** request must carry a valid gateway token (the same token + used for the rest of the gateway API — `Authorization: Bearer `, the + `x-garyx-token` header, or `?token=`). No token / wrong token → `401`. + +This reuses the existing gateway auth token rather than introducing a separate +debug-token config surface. The endpoint is never exposed unauthenticated to +external callers. + +## `GET /api/debug/system-cron-jobs` + +Lists every cron job with `system == true`, plus each job's recent runs. + +### Query parameters + +| Param | Type | Default | Notes | +|---|---|---|---| +| `thread_id` | string | — | Exact match on the job's `thread_id`. Empty / whitespace-only is ignored (returns all system jobs). | +| `since` | string | — | Lower bound on the job's `created_at`. Accepts a **unix-second timestamp** (all digits) or an **RFC3339** datetime. Jobs created strictly before this instant are filtered out. A value that parses as neither form returns `400 invalid_since` — it is never silently treated as "no filter". | +| `runs_limit` | integer | `20` | Max recent `RunRecord`s attached per job, most-recent-first. | + +### Response + +```json +{ + "jobs": [ + { + "id": "followup_4d7c5b8f12ab9e3a", + "label": "schedule_followup(thread::abc)", + "kind": { + "type": "internal_dispatch", + "reason": "background build finished", + "originating_run_id": "run-...", + "scheduled_at": "2026-05-29T07:25:00+00:00", + "delay_seconds_requested": 300 + }, + "schedule": { "once": { "at": "2026-05-29T07:30:00+00:00" } }, + "thread_id": "thread::abc", + "agent_id": null, + "enabled": true, + "system": true, + "delete_after_run": true, + "next_run": "2026-05-29T07:30:00+00:00", + "last_status": "never_run", + "run_count": 0, + "created_at": "2026-05-29T07:25:00+00:00", + "last_run_at": null, + "recent_runs": [ + { + "run_id": "...", + "job_id": "followup_4d7c5b8f12ab9e3a", + "status": "failed", + "started_at": "2026-05-29T07:30:00+00:00", + "finished_at": "2026-05-29T07:30:00+00:00", + "duration_ms": 12, + "thread_id": "thread::abc", + "error": "thread not found" + } + ] + } + ], + "count": 1, + "thread_id": null, + "since": null, + "runs_limit": 20, + "service_available": true +} +``` + +When the cron service is not running, the endpoint returns `200` with +`{"jobs": [], "count": 0, "service_available": false}` (mirroring +`GET /api/cron/jobs`), so a probe never 500s just because cron is disabled. + +### Reading a "followup never fired" incident + +1. Filter to the thread: `GET /api/debug/system-cron-jobs?thread_id=thread::abc`. +2. If the job is **absent**, it already fired and self-deleted + (`delete_after_run: true`) — check `GET /api/cron/runs` or the gateway logs + for its terminal `RunRecord`. +3. If the job is **present** with `last_status: never_run` and a future + `next_run`, it is still pending — the delay simply has not elapsed. +4. If `recent_runs` shows a `failed` record, the `error` field explains why the + dispatch did not reach the thread (e.g. the thread was deleted or had no + provider attached). + +## `POST /api/debug/system-cron-jobs/{id}/run` + +Manually fires a system cron job immediately — a system-only wrapper around +`CronService::run_now`. The debug channel must never be a back door to trigger +user-visible automations, so: + +- A **missing** job → `404 not_found`. +- A job that exists but is **not** `system` → `404 not_found` (same shape as + missing; the debug channel does not enumerate or fire user automations). +- A system job that **cannot run right now** (disabled or already running) → + `409 not_runnable`. +- Otherwise → `200` with the resulting `RunRecord`: + +```json +{ "ran": true, "run": { "run_id": "...", "job_id": "...", "status": "success", "...": "..." } } +``` + +## Implementation notes + +- `GET` reuses `CronService::list_all` (the unfiltered list — `list()` hides + system jobs) and `CronService::list_runs_for_job`. It is strictly read-only + and does not repair or mutate any cron state. +- `POST .../run` reuses `CronService::get` (to enforce the system-only guard) + and `CronService::run_now`. +- Handlers: `garyx-gateway/src/api.rs` + (`debug_system_cron_jobs` / `debug_run_system_cron_job`); routes registered in + `garyx-gateway/src/route_graph.rs` under `operations_routes()`. +- The default `GET /api/cron/jobs` and `GET /api/cron/runs` behavior is + unchanged — these debug routes are additive. diff --git a/docs/schedule-followup.md b/docs/schedule-followup.md index da5aaf8f..a0545f67 100644 --- a/docs/schedule-followup.md +++ b/docs/schedule-followup.md @@ -126,6 +126,14 @@ followup-driven runs distinctly from organic user input: the `AppState` reference is held weakly, no circular `Arc` is formed between `AppState` and `CronService`. +## Observability + +`schedule_followup` jobs are `system: true`, so they do not show up in the +user-facing automation list. To inspect them during an incident — list the +pending followups, see each job's `RunRecord` history, or manually fire one — +use the debug endpoint documented in +[schedule-followup-observability.md](./schedule-followup-observability.md). + ## Backwards compatibility The `CronJobConfig.system` field and the `CronJobKind::InternalDispatch` diff --git a/garyx-gateway/src/api.rs b/garyx-gateway/src/api.rs index 47711947..c6c11dd9 100644 --- a/garyx-gateway/src/api.rs +++ b/garyx-gateway/src/api.rs @@ -2023,6 +2023,236 @@ pub async fn cron_runs( })) } +// --------------------------------------------------------------------------- +// GET /api/debug/system-cron-jobs +// --------------------------------------------------------------------------- +// +// Debug observability for system-managed cron jobs (AXON-692). The default +// user-facing `GET /api/cron/jobs` filters `system == true` jobs out, so +// `schedule_followup`-created followups are invisible there. When an incident +// like "agent promised a followup but it never fired" needs triage, SREs / +// developers reach for this endpoint to see the pending system jobs and each +// job's recent RunRecord history. +// +// Auth: registered under the protected router, so `enforce_gateway_auth` +// already gates it — loopback requests pass, everything else needs a valid +// gateway token. It reuses the existing gateway token rather than introducing +// a separate debug-token config surface. It is never exposed unauthenticated +// to non-loopback callers. + +/// Default number of recent RunRecords attached to each job. +fn default_debug_runs_limit() -> usize { + 20 +} + +#[derive(Deserialize)] +pub struct DebugSystemCronParams { + /// Optional thread filter. Matches `CronJob.thread_id` exactly. An empty + /// or whitespace-only value is ignored (returns all system jobs) rather + /// than matching jobs whose `thread_id` is unset. + #[serde(default)] + pub thread_id: Option, + /// Optional lower bound on job `created_at`. Accepts either a unix-second + /// timestamp (all digits) or an RFC3339 datetime. Jobs created strictly + /// before this instant are filtered out. A value that parses as neither + /// form yields `400`, never a silent full list. + #[serde(default)] + pub since: Option, + /// Max recent RunRecords attached per job (most-recent-first). + #[serde(default = "default_debug_runs_limit")] + pub runs_limit: usize, +} + +/// Parse a `since` query value as a unix-second timestamp or RFC3339 datetime. +fn parse_since(raw: &str) -> Option> { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + if let Ok(secs) = trimmed.parse::() { + return chrono::DateTime::from_timestamp(secs, 0); + } + chrono::DateTime::parse_from_rfc3339(trimmed) + .ok() + .map(|dt| dt.with_timezone(&Utc)) +} + +/// Render a single system cron job (plus its recent runs) into the debug shape. +fn debug_job_json(job: &crate::cron::CronJob, recent_runs: Vec) -> Value { + let kind = match &job.kind { + garyx_models::config::CronJobKind::AutomationPrompt => json!({ "type": "automation_prompt" }), + garyx_models::config::CronJobKind::InternalDispatch { payload } => json!({ + "type": "internal_dispatch", + "reason": payload.reason, + "originating_run_id": payload.originating_run_id, + "scheduled_at": payload.scheduled_at.to_rfc3339(), + "delay_seconds_requested": payload.delay_seconds_requested, + }), + }; + json!({ + "id": job.id, + "label": job.label, + "kind": kind, + "schedule": job.schedule, + "thread_id": job.thread_id, + "agent_id": job.agent_id, + "enabled": job.enabled, + "system": job.system, + "delete_after_run": job.delete_after_run, + "next_run": job.next_run.to_rfc3339(), + "last_status": job.last_status, + "run_count": job.run_count, + "created_at": job.created_at.to_rfc3339(), + "last_run_at": job.last_run_at.map(|t| t.to_rfc3339()), + "recent_runs": recent_runs, + }) +} + +/// Render a RunRecord into JSON (mirrors the `cron_runs` shape, adds thread_id). +fn debug_run_json(r: &crate::cron::RunRecord) -> Value { + json!({ + "run_id": r.run_id, + "job_id": r.job_id, + "status": r.status, + "started_at": r.started_at.to_rfc3339(), + "finished_at": r.finished_at.map(|t| t.to_rfc3339()), + "duration_ms": r.duration_ms, + "thread_id": r.thread_id, + "error": r.error, + }) +} + +/// GET /api/debug/system-cron-jobs - list system cron jobs + RunRecord history. +pub async fn debug_system_cron_jobs( + State(state): State>, + Query(params): Query, +) -> impl IntoResponse { + let cron = match &state.ops.cron_service { + Some(svc) => svc, + None => { + return Json(json!({ + "jobs": [], + "count": 0, + "service_available": false, + })) + .into_response(); + } + }; + + // Parse `since` up front so a bad value fails loudly instead of returning + // an unfiltered list that an SRE might misread as "no jobs since X". + let since = match params.since.as_deref().map(str::trim) { + Some(raw) if !raw.is_empty() => match parse_since(raw) { + Some(ts) => Some(ts), + None => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": "invalid_since", + "message": "since must be a unix-second timestamp or an RFC3339 datetime", + "got": raw, + })), + ) + .into_response(); + } + }, + _ => None, + }; + + let thread_filter = params + .thread_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); + + let mut jobs: Vec = Vec::new(); + for job in cron.list_all().await.into_iter().filter(|j| j.system) { + if let Some(tid) = thread_filter + && job.thread_id.as_deref() != Some(tid) + { + continue; + } + if let Some(since_ts) = since + && job.created_at < since_ts + { + continue; + } + let recent_runs: Vec = cron + .list_runs_for_job(&job.id, params.runs_limit, 0) + .await + .iter() + .map(debug_run_json) + .collect(); + jobs.push(debug_job_json(&job, recent_runs)); + } + + Json(json!({ + "jobs": jobs, + "count": jobs.len(), + "thread_id": thread_filter, + "since": since.map(|t| t.to_rfc3339()), + "runs_limit": params.runs_limit, + "service_available": true, + })) + .into_response() +} + +/// POST /api/debug/system-cron-jobs/{id}/run - manually fire a system cron job. +/// +/// System-only wrapper around `CronService::run_now` (AXON-692 goal #3): the +/// debug channel must never be a back door to trigger user-visible automations, +/// so a non-system job (or a missing one) returns `404`. A job that exists but +/// can't run right now (disabled / already running) returns `409`. +pub async fn debug_run_system_cron_job( + State(state): State>, + Path(id): Path, +) -> impl IntoResponse { + let cron = match &state.ops.cron_service { + Some(svc) => svc, + None => { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "error": "service_unavailable", + "message": "cron service is not running", + })), + ) + .into_response(); + } + }; + + match cron.get(&id).await { + // Hide non-system jobs behind the same 404 as a missing one — the debug + // channel only fires system jobs and must not enumerate user automations. + None => ( + StatusCode::NOT_FOUND, + Json(json!({ "error": "not_found", "message": "no such system cron job", "id": id })), + ) + .into_response(), + Some(job) if !job.system => ( + StatusCode::NOT_FOUND, + Json(json!({ "error": "not_found", "message": "no such system cron job", "id": id })), + ) + .into_response(), + Some(_) => match cron.run_now(&id).await { + Some(record) => Json(json!({ + "ran": true, + "run": debug_run_json(&record), + })) + .into_response(), + None => ( + StatusCode::CONFLICT, + Json(json!({ + "error": "not_runnable", + "message": "job is disabled or already running", + "id": id, + })), + ) + .into_response(), + }, + } +} + // --------------------------------------------------------------------------- // PUT /api/settings // --------------------------------------------------------------------------- diff --git a/garyx-gateway/src/api/tests.rs b/garyx-gateway/src/api/tests.rs index bedf085b..09934202 100644 --- a/garyx-gateway/src/api/tests.rs +++ b/garyx-gateway/src/api/tests.rs @@ -113,6 +113,14 @@ fn api_router(state: Arc) -> Router { ) .route("/api/cron/jobs", axum::routing::get(cron_jobs)) .route("/api/cron/runs", axum::routing::get(cron_runs)) + .route( + "/api/debug/system-cron-jobs", + axum::routing::get(debug_system_cron_jobs), + ) + .route( + "/api/debug/system-cron-jobs/{id}/run", + axum::routing::post(debug_run_system_cron_job), + ) .route("/api/settings", axum::routing::put(settings_update)) .route("/api/settings/reload", axum::routing::post(settings_reload)) .route("/api/restart", axum::routing::post(restart)) @@ -1605,6 +1613,237 @@ async fn test_cron_runs_no_service() { assert_eq!(json["count"], 0); } +// --------------------------------------------------------------------------- +// GET/POST /api/debug/system-cron-jobs (AXON-692) +// --------------------------------------------------------------------------- + +/// Build a CronService seeded with a mix of system + non-system jobs so the +/// debug-endpoint tests can assert filtering behavior. Returns the service +/// (caller wraps it into AppState.ops.cron_service). +async fn seed_cron_service_for_debug() -> crate::cron::CronService { + use garyx_models::config::{ + CronAction, CronJobConfig, CronJobKind, CronSchedule, InternalDispatchJobPayload, + }; + let tmp = tempfile::TempDir::new().unwrap(); + let svc = crate::cron::CronService::new(tmp.path().to_path_buf()); + let _ = tokio::fs::create_dir_all(tmp.path().join("cron").join("jobs")).await; + + let far_future = (chrono::Utc::now() + chrono::Duration::hours(1)).to_rfc3339(); + + // Two system followup jobs on different threads. + for (id, thread) in [("followup_aaa", "thread::alpha"), ("followup_bbb", "thread::beta")] { + svc.add(CronJobConfig { + id: id.to_owned(), + kind: CronJobKind::InternalDispatch { + payload: InternalDispatchJobPayload { + prompt: "resume the build poll".to_owned(), + reason: Some("background build finished".to_owned()), + originating_run_id: Some("run-test-1".to_owned()), + scheduled_at: chrono::Utc::now(), + delay_seconds_requested: 300, + }, + }, + label: Some(format!("schedule_followup({thread})")), + schedule: CronSchedule::Once { + at: far_future.clone(), + }, + ui_schedule: None, + action: CronAction::Log, + target: None, + message: None, + workspace_dir: None, + agent_id: None, + thread_id: Some(thread.to_owned()), + delete_after_run: true, + enabled: true, + system: true, + }) + .await + .unwrap(); + } + + // One ordinary (non-system) automation that must NOT appear in the debug list. + svc.add(CronJobConfig { + id: "user-automation".to_owned(), + kind: Default::default(), + label: Some("daily standup".to_owned()), + schedule: CronSchedule::Interval { interval_secs: 60 }, + ui_schedule: None, + action: CronAction::Log, + target: None, + message: None, + workspace_dir: None, + agent_id: None, + thread_id: Some("thread::alpha".to_owned()), + delete_after_run: false, + enabled: true, + system: false, + }) + .await + .unwrap(); + + svc +} + +#[tokio::test] +async fn test_debug_system_cron_jobs_no_service() { + let state = test_state(); + let router = api_router(state); + + let req = Request::builder() + .uri("/api/debug/system-cron-jobs") + .body(Body::empty()) + .unwrap(); + + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let body = axum::body::to_bytes(resp.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["service_available"], false); + assert_eq!(json["count"], 0); +} + +#[tokio::test] +async fn test_debug_system_cron_jobs_lists_only_system() { + let state = test_state(); + let svc = seed_cron_service_for_debug().await; + let mut state_with_cron = (*state).clone_for_test(); + state_with_cron.ops.cron_service = Some(Arc::new(svc)); + let router = api_router(Arc::new(state_with_cron)); + + let req = Request::builder() + .uri("/api/debug/system-cron-jobs") + .body(Body::empty()) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let body = axum::body::to_bytes(resp.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["service_available"], true); + // Two system jobs; the user-automation is filtered out. + assert_eq!(json["count"], 2); + let ids: Vec<&str> = json["jobs"] + .as_array() + .unwrap() + .iter() + .map(|j| j["id"].as_str().unwrap()) + .collect(); + assert!(ids.contains(&"followup_aaa")); + assert!(ids.contains(&"followup_bbb")); + assert!(!ids.contains(&"user-automation")); + // Each job carries its internal_dispatch kind + a recent_runs array. + let job = &json["jobs"][0]; + assert_eq!(job["kind"]["type"], "internal_dispatch"); + assert!(job["recent_runs"].is_array()); +} + +#[tokio::test] +async fn test_debug_system_cron_jobs_thread_filter() { + let state = test_state(); + let svc = seed_cron_service_for_debug().await; + let mut state_with_cron = (*state).clone_for_test(); + state_with_cron.ops.cron_service = Some(Arc::new(svc)); + let router = api_router(Arc::new(state_with_cron)); + + let req = Request::builder() + .uri("/api/debug/system-cron-jobs?thread_id=thread::beta") + .body(Body::empty()) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let body = axum::body::to_bytes(resp.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["count"], 1); + assert_eq!(json["jobs"][0]["id"], "followup_bbb"); + assert_eq!(json["thread_id"], "thread::beta"); +} + +#[tokio::test] +async fn test_debug_system_cron_jobs_invalid_since_is_400() { + let state = test_state(); + let svc = seed_cron_service_for_debug().await; + let mut state_with_cron = (*state).clone_for_test(); + state_with_cron.ops.cron_service = Some(Arc::new(svc)); + let router = api_router(Arc::new(state_with_cron)); + + let req = Request::builder() + .uri("/api/debug/system-cron-jobs?since=not-a-date") + .body(Body::empty()) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 400); + let body = axum::body::to_bytes(resp.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["error"], "invalid_since"); +} + +#[tokio::test] +async fn test_debug_system_cron_jobs_since_unix_filters() { + let state = test_state(); + let svc = seed_cron_service_for_debug().await; + let mut state_with_cron = (*state).clone_for_test(); + state_with_cron.ops.cron_service = Some(Arc::new(svc)); + let router = api_router(Arc::new(state_with_cron)); + + // A `since` far in the future filters every job out (all created just now). + let future_ts = (chrono::Utc::now() + chrono::Duration::hours(24)).timestamp(); + let req = Request::builder() + .uri(format!("/api/debug/system-cron-jobs?since={future_ts}")) + .body(Body::empty()) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 200); + let body = axum::body::to_bytes(resp.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["count"], 0); +} + +#[tokio::test] +async fn test_debug_run_system_cron_job_missing_is_404() { + let state = test_state(); + let svc = seed_cron_service_for_debug().await; + let mut state_with_cron = (*state).clone_for_test(); + state_with_cron.ops.cron_service = Some(Arc::new(svc)); + let router = api_router(Arc::new(state_with_cron)); + + let req = Request::builder() + .method("POST") + .uri("/api/debug/system-cron-jobs/does-not-exist/run") + .body(Body::empty()) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 404); +} + +#[tokio::test] +async fn test_debug_run_system_cron_job_rejects_non_system() { + let state = test_state(); + let svc = seed_cron_service_for_debug().await; + let mut state_with_cron = (*state).clone_for_test(); + state_with_cron.ops.cron_service = Some(Arc::new(svc)); + let router = api_router(Arc::new(state_with_cron)); + + // user-automation exists but is non-system → debug channel must 404 it, + // never fire a user-visible automation. + let req = Request::builder() + .method("POST") + .uri("/api/debug/system-cron-jobs/user-automation/run") + .body(Body::empty()) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), 404); +} + #[tokio::test] async fn test_settings_update_valid() { let state = test_state(); diff --git a/garyx-gateway/src/route_graph.rs b/garyx-gateway/src/route_graph.rs index f0139d0e..38656a64 100644 --- a/garyx-gateway/src/route_graph.rs +++ b/garyx-gateway/src/route_graph.rs @@ -390,6 +390,17 @@ fn operations_routes() -> Router> { Router::new() .route("/api/cron/jobs", axum::routing::get(api::cron_jobs)) .route("/api/cron/runs", axum::routing::get(api::cron_runs)) + // Debug observability for system-managed cron jobs (AXON-692). Lives in + // the protected router so `enforce_gateway_auth` gates it: loopback + // passes, everything else needs a valid gateway token. + .route( + "/api/debug/system-cron-jobs", + axum::routing::get(api::debug_system_cron_jobs), + ) + .route( + "/api/debug/system-cron-jobs/{id}/run", + axum::routing::post(api::debug_run_system_cron_job), + ) .route( "/api/channels/plugins", axum::routing::get(api::list_channel_plugins),