Skip to content

Grafana dashboard for prerender_html jobs, cross-linked with Indexing #5438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
habdelra merged 4 commits into from
Jul 9, 2026
Merged

Grafana dashboard for prerender_html jobs, cross-linked with Indexing #5438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bd88194
Grafana prerender_html dashboard + index/prerender cross-links (CS-11…
habdelra Jul 8, 2026
8fd02d0
Add PR preview images
habdelra Jul 8, 2026
4b18ed7
Strip PR preview images
habdelra Jul 8, 2026
2b3224c
Escape job-filter vars; align Queued tables with the Pending stat
habdelra Jul 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .prettierignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,9 @@ bench-fixtures/
# pnpm owns this file's formatting; prettier would fight pnpm's serializer
# and rewrite the whole file on every commit that stages it.
pnpm-lock.yaml

# Grafana dashboards/folders are regenerated by `grafanactl resources pull`
# (packages/observability/scripts/pull.sh); prettier reformatting them would
# make every subsequent pull a formatting fight. The package's own lint
# excludes them from prettier for the same reason.
packages/observability/grafanactl/resources/
74 changes: 68 additions & 6 deletions packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,17 @@
"title": "Realms",
"type": "dashboards"
},
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": true,
"tags": [
"workflow:prerender-html"
],
"title": "Prerender HTML",
"type": "dashboards"
},
{
"asDropdown": false,
"icon": "external link",
Expand Down Expand Up @@ -593,7 +604,7 @@
"type": "grafana-postgresql-datasource",
"uid": "cef5v5sl9k7i8f"
},
"description": "One row per indexing job currently held by a worker (across all realm-server / worker tasks). `progress` and `current files` come from `job_progress` (CS-10930), populated by each realm-server's IndexingEventSink write-through. Click the realm cell to drill into the live activity feed.",
"description": "One row per indexing job currently held by a worker (across all realm-server / worker tasks). `progress` and `current files` come from `job_progress` (CS-10930), populated by each realm-server's IndexingEventSink write-through. Click the realm cell to drill into the live activity feed; the job_id cell pivots to the Prerender HTML dashboard filtered to the prerender_html job this pass spawned.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -790,6 +801,24 @@
}
]
},
{
"matcher": {
"id": "byName",
"options": "job_id"
},
"properties": [
{
"id": "links",
"value": [
{
"targetBlank": true,
"title": "View spawned prerender job",
"url": "/d/b5eou6wleg7i6o?${__url_time_range}&var-spawning_job_id=${__data.fields.job_id}&orgId=1"
}
]
}
]
},
{
"matcher": {
"id": "byName",
Expand Down Expand Up @@ -856,7 +885,7 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT\n job_id,\n realm,\n realm_url,\n job_type,\n files_completed,\n total_files,\n percent,\n percent AS pct,\n elapsed_seconds,\n started_at,\n worker_id,\n reservation_id\nFROM (\n SELECT\n j.id AS job_id,\n COALESCE(\n NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n ) AS realm,\n COALESCE(j.args->>'realmURL','') AS realm_url,\n j.job_type,\n COALESCE(jp.files_completed, 0) AS files_completed,\n COALESCE(jp.total_files, 0) AS total_files,\n CASE WHEN COALESCE(jp.total_files, 0) > 0\n THEN (jp.files_completed::float / jp.total_files) * 100\n ELSE 0\n END AS percent,\n EXTRACT(EPOCH FROM (NOW() - jr.created_at)) AS elapsed_seconds,\n jr.created_at AS started_at,\n jr.worker_id,\n jr.id AS reservation_id\n FROM jobs j\n JOIN job_reservations jr ON jr.job_id = j.id\n AND jr.completed_at IS NULL AND jr.locked_until > NOW()\n LEFT JOIN job_progress jp ON jp.job_id = j.id\n WHERE j.job_type IN ('from-scratch-index','incremental-index')\n AND j.finished_at IS NULL\n) active\nORDER BY started_at DESC;",
"rawSql": "SELECT\n job_id,\n realm,\n realm_url,\n job_type,\n files_completed,\n total_files,\n percent,\n percent AS pct,\n elapsed_seconds,\n started_at,\n worker_id,\n reservation_id\nFROM (\n SELECT\n j.id AS job_id,\n COALESCE(\n NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n ) AS realm,\n COALESCE(j.args->>'realmURL','') AS realm_url,\n j.job_type,\n COALESCE(jp.files_completed, 0) AS files_completed,\n COALESCE(jp.total_files, 0) AS total_files,\n CASE WHEN COALESCE(jp.total_files, 0) > 0\n THEN (jp.files_completed::float / jp.total_files) * 100\n ELSE 0\n END AS percent,\n EXTRACT(EPOCH FROM (NOW() - jr.created_at)) AS elapsed_seconds,\n jr.created_at AS started_at,\n jr.worker_id,\n jr.id AS reservation_id\n FROM jobs j\n JOIN job_reservations jr ON jr.job_id = j.id\n AND jr.completed_at IS NULL AND jr.locked_until > NOW()\n LEFT JOIN job_progress jp ON jp.job_id = j.id\n WHERE j.job_type IN ('from-scratch-index','incremental-index')\n AND j.finished_at IS NULL\n AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\n) active\nORDER BY started_at DESC;",
"refId": "A"
}
],
Expand Down Expand Up @@ -1123,7 +1152,7 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n j.id, \n j.priority, \n j.job_type, \n CASE\n WHEN j.job_type = 'full-reindex' THEN '(all realms)'\n ELSE COALESCE(\n NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n )\n END AS realm,\n CASE \n WHEN j.concurrency_group ~ '^indexing:https://[^/]+/.+' THEN REGEXP_REPLACE(j.concurrency_group, '^indexing:https://[^/]+/', '') \n WHEN j.concurrency_group ~ '^indexing:https://[^/]+/?$' THEN REGEXP_REPLACE(j.concurrency_group, '^indexing:https://', '') \n ELSE j.concurrency_group \n END AS concurrency_group, \n j.status AS status, \n j.created_at AS created_at, \n\n\n -- Wait time in seconds\n CASE \n WHEN jr.created_at IS NOT NULL \n THEN EXTRACT(EPOCH FROM (jr.created_at - j.created_at))\n ELSE \n EXTRACT(EPOCH FROM (NOW() - j.created_at))\n END\n AS wait_seconds,\n j.id as job_id\n\nFROM \n jobs j\n \nLEFT JOIN \n job_reservations jr ON j.id = jr.job_id\n\nWHERE\njr.job_id IS NULL AND j.status = 'unfulfilled' AND j.job_type IN ('from-scratch-index','incremental-index','full-reindex') \n \nORDER BY \n j.created_at ASC\nLIMIT 500;",
"rawSql": "SELECT \n j.id, \n j.priority, \n j.job_type, \n CASE\n WHEN j.job_type = 'full-reindex' THEN '(all realms)'\n ELSE COALESCE(\n NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n )\n END AS realm,\n CASE \n WHEN j.concurrency_group ~ '^indexing:https://[^/]+/.+' THEN REGEXP_REPLACE(j.concurrency_group, '^indexing:https://[^/]+/', '') \n WHEN j.concurrency_group ~ '^indexing:https://[^/]+/?$' THEN REGEXP_REPLACE(j.concurrency_group, '^indexing:https://', '') \n ELSE j.concurrency_group \n END AS concurrency_group, \n j.status AS status, \n j.created_at AS created_at, \n\n\n -- Wait time in seconds\n CASE \n WHEN jr.created_at IS NOT NULL \n THEN EXTRACT(EPOCH FROM (jr.created_at - j.created_at))\n ELSE \n EXTRACT(EPOCH FROM (NOW() - j.created_at))\n END\n AS wait_seconds,\n j.id as job_id\n\nFROM \n jobs j\n \nLEFT JOIN \n job_reservations jr ON j.id = jr.job_id\n AND jr.completed_at IS NULL AND jr.locked_until > NOW()\n\nWHERE\njr.id IS NULL AND j.status = 'unfulfilled' AND j.job_type IN ('from-scratch-index','incremental-index','full-reindex') AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\n \nORDER BY\n j.created_at ASC\nLIMIT 500;",
"refId": "A",
"sql": {
"columns": [
Expand All @@ -1146,14 +1175,14 @@
],
"title": "Queued Indexing Jobs",
"type": "table",
"description": "Indexing jobs (from-scratch-index, incremental-index) waiting on a worker. Same query as Job Queue → Waiting Jobs, narrowed to indexing job types."
"description": "Indexing jobs (from-scratch-index, incremental-index, full-reindex) waiting on a worker — no live reservation holds them, so jobs whose lease expired and are claimable again are included."
},
{
"datasource": {
"type": "grafana-postgresql-datasource",
"uid": "cef5v5sl9k7i8f"
},
"description": "Indexing jobs (from-scratch-index, incremental-index, full-reindex) that finished in the past 24 hours, newest first. `wait_seconds` is queue time (created_at → first reservation). `run_seconds` is the duration of the attempt that completed it (latest reservation → finished_at). Force-cancelled jobs that never got a reservation appear with NULL started_at / wait_seconds / run_seconds / worker_id.",
"description": "Indexing jobs (from-scratch-index, incremental-index, full-reindex) that finished in the past 24 hours, newest first. `wait_seconds` is queue time (created_at → first reservation). `run_seconds` is the duration of the attempt that completed it (latest reservation → finished_at). `generation` is the realm index generation the pass committed (NULL for full-reindex orchestration and results from builds that predate generation stamping). The id cell pivots to the Prerender HTML dashboard filtered to the prerender_html job this pass spawned. Force-cancelled jobs that never got a reservation appear with NULL started_at / wait_seconds / run_seconds / worker_id.",
"fieldConfig": {
"defaults": {
"color": {
Expand Down Expand Up @@ -1183,6 +1212,24 @@
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "links",
"value": [
{
"targetBlank": true,
"title": "View spawned prerender job",
"url": "/d/b5eou6wleg7i6o?${__url_time_range}&var-spawning_job_id=${__data.fields.id}&orgId=1"
}
]
}
]
},
{
"matcher": {
"id": "byName",
Expand Down Expand Up @@ -1290,7 +1337,7 @@
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT\n j.id,\n lr.reservation_id,\n j.job_type,\n CASE\n WHEN j.job_type = 'full-reindex' THEN '(all realms)'\n ELSE COALESCE(\n NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n )\n END AS realm,\n j.status,\n j.created_at,\n lr.started_at,\n j.finished_at,\n EXTRACT(EPOCH FROM (fr.first_started_at - j.created_at)) AS wait_seconds,\n EXTRACT(EPOCH FROM (j.finished_at - lr.started_at)) AS run_seconds,\n lr.worker_id\nFROM jobs j\nLEFT JOIN LATERAL (\n SELECT jr.id AS reservation_id, jr.created_at AS started_at, jr.worker_id\n FROM job_reservations jr\n WHERE jr.job_id = j.id\n ORDER BY jr.created_at DESC\n LIMIT 1\n) lr ON TRUE\nLEFT JOIN LATERAL (\n SELECT MIN(jr.created_at) AS first_started_at\n FROM job_reservations jr\n WHERE jr.job_id = j.id\n) fr ON TRUE\nWHERE j.job_type IN ('from-scratch-index','incremental-index','full-reindex')\n AND j.finished_at IS NOT NULL\n AND j.finished_at > NOW() - INTERVAL '24 hours'\nORDER BY j.finished_at DESC\nLIMIT 500;",
"rawSql": "SELECT\n j.id,\n lr.reservation_id,\n j.job_type,\n CASE\n WHEN j.job_type = 'full-reindex' THEN '(all realms)'\n ELSE COALESCE(\n NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n )\n END AS realm,\n j.status,\n j.created_at,\n lr.started_at,\n j.finished_at,\n EXTRACT(EPOCH FROM (fr.first_started_at - j.created_at)) AS wait_seconds,\n EXTRACT(EPOCH FROM (j.finished_at - lr.started_at)) AS run_seconds,\n (j.result->>'generation')::bigint AS generation,\n lr.worker_id\nFROM jobs j\nLEFT JOIN LATERAL (\n SELECT jr.id AS reservation_id, jr.created_at AS started_at, jr.worker_id\n FROM job_reservations jr\n WHERE jr.job_id = j.id\n ORDER BY jr.created_at DESC\n LIMIT 1\n) lr ON TRUE\nLEFT JOIN LATERAL (\n SELECT MIN(jr.created_at) AS first_started_at\n FROM job_reservations jr\n WHERE jr.job_id = j.id\n) fr ON TRUE\nWHERE j.job_type IN ('from-scratch-index','incremental-index','full-reindex')\n AND j.finished_at IS NOT NULL\n AND j.finished_at > NOW() - INTERVAL '24 hours'\n AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\nORDER BY j.finished_at DESC\nLIMIT 500;",
"refId": "A"
}
],
Expand Down Expand Up @@ -1611,6 +1658,21 @@
"query": "REPLACE_AT_APPLY_TIME",
"skipUrlSync": true,
"type": "constant"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"description": "Filter the job tables to a single indexing job id. Prefilled by the 'View originating index job' drill-through on the Prerender HTML dashboard. Empty = no filter.",
"hide": 0,
"label": "Job id filter",
"name": "job_id",
"options": [],
"query": "",
"skipUrlSync": false,
"type": "textbox"
}
]
},
Expand Down
Loading
Loading