Indexing dashboard: generation column and spawn-aware prerender job links

packages/observability/grafanactl/resources/dashboards/boxel-status/indexing.json

@@ -604,7 +604,7 @@
           "type": "grafana-postgresql-datasource",
           "uid": "cef5v5sl9k7i8f"
         },
-        "description": "One row per indexing job currently held by a worker (across all realm-server / worker tasks). `progress` and `current files` come from `job_progress` (CS-10930), populated by each realm-server's IndexingEventSink write-through. Click the realm cell to drill into the live activity feed; the job_id cell pivots to the Prerender HTML dashboard filtered to the prerender_html job this pass spawned.",
+        "description": "One row per indexing job currently held by a worker (across all realm-server / worker tasks). `progress` and `current files` come from `job_progress`, populated by each realm-server's IndexingEventSink write-through. `generation` is the realm generation this pass is producing (`realm_generations.current_generation + 1` — per-realm indexing serializes, so the in-flight pass owns the next generation until it commits it). `prerender_job_id` fills in once this pass's `prerender_html` job exists — typically at commit, when the enqueue lands atomically with the generation bump, or earlier when coalescing attached the work to an existing job — and pivots to the Prerender HTML dashboard filtered to this pass's spawned jobs; an empty cell means no prerender job has been spawned yet. Click the realm cell to drill into the live activity feed.",
         "fieldConfig": {
           "defaults": {
             "color": {
@@ -804,15 +804,15 @@
             {
               "matcher": {
                 "id": "byName",
-                "options": "job_id"
+                "options": "prerender_job_id"
               },
               "properties": [
                 {
                   "id": "links",
                   "value": [
                     {
                       "targetBlank": true,
-                      "title": "View spawned prerender job",
+                      "title": "View spawned prerender jobs",
                       "url": "/d/b5eou6wleg7i6o?${__url_time_range}&var-spawning_job_id=${__data.fields.job_id}&orgId=1"
                     }
                   ]
@@ -885,7 +885,7 @@
             "editorMode": "code",
             "format": "table",
             "rawQuery": true,
-            "rawSql": "SELECT\n  job_id,\n  realm,\n  realm_url,\n  job_type,\n  files_completed,\n  total_files,\n  percent,\n  percent AS pct,\n  elapsed_seconds,\n  started_at,\n  worker_id,\n  reservation_id\nFROM (\n  SELECT\n    j.id AS job_id,\n    COALESCE(\n      NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n      REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n    ) AS realm,\n    COALESCE(j.args->>'realmURL','') AS realm_url,\n    j.job_type,\n    COALESCE(jp.files_completed, 0) AS files_completed,\n    COALESCE(jp.total_files, 0) AS total_files,\n    CASE WHEN COALESCE(jp.total_files, 0) > 0\n      THEN (jp.files_completed::float / jp.total_files) * 100\n      ELSE 0\n    END AS percent,\n    EXTRACT(EPOCH FROM (NOW() - jr.created_at)) AS elapsed_seconds,\n    jr.created_at AS started_at,\n    jr.worker_id,\n    jr.id AS reservation_id\n  FROM jobs j\n  JOIN job_reservations jr ON jr.job_id = j.id\n    AND jr.completed_at IS NULL AND jr.locked_until > NOW()\n  LEFT JOIN job_progress jp ON jp.job_id = j.id\n  WHERE j.job_type IN ('from-scratch-index','incremental-index')\n    AND j.finished_at IS NULL\n    AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\n) active\nORDER BY started_at DESC;",
+            "rawSql": "SELECT\n  job_id,\n  realm,\n  realm_url,\n  job_type,\n  generation,\n  prerender_job_id,\n  files_completed,\n  total_files,\n  percent,\n  percent AS pct,\n  elapsed_seconds,\n  started_at,\n  worker_id,\n  reservation_id\nFROM (\n  SELECT\n    j.id AS job_id,\n    COALESCE(\n      NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n      REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n    ) AS realm,\n    COALESCE(j.args->>'realmURL','') AS realm_url,\n    j.job_type,\n    COALESCE(rg.current_generation, 0) + 1 AS generation,\n    sp.prerender_job_id,\n    COALESCE(jp.files_completed, 0) AS files_completed,\n    COALESCE(jp.total_files, 0) AS total_files,\n    CASE WHEN COALESCE(jp.total_files, 0) > 0\n      THEN (jp.files_completed::float / jp.total_files) * 100\n      ELSE 0\n    END AS percent,\n    EXTRACT(EPOCH FROM (NOW() - jr.created_at)) AS elapsed_seconds,\n    jr.created_at AS started_at,\n    jr.worker_id,\n    jr.id AS reservation_id\n  FROM jobs j\n  JOIN job_reservations jr ON jr.job_id = j.id\n    AND jr.completed_at IS NULL AND jr.locked_until > NOW()\n  LEFT JOIN job_progress jp ON jp.job_id = j.id\n  LEFT JOIN realm_generations rg ON rg.realm_url = j.args->>'realmURL'\n  LEFT JOIN LATERAL (\n    SELECT MAX(pj.id) AS prerender_job_id\n      FROM jobs pj\n     WHERE pj.job_type = 'prerender_html'\n       AND pj.args->>'spawningJobId' IS NOT NULL\n       AND (pj.args->>'spawningJobId')::bigint = j.id\n  ) sp ON TRUE\n  WHERE j.job_type IN ('from-scratch-index','incremental-index')\n    AND j.finished_at IS NULL\n    AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\n) active\nORDER BY started_at DESC;",
             "refId": "A"
           }
         ],
@@ -1182,7 +1182,7 @@
           "type": "grafana-postgresql-datasource",
           "uid": "cef5v5sl9k7i8f"
         },
-        "description": "Indexing jobs (from-scratch-index, incremental-index, full-reindex) that finished in the past 24 hours, newest first. `wait_seconds` is queue time (created_at → first reservation). `run_seconds` is the duration of the attempt that completed it (latest reservation → finished_at). `generation` is the realm index generation the pass committed (NULL for full-reindex orchestration and results from builds that predate generation stamping). The id cell pivots to the Prerender HTML dashboard filtered to the prerender_html job this pass spawned. Force-cancelled jobs that never got a reservation appear with NULL started_at / wait_seconds / run_seconds / worker_id.",
+        "description": "Indexing jobs (from-scratch-index, incremental-index, full-reindex) that finished in the past 24 hours, newest first. `wait_seconds` is queue time (created_at → first reservation). `run_seconds` is the duration of the attempt that completed it (latest reservation → finished_at). `generation` is the realm index generation the pass committed (NULL for full-reindex orchestration and results from builds that predate generation stamping). `prerender_job_id` is the `prerender_html` job this pass spawned (the newest when retries spawned more than one); the cell pivots to the Prerender HTML dashboard filtered to this pass's spawned jobs, and an empty cell means none was spawned (e.g. rows that predate the prerender split). The lookup probes the partial expression index on the spawningJobId the prerender job's args carry. Force-cancelled jobs that never got a reservation appear with NULL started_at / wait_seconds / run_seconds / worker_id.",
         "fieldConfig": {
           "defaults": {
             "color": {
@@ -1215,15 +1215,15 @@
             {
               "matcher": {
                 "id": "byName",
-                "options": "id"
+                "options": "prerender_job_id"
               },
               "properties": [
                 {
                   "id": "links",
                   "value": [
                     {
                       "targetBlank": true,
-                      "title": "View spawned prerender job",
+                      "title": "View spawned prerender jobs",
                       "url": "/d/b5eou6wleg7i6o?${__url_time_range}&var-spawning_job_id=${__data.fields.id}&orgId=1"
                     }
                   ]
@@ -1337,7 +1337,7 @@
             "editorMode": "code",
             "format": "table",
             "rawQuery": true,
-            "rawSql": "SELECT\n  j.id,\n  lr.reservation_id,\n  j.job_type,\n  CASE\n    WHEN j.job_type = 'full-reindex' THEN '(all realms)'\n    ELSE COALESCE(\n      NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n      REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n    )\n  END AS realm,\n  j.status,\n  j.created_at,\n  lr.started_at,\n  j.finished_at,\n  EXTRACT(EPOCH FROM (fr.first_started_at - j.created_at)) AS wait_seconds,\n  EXTRACT(EPOCH FROM (j.finished_at - lr.started_at)) AS run_seconds,\n  (j.result->>'generation')::bigint AS generation,\n  lr.worker_id\nFROM jobs j\nLEFT JOIN LATERAL (\n  SELECT jr.id AS reservation_id, jr.created_at AS started_at, jr.worker_id\n  FROM job_reservations jr\n  WHERE jr.job_id = j.id\n  ORDER BY jr.created_at DESC\n  LIMIT 1\n) lr ON TRUE\nLEFT JOIN LATERAL (\n  SELECT MIN(jr.created_at) AS first_started_at\n  FROM job_reservations jr\n  WHERE jr.job_id = j.id\n) fr ON TRUE\nWHERE j.job_type IN ('from-scratch-index','incremental-index','full-reindex')\n  AND j.finished_at IS NOT NULL\n  AND j.finished_at > NOW() - INTERVAL '24 hours'\n  AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\nORDER BY j.finished_at DESC\nLIMIT 500;",
+            "rawSql": "SELECT\n  j.id,\n  lr.reservation_id,\n  j.job_type,\n  CASE\n    WHEN j.job_type = 'full-reindex' THEN '(all realms)'\n    ELSE COALESCE(\n      NULLIF(RTRIM(REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://[^/]+/', ''), '/'), ''),\n      REGEXP_REPLACE(COALESCE(j.args->>'realmURL',''), '^https?://([^./:]+).*$', '\\1')\n    )\n  END AS realm,\n  j.status,\n  j.created_at,\n  lr.started_at,\n  j.finished_at,\n  EXTRACT(EPOCH FROM (fr.first_started_at - j.created_at)) AS wait_seconds,\n  EXTRACT(EPOCH FROM (j.finished_at - lr.started_at)) AS run_seconds,\n  (j.result->>'generation')::bigint AS generation,\n  sp.prerender_job_id,\n  lr.worker_id\nFROM jobs j\nLEFT JOIN LATERAL (\n  SELECT jr.id AS reservation_id, jr.created_at AS started_at, jr.worker_id\n  FROM job_reservations jr\n  WHERE jr.job_id = j.id\n  ORDER BY jr.created_at DESC\n  LIMIT 1\n) lr ON TRUE\nLEFT JOIN LATERAL (\n  SELECT MIN(jr.created_at) AS first_started_at\n  FROM job_reservations jr\n  WHERE jr.job_id = j.id\n) fr ON TRUE\nLEFT JOIN LATERAL (\n  SELECT MAX(pj.id) AS prerender_job_id\n    FROM jobs pj\n   WHERE pj.job_type = 'prerender_html'\n     AND pj.args->>'spawningJobId' IS NOT NULL\n     AND (pj.args->>'spawningJobId')::bigint = j.id\n) sp ON TRUE\nWHERE j.job_type IN ('from-scratch-index','incremental-index','full-reindex')\n  AND j.finished_at IS NOT NULL\n  AND j.finished_at > NOW() - INTERVAL '24 hours'\n  AND (${job_id:sqlstring} = '' OR j.id::text = ${job_id:sqlstring})\nORDER BY j.finished_at DESC\nLIMIT 500;",
             "refId": "A"
           }
         ],

packages/postgres/migrations/1783724082538_add-jobs-prerender-spawning-job-id-index.js

@@ -0,0 +1,32 @@
+// Partial expression index correlating a prerender_html job back to the
+// indexing pass that spawned it (the spawningJobId its args carry). The
+// Grafana indexing dashboard resolves a prerender_job_id per displayed
+// indexing job through this expression, so each lookup is an index probe
+// instead of a scan over all accumulated prerender_html jobs. The second
+// key column lets a MAX(id)/ORDER BY id probe resolve newest-wins without
+// touching the heap.
+//
+// The index expression must match the dashboard rawSql exactly —
+// ((args->>'spawningJobId')::bigint) — or the planner will not use it.
+//
+// CONCURRENTLY avoids locking queue writes during the build in
+// production; it cannot run inside a transaction, hence noTransaction().
+
+exports.shorthands = undefined;
+
+exports.up = (pgm) => {
+  pgm.noTransaction();
+  pgm.sql(`
+    CREATE INDEX CONCURRENTLY IF NOT EXISTS jobs_prerender_html_spawning_job_id_idx
+      ON jobs (((args->>'spawningJobId')::bigint), id)
+      WHERE job_type = 'prerender_html'
+        AND args->>'spawningJobId' IS NOT NULL;
+  `);
+};
+
+exports.down = (pgm) => {
+  pgm.noTransaction();
+  pgm.sql(`
+    DROP INDEX CONCURRENTLY IF EXISTS jobs_prerender_html_spawning_job_id_idx;
+  `);
+};