Give user index jobs a worker lane render sweeps cannot occupy

packages/matrix/support/isolated-realm-server.ts

@@ -463,16 +463,19 @@ export async function startServer({
     // Worker tiers for this shared, contended stack. Both Playwright
     // workers (fullyParallel) funnel their realm provisioning into one
     // worker manager, and each new workspace enqueues a from-scratch index
-    // plus a ~110-file, ~10s prerender-html sweep. User indexing and
-    // prerender-html are co-equal (both priority 10 — see
-    // runtime-common/queue.ts: a published realm's rendered HTML is as
-    // first-class as its search index), so these three high-tier workers all
-    // floor at 10 and serve both kinds of user work. What keeps indexing
-    // (which gates createRealm / new-user provisioning) and rendering (which
-    // gates published-realm serving) both moving is capacity, not a dedicated
-    // lane: three user-work workers absorb the two-Playwright-worker producer
-    // rate. (The split across the two count knobs is immaterial now that both
-    // floor at 10; kept separate only for symmetry with production.)
+    // plus a prerender-html job whose realm-wide module pre-warm sweep runs
+    // for tens of seconds. User indexing and prerender-html are co-equal
+    // (both priority 10 — see runtime-common/queue.ts: a published realm's
+    // rendered HTML is as first-class as its search index), so priority
+    // alone cannot keep sweeps from occupying every high-tier worker while
+    // a user write's incremental index job sits queued. That queue wait is
+    // what test assertions feel: createRealm blocks on the from-scratch
+    // index, and read-your-writes endpoints (card GET, _publishability)
+    // drain in-flight incremental indexing before responding. The
+    // user-index worker claims only indexing job types, so an index job
+    // always has a lane no render sweep can hold; the two high-priority
+    // workers carry the prerender-html sweeps alongside any other
+    // user-initiated work.
     `--userIndexCount=1`,
     `--highPriorityCount=2`,
 

packages/realm-server/tests/index.ts

@@ -377,6 +377,7 @@ const ALL_TEST_FILES: string[] = [
   './worker-request-signature-test',
   './worker-request-forwarder-test',
   './worker-reader-test',
+  './worker-job-registration-test',
   './realm-index-updater-test',
 ];
 

packages/realm-server/tests/worker-job-registration-test.ts

@@ -0,0 +1,90 @@
+import QUnit from 'qunit';
+const { module, test } = QUnit;
+import { basename } from 'path';
+import {
+  Worker,
+  INDEX_JOB_TYPES,
+  type QueueRunner,
+} from '@cardstack/runtime-common';
+
+// The queue's claim query only dequeues job types a worker has registered
+// handlers for, so a worker's registration set IS its claim policy. These
+// tests pin the registration sets for the two worker flavors:
+//
+//   1. A default worker registers every job type.
+//   2. An `indexJobsOnly` worker registers exactly INDEX_JOB_TYPES — that
+//      restriction is what makes the worker-manager's user-index pool a
+//      dedicated indexing lane that a prerender_html sweep (or any other
+//      job type) can never occupy, independent of priority tiers. Indexing
+//      gates realm provisioning (createRealm blocks on the from-scratch
+//      index) and read-your-writes endpoints (card GET / _publishability
+//      drain in-flight incremental indexing), so an index job must never
+//      sit queued behind long render sweeps.
+
+function makeStubQueue(registered: string[]): QueueRunner {
+  return {
+    register: (jobType: string) => {
+      registered.push(jobType);
+    },
+    start: async () => {},
+    destroy: async () => {},
+  } as QueueRunner;
+}
+
+function makeWorker(queue: QueueRunner, indexJobsOnly?: boolean) {
+  // Worker.run() only touches these dependencies at registration time via
+  // closures inside the task factories, so shallow stubs are sufficient —
+  // no task actually executes in these tests.
+  return new Worker({
+    indexWriter: {} as any,
+    queue,
+    dbAdapter: {} as any,
+    queuePublisher: {} as any,
+    virtualNetwork: { fetch: (() => {}) as any } as any,
+    matrixURL: new URL('http://localhost:8008'),
+    realmServerMatrixUsername: 'realm_server',
+    secretSeed: 'test-seed',
+    prerenderer: {} as any,
+    createPrerenderAuth: () => 'test-auth',
+    ...(indexJobsOnly !== undefined ? { indexJobsOnly } : {}),
+  });
+}
+
+module(basename(import.meta.filename), function () {
+  test('a default worker registers every job type', async function (assert) {
+    let registered: string[] = [];
+    await makeWorker(makeStubQueue(registered)).run();
+
+    assert.deepEqual(
+      registered.sort(),
+      [
+        'copy-index',
+        'daily-credit-grant',
+        'from-scratch-index',
+        'full-reindex',
+        'incremental-index',
+        'lint-source',
+        'prerender-html-reconcile',
+        'prerender_html',
+        'run-command',
+        'screenshot-card',
+      ],
+      'all job types are registered',
+    );
+  });
+
+  test('an indexJobsOnly worker registers exactly the indexing job types', async function (assert) {
+    let registered: string[] = [];
+    await makeWorker(makeStubQueue(registered), true).run();
+
+    assert.deepEqual(
+      registered.sort(),
+      [...INDEX_JOB_TYPES].sort(),
+      'only indexing job types are registered',
+    );
+    assert.notOk(
+      registered.includes('prerender_html'),
+      'the index lane cannot claim prerender_html jobs',
+    );
+  });
+});

packages/realm-server/worker-manager.ts

@@ -150,7 +150,7 @@ let {
     },
     userIndexCount: {
       description:
-        'The number of workers that service only user-initiated indexing (priority 10) and nothing below it — a dedicated index tier that never gets held by the slower user-initiated prerender-html jobs (default 0)',
+        'The number of workers that claim only indexing job types at the user-initiated priority — a dedicated index lane that can never be occupied by prerender-html or other job types (default 0)',
       type: 'number',
     },
     allPriorityCount: {
@@ -675,15 +675,22 @@ let adapter: PgAdapter;
   // claim jobs at or above it, oldest-first among those. User-initiated
   // indexing and prerender-html are co-equal (both `userInitiatedPriority`
   // — a published realm's rendered HTML is as first-class as its search
-  // index), so the user-index and high-priority pools both floor at that
-  // tier and serve all user-initiated work — indexing and prerender-html
-  // alike — and never system-tier jobs. The two counts stay separate knobs
-  // for deployment flexibility but land equivalent floor-10 workers; size
-  // the total to the user-work rate so neither indexing nor rendering
-  // starves the other. The all-priority pool floors at the lowest tier and
-  // serves everything, including system-initiated prerender-html.
+  // index), so a priority floor alone cannot keep the two kinds of work
+  // from occupying the same workers. What separates them is the claim
+  // query's job-type filter: a worker only dequeues job types it has
+  // registered handlers for, and the user-index pool starts with
+  // `--indexJobsOnly` so it registers exactly the indexing job types.
+  // That makes it a lane a prerender-html sweep can never hold — indexing
+  // gates realm provisioning and every write's read-your-writes drain, so
+  // it must stay responsive even when long render sweeps saturate the
+  // high-priority pool. The high-priority pool serves all user-initiated
+  // work, indexing and prerender-html alike; the all-priority pool floors
+  // at the lowest tier and serves everything, including system-initiated
+  // prerender-html.
   for (let i = 0; i < userIndexCount; i++) {
-    await startWorker(userInitiatedPriority, urlMappings);
+    await startWorker(userInitiatedPriority, urlMappings, {
+      indexJobsOnly: true,
+    });
   }
   for (let i = 0; i < highPriorityCount; i++) {
     await startWorker(userInitiatedPrerenderHtmlPriority, urlMappings);
@@ -849,6 +856,7 @@ async function markFailedJob({
 async function startWorker(
   priority: number,
   urlMappings: [URL | string, URL][],
+  opts?: { indexJobsOnly?: boolean },
 ) {
   let worker = spawn(
     'node',
@@ -857,6 +865,7 @@ async function startWorker(
       `--matrixURL='${matrixURL}'`,
       `--prerendererUrl=${prerendererUrl}`,
       `--priority=${priority}`,
+      ...(opts?.indexJobsOnly ? [`--indexJobsOnly`] : []),
       ...flattenDeep(
         urlMappings.map(([from, to]) => [
           `--fromUrl='${from instanceof URL ? from.href : from}'`,
@@ -900,7 +909,7 @@ async function startWorker(
       log.info(
         `worker ${name} exited (code=${code}, signal=${signal}). spawning replacement worker`,
       );
-      startWorker(priority, urlMappings);
+      startWorker(priority, urlMappings, opts);
     }
 
     // Free orphan reservations in the background. The new worker won't be

packages/realm-server/worker.ts

@@ -83,6 +83,7 @@ let {
   priority = 0,
   migrateDB,
   prerendererUrl,
+  indexJobsOnly = false,
 } = yargs(process.argv.slice(2))
   .usage('Start worker')
   .options({
@@ -116,10 +117,19 @@ let {
       demandOption: true,
       type: 'string',
     },
+    indexJobsOnly: {
+      description:
+        'When set, the worker only registers (and therefore only claims) indexing job types, making it a dedicated index lane that other job types cannot occupy',
+      type: 'boolean',
+    },
   })
   .parseSync();
 
-log.info(`starting worker with pid ${process.pid} and priority ${priority}`);
+log.info(
+  `starting worker with pid ${process.pid} and priority ${priority}${
+    indexJobsOnly ? ' (index jobs only)' : ''
+  }`,
+);
 
 let prerenderer = createRemotePrerenderer(prerendererUrl);
 let createPrerenderAuth = buildCreatePrerenderAuth(REALM_SECRET_SEED);
@@ -204,6 +214,7 @@ let autoMigrate = migrateDB || undefined;
     queuePublisher: new PgQueuePublisher(dbAdapter),
     prerenderer,
     createPrerenderAuth,
+    indexJobsOnly,
   });
 
   await worker.run();

packages/runtime-common/realm.ts

@@ -5988,10 +5988,17 @@ export class Realm {
     // +source POSTs, an immediately-following publishability call could
     // otherwise see a stale snapshot and miss real violations (e.g. a
     // leaky card that just landed but isn't indexed yet).
+    //
+    // The drain's wait is unbounded: the incremental job it awaits can sit
+    // queued behind other realms' work when the worker pool is saturated.
+    // The timing log below attributes a slow or empty-looking report to
+    // the drain vs. the scan itself.
+    let drainStartMs = Date.now();
     let pending = this.incrementalIndexing();
     if (pending) {
       await pending;
     }
+    let drainMs = Date.now() - drainStartMs;
     let sourceRealmURL = ensureTrailingSlash(this.url);
     let resourceEntries = new Map<string, ResourceIndexEntry[]>();
     let visibilityCache = new Map<string, RealmVisibility>();
@@ -6332,6 +6339,14 @@ export class Realm {
     let publishable =
       privateDependencyViolations.length === 0 && errorViolations.length === 0;
 
+    this.#log.info(
+      `publishability for ${sourceRealmURL}: drained incremental indexing ` +
+        `in ${drainMs}ms; ${rootResources.length} instances scanned, ` +
+        `${errorRows.length} error rows, ` +
+        `${privateDependencyViolations.length} private-dependency violations, ` +
+        `publishable=${publishable}`,
+    );
+
     let doc = {
       data: {
         type: 'realm-publishability',

packages/runtime-common/router.ts

@@ -210,6 +210,15 @@ export class Router {
       return await handler(request, requestContext);
     } catch (err) {
       if (err instanceof CardError) {
+        // Without this line a thrown CardError is indistinguishable in the
+        // request log from a handler that returned the same status
+        // deliberately (e.g. a 404 from a routed endpoint that can't
+        // otherwise produce one), which makes such failures undiagnosable
+        // from CI logs alone.
+        this.log.warn(
+          `handler for ${request.method} ${request.url} threw CardError ` +
+            `(status ${err.status}): ${err.message}`,
+        );
         return responseWithError(err, requestContext);
       }