diff --git a/packages/matrix/support/isolated-realm-server.ts b/packages/matrix/support/isolated-realm-server.ts index fdaaf1e58a6..820a44f50db 100644 --- a/packages/matrix/support/isolated-realm-server.ts +++ b/packages/matrix/support/isolated-realm-server.ts @@ -463,16 +463,19 @@ export async function startServer({ // Worker tiers for this shared, contended stack. Both Playwright // workers (fullyParallel) funnel their realm provisioning into one // worker manager, and each new workspace enqueues a from-scratch index - // plus a ~110-file, ~10s prerender-html sweep. User indexing and - // prerender-html are co-equal (both priority 10 — see - // runtime-common/queue.ts: a published realm's rendered HTML is as - // first-class as its search index), so these three high-tier workers all - // floor at 10 and serve both kinds of user work. What keeps indexing - // (which gates createRealm / new-user provisioning) and rendering (which - // gates published-realm serving) both moving is capacity, not a dedicated - // lane: three user-work workers absorb the two-Playwright-worker producer - // rate. (The split across the two count knobs is immaterial now that both - // floor at 10; kept separate only for symmetry with production.) + // plus a prerender-html job whose realm-wide module pre-warm sweep runs + // for tens of seconds. User indexing and prerender-html are co-equal + // (both priority 10 — see runtime-common/queue.ts: a published realm's + // rendered HTML is as first-class as its search index), so priority + // alone cannot keep sweeps from occupying every high-tier worker while + // a user write's incremental index job sits queued. That queue wait is + // what test assertions feel: createRealm blocks on the from-scratch + // index, and read-your-writes endpoints (card GET, _publishability) + // drain in-flight incremental indexing before responding. The + // user-index worker claims only indexing job types, so an index job + // always has a lane no render sweep can hold; the two high-priority + // workers carry the prerender-html sweeps alongside any other + // user-initiated work. `--userIndexCount=1`, `--highPriorityCount=2`, diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index cb985a86221..423de6c135e 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -377,6 +377,7 @@ const ALL_TEST_FILES: string[] = [ './worker-request-signature-test', './worker-request-forwarder-test', './worker-reader-test', + './worker-job-registration-test', './realm-index-updater-test', ]; diff --git a/packages/realm-server/tests/worker-job-registration-test.ts b/packages/realm-server/tests/worker-job-registration-test.ts new file mode 100644 index 00000000000..23f8ff989d3 --- /dev/null +++ b/packages/realm-server/tests/worker-job-registration-test.ts @@ -0,0 +1,90 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import { basename } from 'path'; +import { + Worker, + INDEX_JOB_TYPES, + type QueueRunner, +} from '@cardstack/runtime-common'; + +// The queue's claim query only dequeues job types a worker has registered +// handlers for, so a worker's registration set IS its claim policy. These +// tests pin the registration sets for the two worker flavors: +// +// 1. A default worker registers every job type. +// 2. An `indexJobsOnly` worker registers exactly INDEX_JOB_TYPES — that +// restriction is what makes the worker-manager's user-index pool a +// dedicated indexing lane that a prerender_html sweep (or any other +// job type) can never occupy, independent of priority tiers. Indexing +// gates realm provisioning (createRealm blocks on the from-scratch +// index) and read-your-writes endpoints (card GET / _publishability +// drain in-flight incremental indexing), so an index job must never +// sit queued behind long render sweeps. + +function makeStubQueue(registered: string[]): QueueRunner { + return { + register: (jobType: string) => { + registered.push(jobType); + }, + start: async () => {}, + destroy: async () => {}, + } as QueueRunner; +} + +function makeWorker(queue: QueueRunner, indexJobsOnly?: boolean) { + // Worker.run() only touches these dependencies at registration time via + // closures inside the task factories, so shallow stubs are sufficient — + // no task actually executes in these tests. + return new Worker({ + indexWriter: {} as any, + queue, + dbAdapter: {} as any, + queuePublisher: {} as any, + virtualNetwork: { fetch: (() => {}) as any } as any, + matrixURL: new URL('http://localhost:8008'), + realmServerMatrixUsername: 'realm_server', + secretSeed: 'test-seed', + prerenderer: {} as any, + createPrerenderAuth: () => 'test-auth', + ...(indexJobsOnly !== undefined ? { indexJobsOnly } : {}), + }); +} + +module(basename(import.meta.filename), function () { + test('a default worker registers every job type', async function (assert) { + let registered: string[] = []; + await makeWorker(makeStubQueue(registered)).run(); + + assert.deepEqual( + registered.sort(), + [ + 'copy-index', + 'daily-credit-grant', + 'from-scratch-index', + 'full-reindex', + 'incremental-index', + 'lint-source', + 'prerender-html-reconcile', + 'prerender_html', + 'run-command', + 'screenshot-card', + ], + 'all job types are registered', + ); + }); + + test('an indexJobsOnly worker registers exactly the indexing job types', async function (assert) { + let registered: string[] = []; + await makeWorker(makeStubQueue(registered), true).run(); + + assert.deepEqual( + registered.sort(), + [...INDEX_JOB_TYPES].sort(), + 'only indexing job types are registered', + ); + assert.notOk( + registered.includes('prerender_html'), + 'the index lane cannot claim prerender_html jobs', + ); + }); +}); diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index d6674e0ec13..86593959f6c 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -150,7 +150,7 @@ let { }, userIndexCount: { description: - 'The number of workers that service only user-initiated indexing (priority 10) and nothing below it — a dedicated index tier that never gets held by the slower user-initiated prerender-html jobs (default 0)', + 'The number of workers that claim only indexing job types at the user-initiated priority — a dedicated index lane that can never be occupied by prerender-html or other job types (default 0)', type: 'number', }, allPriorityCount: { @@ -675,15 +675,22 @@ let adapter: PgAdapter; // claim jobs at or above it, oldest-first among those. User-initiated // indexing and prerender-html are co-equal (both `userInitiatedPriority` // — a published realm's rendered HTML is as first-class as its search - // index), so the user-index and high-priority pools both floor at that - // tier and serve all user-initiated work — indexing and prerender-html - // alike — and never system-tier jobs. The two counts stay separate knobs - // for deployment flexibility but land equivalent floor-10 workers; size - // the total to the user-work rate so neither indexing nor rendering - // starves the other. The all-priority pool floors at the lowest tier and - // serves everything, including system-initiated prerender-html. + // index), so a priority floor alone cannot keep the two kinds of work + // from occupying the same workers. What separates them is the claim + // query's job-type filter: a worker only dequeues job types it has + // registered handlers for, and the user-index pool starts with + // `--indexJobsOnly` so it registers exactly the indexing job types. + // That makes it a lane a prerender-html sweep can never hold — indexing + // gates realm provisioning and every write's read-your-writes drain, so + // it must stay responsive even when long render sweeps saturate the + // high-priority pool. The high-priority pool serves all user-initiated + // work, indexing and prerender-html alike; the all-priority pool floors + // at the lowest tier and serves everything, including system-initiated + // prerender-html. for (let i = 0; i < userIndexCount; i++) { - await startWorker(userInitiatedPriority, urlMappings); + await startWorker(userInitiatedPriority, urlMappings, { + indexJobsOnly: true, + }); } for (let i = 0; i < highPriorityCount; i++) { await startWorker(userInitiatedPrerenderHtmlPriority, urlMappings); @@ -849,6 +856,7 @@ async function markFailedJob({ async function startWorker( priority: number, urlMappings: [URL | string, URL][], + opts?: { indexJobsOnly?: boolean }, ) { let worker = spawn( 'node', @@ -857,6 +865,7 @@ async function startWorker( `--matrixURL='${matrixURL}'`, `--prerendererUrl=${prerendererUrl}`, `--priority=${priority}`, + ...(opts?.indexJobsOnly ? [`--indexJobsOnly`] : []), ...flattenDeep( urlMappings.map(([from, to]) => [ `--fromUrl='${from instanceof URL ? from.href : from}'`, @@ -900,7 +909,7 @@ async function startWorker( log.info( `worker ${name} exited (code=${code}, signal=${signal}). spawning replacement worker`, ); - startWorker(priority, urlMappings); + startWorker(priority, urlMappings, opts); } // Free orphan reservations in the background. The new worker won't be diff --git a/packages/realm-server/worker.ts b/packages/realm-server/worker.ts index cf580ce8133..87751c131a0 100644 --- a/packages/realm-server/worker.ts +++ b/packages/realm-server/worker.ts @@ -83,6 +83,7 @@ let { priority = 0, migrateDB, prerendererUrl, + indexJobsOnly = false, } = yargs(process.argv.slice(2)) .usage('Start worker') .options({ @@ -116,10 +117,19 @@ let { demandOption: true, type: 'string', }, + indexJobsOnly: { + description: + 'When set, the worker only registers (and therefore only claims) indexing job types, making it a dedicated index lane that other job types cannot occupy', + type: 'boolean', + }, }) .parseSync(); -log.info(`starting worker with pid ${process.pid} and priority ${priority}`); +log.info( + `starting worker with pid ${process.pid} and priority ${priority}${ + indexJobsOnly ? ' (index jobs only)' : '' + }`, +); let prerenderer = createRemotePrerenderer(prerendererUrl); let createPrerenderAuth = buildCreatePrerenderAuth(REALM_SECRET_SEED); @@ -204,6 +214,7 @@ let autoMigrate = migrateDB || undefined; queuePublisher: new PgQueuePublisher(dbAdapter), prerenderer, createPrerenderAuth, + indexJobsOnly, }); await worker.run(); diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 19a66489bc3..562fb3457c7 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -5988,10 +5988,17 @@ export class Realm { // +source POSTs, an immediately-following publishability call could // otherwise see a stale snapshot and miss real violations (e.g. a // leaky card that just landed but isn't indexed yet). + // + // The drain's wait is unbounded: the incremental job it awaits can sit + // queued behind other realms' work when the worker pool is saturated. + // The timing log below attributes a slow or empty-looking report to + // the drain vs. the scan itself. + let drainStartMs = Date.now(); let pending = this.incrementalIndexing(); if (pending) { await pending; } + let drainMs = Date.now() - drainStartMs; let sourceRealmURL = ensureTrailingSlash(this.url); let resourceEntries = new Map(); let visibilityCache = new Map(); @@ -6332,6 +6339,14 @@ export class Realm { let publishable = privateDependencyViolations.length === 0 && errorViolations.length === 0; + this.#log.info( + `publishability for ${sourceRealmURL}: drained incremental indexing ` + + `in ${drainMs}ms; ${rootResources.length} instances scanned, ` + + `${errorRows.length} error rows, ` + + `${privateDependencyViolations.length} private-dependency violations, ` + + `publishable=${publishable}`, + ); + let doc = { data: { type: 'realm-publishability', diff --git a/packages/runtime-common/router.ts b/packages/runtime-common/router.ts index 4ba7800f692..7741a2e1643 100644 --- a/packages/runtime-common/router.ts +++ b/packages/runtime-common/router.ts @@ -210,6 +210,15 @@ export class Router { return await handler(request, requestContext); } catch (err) { if (err instanceof CardError) { + // Without this line a thrown CardError is indistinguishable in the + // request log from a handler that returned the same status + // deliberately (e.g. a 404 from a routed endpoint that can't + // otherwise produce one), which makes such failures undiagnosable + // from CI logs alone. + this.log.warn( + `handler for ${request.method} ${request.url} threw CardError ` + + `(status ${err.status}): ${err.message}`, + ); return responseWithError(err, requestContext); } diff --git a/packages/runtime-common/worker.ts b/packages/runtime-common/worker.ts index f262f45571c..ff8a31cc5dd 100644 --- a/packages/runtime-common/worker.ts +++ b/packages/runtime-common/worker.ts @@ -125,6 +125,17 @@ export interface IndexingProgressEvent { stats?: Stats; } +// The job types an `indexJobsOnly` worker registers. The queue's claim +// query only dequeues job types a worker has registered handlers for, so +// restricting registration is what makes such a worker an indexing-only +// lane: it can never be held by a prerender-html sweep (or any other job +// type), no matter how the priority tiers are configured. +export const INDEX_JOB_TYPES = [ + 'from-scratch-index', + 'incremental-index', + 'copy-index', +] as const; + export class Worker { #log = logger('worker'); #indexWriter: IndexWriter; @@ -141,6 +152,7 @@ export class Worker { #reportProgress: ((event: IndexingProgressEvent) => void) | undefined; #reportRealmEvent: ((event: RealmEventContent) => void) | undefined; #realmServerMatrixUsername; + #indexJobsOnly: boolean; #createPrerenderAuth: ( userId: string, permissions: RealmPermissions, @@ -160,6 +172,7 @@ export class Worker { reportRealmEvent, prerenderer, createPrerenderAuth, + indexJobsOnly, }: { indexWriter: IndexWriter; queue: QueueRunner; @@ -173,6 +186,9 @@ export class Worker { reportStatus?: (args: StatusArgs) => void; reportProgress?: (event: IndexingProgressEvent) => void; reportRealmEvent?: (event: RealmEventContent) => void; + // When true, register handlers only for INDEX_JOB_TYPES so this worker + // is a dedicated indexing lane — see INDEX_JOB_TYPES above. + indexJobsOnly?: boolean; createPrerenderAuth: ( userId: string, permissions: RealmPermissions, @@ -191,6 +207,7 @@ export class Worker { this.#queuePublisher = queuePublisher; this.#prerenderer = prerenderer; this.#createPrerenderAuth = createPrerenderAuth; + this.#indexJobsOnly = indexJobsOnly ?? false; } async run() { @@ -217,33 +234,50 @@ export class Worker { createPrerenderAuth: this.#createPrerenderAuth, }; - await Promise.all([ - this.#queue.register( - `from-scratch-index`, - Tasks['fromScratchIndex'](taskArgs), - ), - this.#queue.register( - `incremental-index`, - Tasks['incrementalIndex'](taskArgs), - ), - this.#queue.register(`prerender_html`, Tasks['prerenderHtml'](taskArgs)), - this.#queue.register( - `prerender-html-reconcile`, - Tasks['prerenderHtmlReconcile'](taskArgs), - ), - this.#queue.register(`copy-index`, Tasks['copy'](taskArgs)), - this.#queue.register(`lint-source`, Tasks['lintSource'](taskArgs)), - this.#queue.register(`full-reindex`, Tasks['fullReindex'](taskArgs)), - this.#queue.register( - `daily-credit-grant`, - Tasks['dailyCreditGrant'](taskArgs), - ), - this.#queue.register(`run-command`, Tasks['runCommand'](taskArgs)), - this.#queue.register( - `screenshot-card`, - Tasks['screenshotCard'](taskArgs), - ), - ]); + let registrations: Record Promise | unknown> = { + 'from-scratch-index': () => + this.#queue.register( + `from-scratch-index`, + Tasks['fromScratchIndex'](taskArgs), + ), + 'incremental-index': () => + this.#queue.register( + `incremental-index`, + Tasks['incrementalIndex'](taskArgs), + ), + prerender_html: () => + this.#queue.register( + `prerender_html`, + Tasks['prerenderHtml'](taskArgs), + ), + 'prerender-html-reconcile': () => + this.#queue.register( + `prerender-html-reconcile`, + Tasks['prerenderHtmlReconcile'](taskArgs), + ), + 'copy-index': () => + this.#queue.register(`copy-index`, Tasks['copy'](taskArgs)), + 'lint-source': () => + this.#queue.register(`lint-source`, Tasks['lintSource'](taskArgs)), + 'full-reindex': () => + this.#queue.register(`full-reindex`, Tasks['fullReindex'](taskArgs)), + 'daily-credit-grant': () => + this.#queue.register( + `daily-credit-grant`, + Tasks['dailyCreditGrant'](taskArgs), + ), + 'run-command': () => + this.#queue.register(`run-command`, Tasks['runCommand'](taskArgs)), + 'screenshot-card': () => + this.#queue.register( + `screenshot-card`, + Tasks['screenshotCard'](taskArgs), + ), + }; + let jobTypes = this.#indexJobsOnly + ? (INDEX_JOB_TYPES as readonly string[]) + : Object.keys(registrations); + await Promise.all(jobTypes.map((jobType) => registrations[jobType]())); await this.#queue.start(); }