diff --git a/packages/boxel-cli/src/commands/realm/publish.ts b/packages/boxel-cli/src/commands/realm/publish.ts index 00ce530de33..b78856be7af 100644 --- a/packages/boxel-cli/src/commands/realm/publish.ts +++ b/packages/boxel-cli/src/commands/realm/publish.ts @@ -159,6 +159,9 @@ export async function publishRealm( await waitForReady(client, { publishedRealmURL: result.publishedRealmURL, timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + // A published realm's rendered HTML is its deliverable, so wait until it + // is live (not just indexed) before reporting the publish complete. + awaitPrerenderHtml: true, }); } diff --git a/packages/host/app/services/realm-server.ts b/packages/host/app/services/realm-server.ts index 95c423d2ce0..a1bfbbff971 100644 --- a/packages/host/app/services/realm-server.ts +++ b/packages/host/app/services/realm-server.ts @@ -1222,6 +1222,10 @@ export default class RealmServerService extends Service { publishedRealmURL, timeoutMs: opts?.timeoutMs, pollIntervalMs: opts?.pollIntervalMs, + // A published realm's rendered HTML is its deliverable, so hold the + // Publish UI's "Publishing…" state until the HTML is live, not just the + // index. + awaitPrerenderHtml: true, }); } diff --git a/packages/matrix/support/isolated-realm-server.ts b/packages/matrix/support/isolated-realm-server.ts index b7e8410d263..fdaaf1e58a6 100644 --- a/packages/matrix/support/isolated-realm-server.ts +++ b/packages/matrix/support/isolated-realm-server.ts @@ -463,25 +463,17 @@ export async function startServer({ // Worker tiers for this shared, contended stack. Both Playwright // workers (fullyParallel) funnel their realm provisioning into one // worker manager, and each new workspace enqueues a from-scratch index - // (priority 10) plus a ~110-file, ~10s prerender-html sweep (priority 9). - // Priority is a pool-reservation floor, not an ordering — within a pool - // the queue dequeues oldest-first (see runtime-common/queue.ts) — so a - // newer index job is NOT pulled ahead of an already-queued prerender-html - // sweep in the same pool. That is what made createRealm (and new-user - // provisioning, which blocks on a personal-realm index before the - // workspace chooser renders) time out: the index sat behind ~10s render - // sweeps until it blew its settle budget. - // - // A dedicated user-index worker (floor 10) is the fix: it claims only - // indexing jobs and never the slower prerender-html tier below it, so an - // index job always has a lane that a render sweep can't hold. Index jobs - // are short (a fresh realm indexes in a few seconds once dequeued), so - // one is enough for the two-Playwright-worker producer rate. + // plus a ~110-file, ~10s prerender-html sweep. User indexing and + // prerender-html are co-equal (both priority 10 — see + // runtime-common/queue.ts: a published realm's rendered HTML is as + // first-class as its search index), so these three high-tier workers all + // floor at 10 and serve both kinds of user work. What keeps indexing + // (which gates createRealm / new-user provisioning) and rendering (which + // gates published-realm serving) both moving is capacity, not a dedicated + // lane: three user-work workers absorb the two-Playwright-worker producer + // rate. (The split across the two count knobs is immaterial now that both + // floor at 10; kept separate only for symmetry with production.) `--userIndexCount=1`, - // Two high-priority workers still carry the prerender-html sweeps (and - // spill over onto indexing when free). Keep two, not one: republishing - // waits on the prerender-html job to regenerate the published HTML, so - // this tier must not become the new bottleneck. `--highPriorityCount=2`, `--fromUrl='https://localhost:4205/test/'`, diff --git a/packages/matrix/tests/host-mode.spec.ts b/packages/matrix/tests/host-mode.spec.ts index 13ba6cd9e96..1489c3cd231 100644 --- a/packages/matrix/tests/host-mode.spec.ts +++ b/packages/matrix/tests/host-mode.spec.ts @@ -60,6 +60,31 @@ async function publishRealm( }, { realmURL, publishedRealmURL }, ); + + // Publishing is 202/async. A published realm's rendered HTML is its + // deliverable, so wait until its readiness check reports both indexed AND + // rendered — `awaitPrerenderHtml` holds the readiness response until the + // prerendered HTML is live, not just the index (readiness returns 503 while + // it isn't). Poll until 200 rather than polling the served URL and racing + // the HTML job. + let readinessURL = `${publishedRealmURL}_readiness-check?awaitPrerenderHtml=true`; + let lastStatus: number | undefined; + try { + await waitUntil(async () => { + let readiness = await page.request.get(readinessURL, { + headers: { Accept: 'text/html' }, + timeout: 120_000, + }); + lastStatus = readiness.status(); + return readiness.ok(); + }, 120_000); + } catch { + throw new Error( + `published realm did not become ready (indexed + rendered): last HTTP ${ + lastStatus ?? 'none' + } for ${readinessURL}`, + ); + } } // Create a fresh source realm, seed it with the host-mode fixture cards, and diff --git a/packages/realm-server/handlers/handle-publish-realm.ts b/packages/realm-server/handlers/handle-publish-realm.ts index 4eaed13755f..2cb42f5750a 100644 --- a/packages/realm-server/handlers/handle-publish-realm.ts +++ b/packages/realm-server/handlers/handle-publish-realm.ts @@ -689,8 +689,11 @@ export default function handlePublishRealm({ // realm's readiness check, which resolves once it is indexed and // viewable, and `Retry-After` hints the poll interval. This lets a // consumer discover where to wait for completion from the response - // itself rather than hard-coding the readiness URL. - let readinessCheckURL = `${publishedRealmURL}_readiness-check`; + // itself rather than hard-coding the readiness URL. `awaitPrerenderHtml` + // holds that readiness until the rendered HTML is live, not just the + // index — a published realm's HTML is its deliverable, so a publish is + // not complete until it exists. + let readinessCheckURL = `${publishedRealmURL}_readiness-check?awaitPrerenderHtml=true`; let response = new Response( JSON.stringify( { diff --git a/packages/realm-server/tests/publish-unpublish-realm-test.ts b/packages/realm-server/tests/publish-unpublish-realm-test.ts index 94b61045dec..f62909942eb 100644 --- a/packages/realm-server/tests/publish-unpublish-realm-test.ts +++ b/packages/realm-server/tests/publish-unpublish-realm-test.ts @@ -257,8 +257,8 @@ module(basename(import.meta.filename), function () { ); assert.strictEqual( response.headers['location'], - `${response.body.data.attributes.publishedRealmURL}_readiness-check`, - 'Location points at the readiness-check status monitor for the 202', + `${response.body.data.attributes.publishedRealmURL}_readiness-check?awaitPrerenderHtml=true`, + 'Location points at the readiness-check status monitor for the 202, gated on the published HTML being rendered', ); assert.ok( response.headers['retry-after'], diff --git a/packages/realm-server/worker-manager.ts b/packages/realm-server/worker-manager.ts index 94ef8ac3fa7..d6674e0ec13 100644 --- a/packages/realm-server/worker-manager.ts +++ b/packages/realm-server/worker-manager.ts @@ -672,16 +672,16 @@ let adapter: PgAdapter; eventSink.setAdapter(adapter); // Each pool's minimum priority is a dequeue floor: its workers only - // claim jobs at or above it, oldest-first among those. The user-index - // pool floors at the user-initiated indexing tier, so it claims only - // user indexing jobs and never the (orders-of-magnitude slower) - // prerender-html work one tier below — a dedicated lane that keeps - // index jobs (which gate createRealm and new-user provisioning) from - // waiting behind a prerender-html sweep already queued in the shared - // high-priority pool. The high-priority pool floors at the - // user-initiated prerender-html tier so it serves all user-initiated - // work — prerender-html included — and never system-tier jobs; the - // all-priority pool floors at the lowest tier and serves everything. + // claim jobs at or above it, oldest-first among those. User-initiated + // indexing and prerender-html are co-equal (both `userInitiatedPriority` + // — a published realm's rendered HTML is as first-class as its search + // index), so the user-index and high-priority pools both floor at that + // tier and serve all user-initiated work — indexing and prerender-html + // alike — and never system-tier jobs. The two counts stay separate knobs + // for deployment flexibility but land equivalent floor-10 workers; size + // the total to the user-work rate so neither indexing nor rendering + // starves the other. The all-priority pool floors at the lowest tier and + // serves everything, including system-initiated prerender-html. for (let i = 0; i < userIndexCount; i++) { await startWorker(userInitiatedPriority, urlMappings); } diff --git a/packages/runtime-common/jobs/prerender-html.ts b/packages/runtime-common/jobs/prerender-html.ts index ba6862981e3..cb154546d00 100644 --- a/packages/runtime-common/jobs/prerender-html.ts +++ b/packages/runtime-common/jobs/prerender-html.ts @@ -5,14 +5,16 @@ import { type Job, type QueuePublisher, } from '../queue.ts'; -import type { PgPrimitive } from '../expression.ts'; +import { param, query, type PgPrimitive } from '../expression.ts'; +import type { DBAdapter } from '../db.ts'; +import { Deferred } from '../deferred.ts'; import type { IncrementalChange } from '../tasks/indexer.ts'; import type { PrerenderHtmlArgs } from '../tasks/prerender-html.ts'; -// The prerender-html tier sits one notch below its initiator's tier (see the -// tier table in queue.ts): the high-priority worker pool still serves -// user-initiated HTML work, while system-initiated HTML work drops to the -// tier only the all-priority pool takes. +// User-initiated HTML work shares its initiator's tier — for a published +// realm the rendered HTML is a first-class artifact, as important as the +// search index (see the tier table in queue.ts). System-initiated HTML work +// still drops to the background tier only the all-priority pool takes. export function prerenderHtmlPriority(spawningPriority: number): number { return spawningPriority >= userInitiatedPriority ? userInitiatedPrerenderHtmlPriority @@ -42,6 +44,126 @@ export function prerenderHtmlConcurrencyGroup(realmURL: string): string { return `prerender-html:${realmURL}`; } +// Await the prerender-html channel having caught up to a realm's current +// index generation. The index pass spawns the prerender-html job +// fire-and-forget and completes without it, so "indexed" does not imply +// "viewable" — a freshly published realm can be reachable and searchable +// while still serving a shell without card markup. Publishing awaits this so a +// published realm reports ready only once its current generation has been +// rendered. +// +// Signal: `prerendered_html` carrying any row at >= the realm's current +// generation. `batch.done()` swaps a generation's rendered rows into the +// production table atomically, so a row at the current generation means that +// generation's render batch has landed (a successful render leaves the +// isolated HTML on those rows; a failed one leaves error rows — either way the +// async render work for this publish is done, and a genuine render failure +// surfaces downstream as missing markup rather than hanging readiness). Gating +// on `generation` (not job status) sidesteps the fire-and-forget enqueue race +// and never settles on a prior publish's stale (lower-generation) rows. +// Resolves true when caught up, false on timeout. +// +// Woken by NOTIFY rather than tight-polling: pg-queue emits `NOTIFY +// jobs_finished` when a job's finalize transaction commits, and the +// prerender-html batch's swap is already durable by then, so re-checking on +// that signal catches the current generation landing near-instantly. The +// periodic poll is a safety net for a missed notification and for adapters +// without pub/sub (SQLite has no LISTEN), so it stays coarse. +export async function awaitPublishedHtmlReady( + dbAdapter: DBAdapter, + realmURL: string, + opts?: { timeoutMs?: number; pollIntervalMs?: number }, +): Promise { + let timeoutMs = opts?.timeoutMs ?? 60_000; + let pollIntervalMs = opts?.pollIntervalMs ?? 1000; + let [genRow] = (await query(dbAdapter, [ + 'SELECT current_generation FROM realm_generations WHERE realm_url =', + param(realmURL), + ])) as { current_generation: number }[]; + let currentGeneration = genRow?.current_generation; + if (currentGeneration == null) { + // The realm has never been indexed — there is no generation to await. + return true; + } + + let hasCaughtUp = async () => { + let rows = await query(dbAdapter, [ + 'SELECT 1 FROM prerendered_html WHERE realm_url =', + param(realmURL), + 'AND generation >=', + param(currentGeneration), + 'LIMIT 1', + ]); + return rows.length > 0; + }; + + if (await hasCaughtUp()) { + return true; + } + + // Feature-detected: PgAdapter exposes `subscribe`; SQLite does not and falls + // back to the poll below. + let subscribe = ( + dbAdapter as unknown as { + subscribe?: ( + channel: string, + handler: () => void, + ) => Promise<{ unsubscribe: () => Promise }>; + } + ).subscribe; + + let ready = new Deferred(); + let settled = false; + let settle = (value: boolean) => { + if (!settled) { + settled = true; + ready.fulfill(value); + } + }; + let recheck = () => { + hasCaughtUp().then( + (caughtUp) => { + if (caughtUp) { + settle(true); + } + }, + () => { + // A transient query error just waits for the next signal / poll tick. + }, + ); + }; + + let subscription: { unsubscribe: () => Promise } | undefined; + let poll = setInterval(recheck, pollIntervalMs); + let timer = setTimeout(() => settle(false), timeoutMs); + // Subscribe fire-and-forget: the poll is the guarantee, so a slow or failed + // LISTEN must never block the result or the timeout. If it comes up after + // we've already settled, just tear it down; otherwise re-check once, since a + // row may have landed between the check above and the LISTEN establishing. + if (subscribe) { + subscribe.call(dbAdapter, 'jobs_finished', recheck).then( + (sub) => { + if (settled) { + void sub.unsubscribe(); + } else { + subscription = sub; + recheck(); + } + }, + () => { + // LISTEN setup failed — rely on the poll. + }, + ); + } + try { + return await ready.promise; + } finally { + clearInterval(poll); + clearTimeout(timer); + await subscription?.unsubscribe(); + } +} + // Publish a `prerender_html` job through the normal queue-publish path. The // registered coalesce handler (tasks/prerender-html.ts) merges same-realm // publishes: per-URL update-wins merge, max generation/priority/timeout. diff --git a/packages/runtime-common/queue.ts b/packages/runtime-common/queue.ts index ee9d57cffca..326e85fa422 100644 --- a/packages/runtime-common/queue.ts +++ b/packages/runtime-common/queue.ts @@ -8,23 +8,23 @@ import type { Deferred } from './deferred.ts'; // // The tiers: // -// | priority | job | -// | -------- | ---------------------------------- | -// | 10 | any user-initiated job | -// | 9 | user-initiated prerender-html | -// | 1 | any system-initiated job | -// | 0 | system-initiated prerender-html | +// | priority | job | +// | -------- | -------------------------------------------- | +// | 10 | any user-initiated job, incl. prerender-html | +// | 1 | system-initiated job (non-prerender-html) | +// | 0 | system-initiated prerender-html | // -// The prerender-html tiers sit one notch below their initiator tier so -// a pool's floor can take in an initiator's plain jobs with or without -// its (orders-of-magnitude slower) HTML rendering work. Two pools -// exist: the high-priority pool floors at -// `userInitiatedPrerenderHtmlPriority`, serving all user-initiated -// work — prerender-html included — and never system-tier jobs; the -// all-priority pool floors at `systemInitiatedPrerenderHtmlPriority` -// and serves everything. +// User-initiated prerender-html is co-equal with user indexing (both 10): +// for a published realm the rendered HTML is the deliverable served to +// visitors — as important as the search index — so it is NOT deprioritized +// below its initiator tier. System-initiated prerender-html stays one notch +// below system work (background); boot rendering is gated separately and must +// not crowd out user-tier jobs. The high-priority pool floors at +// `userInitiatedPrerenderHtmlPriority` (serving all user-initiated work and +// never system-tier jobs); the all-priority pool floors at +// `systemInitiatedPrerenderHtmlPriority` and serves everything. export const userInitiatedPriority = 10; -export const userInitiatedPrerenderHtmlPriority = 9; +export const userInitiatedPrerenderHtmlPriority = 10; export const systemInitiatedPriority = 1; export const systemInitiatedPrerenderHtmlPriority = 0; diff --git a/packages/runtime-common/realm-operations.ts b/packages/runtime-common/realm-operations.ts index 8be949d1bf0..01752b855c4 100644 --- a/packages/runtime-common/realm-operations.ts +++ b/packages/runtime-common/realm-operations.ts @@ -286,6 +286,11 @@ export interface WaitForReadyInput { timeoutMs?: number; // Defaults to 1000ms. pollIntervalMs?: number; + // When true, hold readiness until the realm's published HTML is live for its + // current generation, not just the index. A published realm's rendered HTML + // is its deliverable (served to visitors), so publish callers set this; + // index-only readiness (e.g. createRealm) leaves it off to stay fast. + awaitPrerenderHtml?: boolean; } // Polls `_readiness-check` until it returns ok (the realm is @@ -302,7 +307,11 @@ export const waitForReady: RealmOperation = async ( let timeoutMs = input.timeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS; let pollIntervalMs = input.pollIntervalMs ?? DEFAULT_READINESS_POLL_INTERVAL_MS; - let readinessUrl = new URL('_readiness-check', publishedRealmURL).href; + let readinessUrlObj = new URL('_readiness-check', publishedRealmURL); + if (input.awaitPrerenderHtml) { + readinessUrlObj.searchParams.set('awaitPrerenderHtml', 'true'); + } + let readinessUrl = readinessUrlObj.href; let startedAt = Date.now(); let lastError: string | undefined; diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index a594b20e147..f9c7ac903f2 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -1,4 +1,5 @@ import { Deferred } from './deferred.ts'; +import { awaitPublishedHtmlReady } from './jobs/prerender-html.ts'; import type { RealmVisibility } from './realm-visibility.ts'; import type { SearchOpts } from './search-utils.ts'; import { buildSearchErrorBody, SearchRequestError } from './search-utils.ts'; @@ -1171,7 +1172,7 @@ export class Realm { } private async readinessCheck( - _request: Request, + request: Request, requestContext: RequestContext, ) { await this.#startedUp.promise; @@ -1180,12 +1181,40 @@ export class Realm { // resolved #startedUp, so awaiting it alone would report ready before the // reindex of the swapped files completes. Also await any in-flight full or // incremental index so a publish poll only succeeds once the just-published - // content is indexed and viewable. + // content is indexed. let inflight = this.indexing(); if (inflight) { await inflight; } + // Opt-in: also await the published HTML being live for the current + // generation. Indexing makes a realm searchable; prerendering makes it + // viewable — and for a published realm the HTML is the deliverable. That + // work lands on a separate (fire-and-forget) channel, so the publish flow + // sets `awaitPrerenderHtml` to hold readiness until the rendered HTML + // exists, not just the index. Left off by default so createRealm / boot + // readiness stay index-only and fast. + if ( + new URL(request.url).searchParams.get('awaitPrerenderHtml') === 'true' + ) { + let htmlReady = await awaitPublishedHtmlReady(this.#dbAdapter, this.url); + if (!htmlReady) { + // The current generation's HTML never became live within budget (a + // stuck/failed render, or a queue backlog longer than the wait). Report + // not-ready rather than a false 200 so a poller keeps waiting and a + // single-shot caller sees the failure instead of treating the publish + // as complete on an unrendered realm. + return createResponse({ + body: null, + init: { + headers: { 'content-type': 'text/html', 'Retry-After': '1' }, + status: 503, + }, + requestContext, + }); + } + } + return createResponse({ body: null, init: {