diff --git a/scripts/recovery_e2e.py b/scripts/recovery_e2e.py index 72db69bd2..a0ebe484c 100644 --- a/scripts/recovery_e2e.py +++ b/scripts/recovery_e2e.py @@ -14,14 +14,80 @@ Usage:: - python3 scripts/recovery_e2e.py # run all three scenarios + python3 scripts/recovery_e2e.py # run every scenario python3 scripts/recovery_e2e.py --scenario storm python3 scripts/recovery_e2e.py --scenario restart python3 scripts/recovery_e2e.py --scenario coord-restart + python3 scripts/recovery_e2e.py --scenario fail-refetch # D (#890) + python3 scripts/recovery_e2e.py --scenario stale-ref-reload # E1 (#890) + python3 scripts/recovery_e2e.py --scenario rewind-window # E2 (#890) + python3 scripts/recovery_e2e.py --scenario rewind-failed-window # E3 (#890) + python3 scripts/recovery_e2e.py --scenario stale-backstop # E4 (#890) python3 scripts/recovery_e2e.py --scenario both # A+B only (legacy) python3 scripts/recovery_e2e.py --keep-open 8971 # serve the storm page # for manual inspection +The #890 guard-before-wipe family (all against the interactive pane, no +coordinator): + +Scenario D (fail-refetch): clones B, but arms one forced /history 500 +(``RecoveryServer.fail_history``) just before the show edge so the first +truncated resync FAILS. Asserts the failed fetch is a DOM/ref no-op — the +pre-restart rows stay, no empty-state, sentinel un-healed — with the backend +proof ``history_fail_remaining == 0``; then the connect-chokepoint retry's +second /history heals it. Stamps ``RECOVERY-READY-FAILFETCH-stale1-healed1``. + +Scenario E1 (stale-ref-reload): a mid-content transport teardown leaves a +live assistant-bubble ref; a same-ws UNARMED re-auth reload (the factory's +onLogin fan-out) must reset it so the next turn's text builds a FRESH bubble +instead of concatenating into the stale one. Stamps +``RECOVERY-READY-STALEREF-fresh1``. + +Scenario E2 (rewind-window): three completed turns; a REAL rewind click POSTs +and its clear_ui refetch is held open (``RecoveryServer.delay_history``), +keeping the quiesce armed; a second REAL rewind click mid-rebuild must be +gated by ``busy || _historyStale`` and never reach the server. Backend proof +``rewind_requests == 1``. Stamps ``RECOVERY-READY-REWINDWIN-posts1``. + +Scenario E3 (rewind-failed-window): the FAILED-refetch sibling of E2. Three +completed turns; a REAL rewind click POSTs and its clear_ui refetch is forced +to 500 (``RecoveryServer.fail_history``) instead of held open. The failed +fetch releases the transient ``_replayQueue`` quiesce but the ``_historyStale`` +latch SURVIVES (cleared ONLY by a successful ``replayHistory`` render), so a +second REAL rewind click over the stale-but-real transcript must stay gated by +``busy || _historyStale`` and never reach the server — the exact aftermath +where pre-latch (quiesce-only) code reopened the gate and let a second rewind +over-rewind. The bounded 2s retry then heals the transcript (rewound to ONE +user row) and reopens the gate so a fresh rewind legitimately lands. Backend +proofs: ``rewind_requests`` 1 -> (gated) 1 -> 2, ``history_fail_remaining == +0``, ``history_requests >= 2``. Stamps +``RECOVERY-READY-REWINDFAIL-posts2-heal1``. + +Scenario E4 (stale-backstop): the DOUBLE-failure sibling of E3, proving the +``_historyStale`` latch's TRANSPORT-FREE idle-edge backstop (#890, the +round-5 critical). Three completed turns; a REAL rewind click POSTs and +BOTH its clear_ui refetch AND its one bounded 2s retry are forced to 500 +(``RecoveryServer.fail_history(2)``), so the latch cannot self-heal and +rewind/edit stay latch-gated over the stale-but-real transcript (a row-0 +rewind click stays gated, ``rewind_requests`` holds at 1; the three user +rows survive). A plain send — sends are deliberately NOT latch-gated — runs +a fourth scripted ``final_text`` turn whose ORGANIC turn-settle idle edge +fires the backstop: a quiesced, same-token REST ``_refetchHistory`` +(deliberately NOT ``_loadHistoryThenConnect`` — the old reload backstop drew +the server's synthetic ``state_change:idle`` on its fresh reconnect and +re-triggered itself, a zero-backoff reconnect/refetch storm against a +recovering node). With the fault budget now exhausted the refetch succeeds +and rebuilds the rewound (ONE user turn, index 1 of 3 rewinds 2) + sent (a +second user turn) transcript to TWO user rows, clearing the latch, so a +fresh rewind on a remaining row lands (``rewind_requests`` -> 2). THE r5 +PROOF, both counted at the fault layer: ``events_requests`` is UNCHANGED +across the whole heal episode (``sse0`` — zero new EventSource connections; +the storm would have opened one per reconnect) and ``history_requests`` grew +by exactly ONE (the backstop's single fetch — the plain fourth turn emits no +clear_ui). All polls are deadline-bounded so a regressed looping backstop +stamps a clean FAILED, never a hang. Stamps +``RECOVERY-READY-STALEBACKSTOP-heal1-sse0``. + Scenario A (storm): the page connects, POSTs ``/send`` on stream-open (so the listener is registered first), the node runs a 4-parallel-bash ``seq 1 500`` storm plus a task_agent whose sub-tools are chatty bashes; @@ -129,6 +195,21 @@ # its shell source verbatim, which contains the keyword ``done``. HEALED_SENTINEL = "HEALED-e5b1" +# Second-turn sentinel for scenario E1 (stale-ref-reload): injected as the +# scripted turn-2 final text AND threaded to the page via ``?second=``. +# Same collision-proof discipline as HEALED_SENTINEL — it must not appear in +# turn 1's assistant bubble or any command/output, so "the sentinel landed in +# a fresh bubble, not the stale one" is an honest DOM check. +SECOND_SENTINEL = "SECOND-a7f3" + +# Fourth-turn sentinel for scenario E4 (stale-backstop): the scripted +# final_text of the plain send that drives the idle-edge backstop. Same +# collision-proof discipline — the E4 transcript is pure final_text turns +# ("one"/"two"/"three" + the seed messages), so this hex-suffixed token +# proves the sent turn reached the healed /history render, distinct from +# everything else on screen. +BACKSTOP_SENTINEL = "BACKSTOP-b2e4" + # --------------------------------------------------------------------------- # The recovery page — served same-origin by the node at /recovery. # --------------------------------------------------------------------------- @@ -168,6 +249,9 @@ // Healed-gap sentinel, threaded from the runner (HEALED_SENTINEL) // so the injected turn text and this check cannot drift apart. const healedSentinel = q.get("healed") || ""; + // Turn-2 sentinel for stale-ref-reload (SECOND_SENTINEL), same + // single-source discipline via the ?second= param. + const secondSentinel = q.get("second") || ""; // REAL pane against THIS origin (base=""): real authFetch (cookie) and // real EventSource. The default host provides all SSE seams. @@ -227,12 +311,73 @@ origOpen(p); window.__streamOpen = (window.__streamOpen || 0) + 1; if (scenario === "storm") sendOnce("run the storm"); - else if (scenario === "restart" && window.__streamOpen === 1) sendOnce("run a turn"); + else if ( + (scenario === "restart" || + scenario === "fail-refetch" || + scenario === "stale-ref-reload") && + window.__streamOpen === 1 + ) + sendOnce("run a turn"); + // rewind-window drives its turns SERVER-side before navigation (a + // /send never emits a live user row — only /history replay does), so + // the page never auto-sends there. }; + // Shared transport instrumentation for the truncated-recovery + // scenarios: count replay_truncated envelopes (the original restart + // idiom) and, for stale-ref-reload, tear the transport down the + // instant the first assistant `content` paints. + function installStreamWrap() { + window.__truncatedSeen = 0; + window.__teardownDone = 0; + const origHandle = pane.handleEvent.bind(pane); + pane.handleEvent = function (ev) { + if (ev && ev.type === "replay_truncated") window.__truncatedSeen += 1; + const r = origHandle(ev); + // stale-ref-reload: the fix's regression trap needs a NON-null + // streaming ref surviving into an unarmed same-ws reload. The + // scripted provider emits `content` atomically and the segment's + // stream_end frame (which nulls those refs) follows with no + // pollable gap, so the teardown must RIDE the content event: once + // it has been applied (currentAssistantEl now set), close the + // EventSource via the REAL disconnectSSE() before stream_end can + // dispatch — leaving exactly the stale ref a mid-content transport + // drop would. + if ( + scenario === "stale-ref-reload" && + !window.__teardownDone && + ev && + ev.type === "content" + ) { + window.__teardownDone = 1; + pane.disconnectSSE(); + // Non-vacuity guard: record that the teardown genuinely left a + // live streaming ref (the stale-ref precondition). If the + // segment's stream_end had already nulled it, the reload reset + // is a no-op and a green verdict would be meaningless — so + // __verifyStaleRef fails loudly when this is false. + window.__staleRefWasSet = pane.currentAssistantBodyEl != null; + } + return r; + }; + } + // First paint the REAL way: /history then connect SSE. pane._loadHistoryThenConnect(wsId); + // Shared by the rewind scenarios (E2/E3): click the REAL rewind + // button on the idx-th user row. Depends only on `pane`. + window.__clickRewind = function (idx) { + const rows = pane.messagesEl.querySelectorAll(".msg.user"); + const row = rows[idx]; + if (!row) return false; + const icon = row.querySelector(".icon-rewind"); + const btn = icon ? icon.closest("button") : null; + if (!btn) return false; + btn.click(); + return true; + }; + if (scenario === "storm") { const deadline = Date.now() + 40000; const poll = () => { @@ -256,12 +401,7 @@ } else if (scenario === "restart") { // The runner drives hide -> (restart node) -> show via window.__hide/ // __show. We watch for the truncated-triggered rebuild + idle settle. - window.__truncatedSeen = 0; - const origHandle = pane.handleEvent.bind(pane); - pane.handleEvent = function (ev) { - if (ev && ev.type === "replay_truncated") window.__truncatedSeen += 1; - return origHandle(ev); - }; + installStreamWrap(); window.__verifyRestart = function () { // Browser-level restart RECOVERY, full contract: the runner hid // the tab MID-turn (cursor frozen below the commits that land @@ -299,6 +439,211 @@ "-busy" + (pane.busy ? 1 : 0) + "-disc" + (disc ? 1 : 0) + "-healed" + (healed ? 1 : 0) + "-trunc" + window.__truncatedSeen; }; + } else if (scenario === "fail-refetch") { + // Scenario D — a FAILED truncated-resync /history (the runner arms + // node.fail_history(1) just before the show edge). PHASE 1: the + // failed fetch is a DOM/ref no-op (#890 guard-before-wipe) — the + // pre-restart rows survive, no empty-state is appended, and the + // hidden-window sentinel is NOT healed yet. PHASE 2: the connect + // chokepoint's retry redraws replay_truncated and the SECOND + // /history succeeds, healing the gap. + installStreamWrap(); + window.__failFetchStale = { rows: 0, ok: false }; + window.__verifyFailFetchStale = function () { + const rows = pane.messagesEl.querySelectorAll( + ".conv-row[data-call-id]", + ).length; + const emptyState = + pane.messagesEl.querySelector(".empty-state") !== null; + const healedAbsent = !( + pane.messagesEl.textContent || "" + ).includes(healedSentinel); + const ok = + rows >= 1 && + !emptyState && + healedAbsent && + window.__truncatedSeen >= 1; + window.__failFetchStale = { + rows: rows, + emptyState: emptyState, + healedAbsent: healedAbsent, + ok: ok, + }; + return window.__failFetchStale; + }; + window.__verifyFailFetch = function () { + const rows = pane.messagesEl.querySelectorAll( + ".conv-row[data-call-id]", + ).length; + const healed = (pane.messagesEl.textContent || "").includes( + healedSentinel, + ); + const stale = window.__failFetchStale || { ok: false }; + const ok = stale.ok && healed && rows >= 1 && !pane.busy; + document.title = ok + ? "RECOVERY-READY-FAILFETCH-stale1-healed1" + : "RECOVERY-FAILED-FAILFETCH-stale" + + (stale.ok ? 1 : 0) + + "-healed" + + (healed ? 1 : 0) + + "-rows" + + rows + + "-busy" + + (pane.busy ? 1 : 0) + + "-trunc" + + window.__truncatedSeen; + }; + } else if (scenario === "stale-ref-reload") { + // Scenario E1 — regression trap for the #890 streaming-ref reset. + // installStreamWrap's teardown hook left a stale currentAssistantEl; + // the runner then re-auth reloads (unarmed, same-ws) and drives turn + // 2. Phase 1 (post reload, pre turn-2) captures the FIRST assistant + // bubble element + its text; __verifyStaleRef proves turn 2's + // sentinel landed in a DIFFERENT (fresh) bubble and the captured + // bubble is byte-for-byte unchanged. Pre-fix the unarmed reload kept + // the stale ref and turn 2 concatenated into the old bubble. + installStreamWrap(); + window.__stalePhase1 = { rows: 0, text: "" }; + window.__captureStaleRefPhase1 = function () { + const first = pane.messagesEl.querySelector(".msg.assistant"); + window.__staleFirstBubble = first || null; + const text = first ? first.textContent || "" : ""; + const rows = + pane.messagesEl.querySelectorAll(".msg.assistant").length; + window.__stalePhase1 = { rows: rows, text: text }; + return window.__stalePhase1; + }; + window.__verifyStaleRef = function () { + const bubbles = pane.messagesEl.querySelectorAll(".msg.assistant"); + let sentinelEl = null; + bubbles.forEach(function (b) { + if (!sentinelEl && (b.textContent || "").includes(secondSentinel)) + sentinelEl = b; + }); + const first = window.__staleFirstBubble; + const firstTextNow = first ? first.textContent || "" : ""; + const present = !!sentinelEl; + const fresh = present && sentinelEl !== first; + const unchanged = firstTextNow === (window.__stalePhase1.text || ""); + const staleSet = window.__staleRefWasSet === true; + const ok = staleSet && present && fresh && unchanged; + document.title = ok + ? "RECOVERY-READY-STALEREF-fresh1" + : "RECOVERY-FAILED-STALEREF-staleset" + + (staleSet ? 1 : 0) + + "-present" + + (present ? 1 : 0) + + "-fresh" + + (fresh ? 1 : 0) + + "-unchanged" + + (unchanged ? 1 : 0); + }; + } else if (scenario === "rewind-window") { + // Scenario E2 — the row affordance gate (busy || _historyStale). The + // runner clicks a REAL rewind button (POSTs), then a SECOND one while + // the runner-delayed clear_ui refetch holds the quiesce armed; the + // gated click must return before POSTing. ``posts`` is the + // authoritative server-side rewind count the runner threads in. + // (__clickRewind is hoisted above the scenario dispatch.) + window.__verifyRewindWindow = function (posts) { + const userRows = + pane.messagesEl.querySelectorAll(".msg.user").length; + // One rewind took effect (the 2nd-of-3 user row = rewind 2 turns => + // one user row left); the gated 1st-row click never reached the + // server (posts stays 1). A broken gate => posts 2, rows 0. + const ok = posts === 1 && userRows === 1; + document.title = ok + ? "RECOVERY-READY-REWINDWIN-posts" + posts + : "RECOVERY-FAILED-REWINDWIN-posts" + posts + "-rows" + userRows; + }; + } else if (scenario === "rewind-failed-window") { + // Scenario E3 — the FAILED clear_ui refetch aftermath (#890). The + // row affordance gate is the _historyStale LATCH, not the transient + // _replayQueue quiesce: on a failed clear_ui refetch the quiesce + // releases (_replayQueue -> null) but the latch SURVIVES (only a + // successful replayHistory render clears it), so rewind/edit stay + // gated over the stale transcript. Pre-latch code reopened the gate + // the moment the failed fetch released the quiesce, letting a second + // rewind over-rewind — this scenario is that regression's trap. Same + // real-button click helper hoisted above the scenario dispatch; the + // runner threads the authoritative server-side counts into the + // verdict. + window.__verifyRewindFail = function (posts, closedPosts, healed) { + const userRows = + pane.messagesEl.querySelectorAll(".msg.user").length; + // Three legs, all runner-observed and threaded in: + // - closedPosts === 1: the FIRST-row rewind, clicked while the + // latch was set (the failed refetch already released the + // quiesce), was gated before POSTing — the leg that regresses to + // 2 on the pre-latch quiesce-only gate; + // - healed: the bounded 2s retry re-fetched and rebuilt the + // rewound transcript to ONE user row; + // - posts === 2: the healing render cleared the latch, so a fresh + // rewind on the remaining row reopened the gate and landed. + const ok = closedPosts === 1 && healed && posts === 2; + document.title = ok + ? "RECOVERY-READY-REWINDFAIL-posts2-heal1" + : "RECOVERY-FAILED-REWINDFAIL-closed" + + closedPosts + + "-heal" + + (healed ? 1 : 0) + + "-posts" + + posts + + "-rows" + + userRows; + }; + } else if (scenario === "stale-backstop") { + // Scenario E4 — the _historyStale latch's TRANSPORT-FREE idle-edge + // backstop (#890, the round-5 critical). A rewind's clear_ui refetch + // AND its one bounded 2s retry both 500, so the latch cannot + // self-heal and rewind/edit stay gated over the stale-but-real + // transcript. A plain send (deliberately NOT latch-gated) runs a + // fresh turn whose ORGANIC turn-settle idle edge fires the backstop — + // a quiesced, same-token REST _refetchHistory, NOT + // _loadHistoryThenConnect (the old reload backstop drew the server's + // synthetic state_change:idle on its fresh reconnect and re-triggered + // itself: a zero-backoff reconnect/refetch storm). The runner + // observes the heal + threads the authoritative fault-layer counters + // in; the r5 headline is sseDelta === 0 — the heal opened ZERO new + // SSE connections. (__clickRewind is hoisted above the dispatch.) + window.__verifyStaleBackstop = function ( + healed, + sseDelta, + histDelta, + gatedPosts, + posts, + ) { + const userRows = + pane.messagesEl.querySelectorAll(".msg.user").length; + // heal1 = the backstop's quiesced REST refetch rebuilt the rewound + // (ONE user turn) + sent (a second) transcript => TWO user rows, + // latch cleared. sse0 = it touched the transport ZERO times + // (sseDelta 0 — the storm regression opens one EventSource per + // reconnect). histDelta 1 = the backstop's single fetch (the + // plain fourth turn emits no clear_ui). gatedPosts 1 = the row-0 + // rewind stayed latch-gated while stale. posts 2 = the healed + // render reopened the gate and a fresh rewind landed. + const ok = + healed && + sseDelta === 0 && + histDelta === 1 && + gatedPosts === 1 && + posts === 2; + document.title = ok + ? "RECOVERY-READY-STALEBACKSTOP-heal1-sse0" + : "RECOVERY-FAILED-STALEBACKSTOP-heal" + + (healed ? 1 : 0) + + "-sse" + + sseDelta + + "-hist" + + histDelta + + "-gated" + + gatedPosts + + "-posts" + + posts + + "-rows" + + userRows; + }; } @@ -890,6 +1235,597 @@ def run_coord_restart(chrome: str) -> str: node.stop() +def _poll_until(pred: Any, timeout: float, interval: float = 0.1) -> bool: + """Poll ``pred()`` until truthy or the deadline elapses; return whether it + became truthy. The livepass convention — prefer an observable edge to a + bare sleep wherever one exists.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(interval) + return False + + +def _send_in_page(cdp: CDP, message: str) -> None: + """POST /send from inside the page via the pane's own authFetch (cookie + auth, node-proxy base) — the one shared shape for scenarios that drive a + turn mid-flight (E1/E4). A raw POST emits no live user row, so the sent + turn appears only via the next /history render.""" + cdp.evaluate( + "window.authFetch('/v1/api/workstreams/' + " + "encodeURIComponent(window.__pane.wsId) + '/send', {method:'POST'," + "headers:{'Content-Type':'application/json'}," + "body: JSON.stringify({message:" + json.dumps(message) + "})})" + ".then(function(r){return 'sent-'+r.status;})" + ".catch(function(e){return 'err-'+e;})" + ) + + +def run_fail_refetch(chrome: str) -> str: + """Scenario D — a FAILED truncated-resync /history must PRESERVE the pane + (#890 guard-before-wipe), and the connect-chokepoint retry must then heal + the gap. Clones run_restart's hide -> restart -> show flow, but arms one + forced /history failure before the show edge so the first jittered resync + 500s. PHASE 1 asserts the failed fetch left the pre-restart rows on + screen with no empty-state and the sentinel un-healed (plus the backend + proof node.history_fail_remaining == 0); PHASE 2 asserts the retry's + second /history healed it.""" + from tests._sse_recovery_server import final_text_script, parallel_bash_script + + port = _free_port() + node = _boot_node(port=port) + paced = parallel_bash_script({"r0": "for i in $(seq 1 40); do echo r-$i; sleep 0.05; done"}) + ws_id = node.create_workstream( + paced, final_text_script(HEALED_SENTINEL), name="browser-fail-refetch" + ) + profile = Path(_scratch()) / "chrome-fail-refetch" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = ( + f"{node.base_url}/recovery?ws_id={ws_id}&scenario=fail-refetch&healed={HEALED_SENTINEL}" + ) + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + # Hide the moment the first streamed line paints (mid-turn cursor + # frozen below the hidden-window commits) — the same edge run_restart + # uses to force a truncated show-edge reconnect. + if not _poll_until( + lambda: cdp.evaluate("document.querySelector('.tool-output-stream') !== null"), + 15, + 0.2, + ): + raise AssertionError("fail-refetch: first streamed line never painted") + cdp.evaluate("window.__hide && window.__hide()") + node.wait_turn(ws_id, timeout=30) + # Restart on the SAME port, then ARM one /history failure BEFORE the + # show edge: the show-edge reconnect draws replay_truncated and the + # truncated resync's FIRST /history 500s. + node.stop() + node = _boot_node(port=port) + node.open_workstream(ws_id) + node.fail_history(1) + cdp.evaluate("window.__show && window.__show()") + # PHASE 1 edge: the failed fetch consumed the fail budget (the + # jittered resync fired, <=10s). Poll the backend rather than sleep, + # and capture the stale-but-preserved DOM the instant it lands — + # before the retry can heal it. + if not _poll_until(lambda: node.history_fail_remaining == 0, 20): + raise AssertionError("fail-refetch: forced /history failure never fired") + stale = cdp.evaluate("JSON.stringify(window.__verifyFailFetchStale())") + # Backend proof the failure actually happened (never scripted absence). + assert node.history_fail_remaining == 0, "fail-refetch: fail budget not consumed" + # PHASE 2 edge: the connect-chokepoint retry redraws replay_truncated + # and the SECOND /history succeeds — poll the DOM for the heal. + _poll_until( + lambda: cdp.evaluate( + "(window.__pane.messagesEl.textContent||'').includes(" + + json.dumps(HEALED_SENTINEL) + + ")" + ), + 20, + 0.2, + ) + print(f" fail-refetch phase-1 (stale): {stale}") + # The heal must have come from the connect-chokepoint RETRY, + # not a scripted accident: the restart reset the counter to 0 + # and the first resync 500'd (1), so the healing fetch makes it + # >= 2. ">= 2" not "== 2": a legitimate extra jitter/churn + # resync cycle may add a third. Load-bearing counter — do not + # let history_requests drop back to write-only. + assert node.history_requests >= 2, ( + f"fail-refetch: heal did not re-fetch (history_requests={node.history_requests})" + ) + cdp.evaluate("window.__verifyFailFetch && window.__verifyFailFetch()") + return _poll_title(cdp, 20) + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + +def run_stale_ref(chrome: str) -> str: + """Scenario E1 — regression test for the #890 streaming-ref reset. A + mid-content transport teardown leaves a live assistant bubble ref; a + same-ws UNARMED re-auth reload (the factory's onLogin fan-out) must reset + it, so the NEXT turn's text builds a FRESH bubble instead of concatenating + into the stale one. No node restart: the page's teardown hook drops the + transport on turn 1's content, the runner re-auth reloads through a forced + /history failure, then drives turn 2 and proves the sentinel landed in a + different bubble with the first bubble unchanged.""" + node = _boot_node() + # Turn 1 carries assistant CONTENT (the bubble that must go stale) AND a + # paced bash, so the turn is genuinely mid-flight when the transport drops + # (unlike run_restart's pure-bash turn, which sets no content ref — the + # concatenation bug is specifically about a content bubble). The content + # event trips the page's teardown hook; the bash + closing text then + # complete server-side during the outage. + turn1 = { + "content": "First-turn assistant answer.", + "tool_calls": [ + { + "id": "r0", + "name": "bash", + "arguments": json.dumps( + {"command": "for i in $(seq 1 40); do echo r-$i; sleep 0.05; done"} + ), + } + ], + "finish_reason": "tool_calls", + } + from tests._sse_recovery_server import final_text_script + + ws_id = node.create_workstream( + turn1, + final_text_script("turn one closed"), + final_text_script(SECOND_SENTINEL), + name="browser-stale-ref", + ) + profile = Path(_scratch()) / "chrome-stale-ref" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = ( + f"{node.base_url}/recovery?ws_id={ws_id}" + f"&scenario=stale-ref-reload&second={SECOND_SENTINEL}" + ) + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + # The page auto-sends turn 1; its content event trips the teardown + # hook (REAL pane.disconnectSSE() mid-segment). Poll the exposed flag + # rather than guess a sleep. + if not _poll_until(lambda: cdp.evaluate("window.__teardownDone === 1"), 20): + raise AssertionError("stale-ref: mid-content teardown never fired") + # Turn 1 completes server-side during the outage. + node.wait_turn(ws_id, timeout=30) + # Arm one /history failure, then re-auth reload EXACTLY as the + # factory's onLogin does — same-ws, unarmed (no truncation cursor was + # ever recorded, so nothing schedules a jittered resync). + node.fail_history(1) + cdp.evaluate("window.__pane._loadHistoryThenConnect(window.__pane.wsId)") + # Failed fetch + cursorless reconnect; the only backend edge is the + # fail budget draining to 0 (no resync jitter here). + if not _poll_until(lambda: node.history_fail_remaining == 0, 10, 0.05): + raise AssertionError("stale-ref: forced /history failure never fired") + # The reconnect must be open before turn 2 (its listener catches the + # content), and the reload must have reset the stale ref. + _poll_until(lambda: cdp.evaluate("(window.__streamOpen || 0) >= 2"), 10) + phase1 = cdp.evaluate("JSON.stringify(window.__captureStaleRefPhase1())") + # Drive turn 2 in-page via authFetch (the SECOND_SENTINEL final text). + _send_in_page(cdp, "second turn") + _poll_until( + lambda: cdp.evaluate( + "(window.__pane.messagesEl.textContent||'').includes(" + + json.dumps(SECOND_SENTINEL) + + ")" + ), + 20, + 0.2, + ) + print(f" stale-ref phase-1 (bubble): {phase1}") + cdp.evaluate("window.__verifyStaleRef && window.__verifyStaleRef()") + return _poll_title(cdp, 20) + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + +def _seed_three_completed_turns(name: str, extra_scripts: tuple[Any, ...] = ()) -> tuple[Any, str]: + """Boot a node and drive THREE completed ``final_text`` turns, returning + ``(node, ws_id)`` — the byte-identical seeding the rewind scenarios + (E2/E3/E4) share, extracted so their rewind arithmetic provably reads off + the SAME transcript. Each of the three ``node.send`` calls consumes one + scripted turn; a /send never emits a live user row (only /history replay + does), so the initial page-load /history render is what paints all three + user rows. + + ``extra_scripts`` are appended to the scripted client AFTER the three + seeding scripts and left UNSENT: E4 queues a fourth ``final_text`` turn + there for the later backstop-driving send (its positional script stays in + sync because the three seeding sends consume exactly the three seeding + scripts).""" + from tests._sse_recovery_server import final_text_script + + node = _boot_node() + ws_id = node.create_workstream( + final_text_script("one"), + final_text_script("two"), + final_text_script("three"), + *extra_scripts, + name=name, + ) + for msg in ("first", "second", "third"): + node.send(ws_id, msg) + node.wait_turn(ws_id) + return node, ws_id + + +def run_rewind_window(chrome: str) -> str: + """Scenario E2 — the row affordance gate (``busy || _historyStale``, #890). + Three completed turns => three user rows; a REAL rewind click on the + second row POSTs and its clear_ui refetch is held open by + node.delay_history, keeping the quiesce armed; a second REAL rewind click + (first row) mid-rebuild must return before POSTing. The backend proof is + node.rewind_requests == 1 (only the first click reached the server).""" + node, ws_id = _seed_three_completed_turns("browser-rewind-window") + profile = Path(_scratch()) / "chrome-rewind-window" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = f"{node.base_url}/recovery?ws_id={ws_id}&scenario=rewind-window" + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + # Wait for the initial /history to paint all three user rows. + if not _poll_until( + lambda: ( + cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") == 3 + ), + 20, + 0.2, + ): + raise AssertionError("rewind-window: three user rows never rendered") + # Hold every /history 3s so the clear_ui refetch keeps the quiesce + # armed long enough to click the second rewind mid-rebuild. + node.delay_history(3000) + # Click #1 — the REAL rewind button on the SECOND user row: POSTs, + # server emits clear_ui, the refetch is now held. + if not cdp.evaluate("window.__clickRewind(1)"): + raise AssertionError("rewind-window: second-row rewind button missing") + # Wait for the clear_ui refetch to arm the quiesce (observable edge). + if not _poll_until(lambda: cdp.evaluate("window.__pane._replayQueue != null"), 5, 0.05): + raise AssertionError("rewind-window: clear_ui never armed the quiesce") + # Click #2 — the rewind button on the FIRST user row WHILE the quiesce + # is armed: the #890 gate must return before POSTing. + if not cdp.evaluate("window.__clickRewind(0)"): + raise AssertionError("rewind-window: first-row rewind button missing") + # The gate leaves no positive edge (a POST that never happens), so + # confirm the NON-occurrence over a bounded window: a failed gate's + # /rewind is NOT delayed and would land within ~200ms. + _poll_until(lambda: node.rewind_requests != 1, 1.5, 0.05) + posts = node.rewind_requests + # Release the hold and let the single in-flight rewind settle to one + # user row (2nd-of-3 row rewound 2 turns => one user turn remains). + node.delay_history(0) + _poll_until( + lambda: ( + (not cdp.evaluate("window.__pane._replayQueue != null")) + and cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") + == 1 + ), + 8, + ) + print(f" rewind-window rewind_requests={posts}") + cdp.evaluate(f"window.__verifyRewindWindow({posts})") + return _poll_title(cdp, 15) + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + +def run_rewind_failed_window(chrome: str) -> str: + """Scenario E3 — the FAILED clear_ui refetch aftermath (#890). Clones + run_rewind_window's three-turn seeding, but the rewind's clear_ui refetch + is forced to 500 (node.fail_history) instead of held open. The + _historyStale LATCH (set at clear_ui, cleared ONLY by a successful + replayHistory render) must keep the row affordances gated over the + stale-but-real transcript AFTER the failed fetch releases the transient + _replayQueue quiesce — the exact aftermath where pre-latch (quiesce-only) + code reopened the gate and let a second rewind over-rewind. The bounded 2s + retry then heals the transcript and reopens the gate for a fresh, + legitimate rewind. Backend proofs: rewind_requests 1 -> (gated) 1 -> 2, + history_fail_remaining == 0, history_requests >= 2.""" + node, ws_id = _seed_three_completed_turns("browser-rewind-failed-window") + profile = Path(_scratch()) / "chrome-rewind-failed-window" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = f"{node.base_url}/recovery?ws_id={ws_id}&scenario=rewind-failed-window" + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + # Wait for the initial /history to paint all three user rows. This + # load MUST succeed, so arm the forced failure only AFTERWARDS. + if not _poll_until( + lambda: ( + cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") == 3 + ), + 20, + 0.2, + ): + raise AssertionError("rewind-failed-window: three user rows never rendered") + # Arm ONE forced /history 500: the NEXT /history — the rewind's + # clear_ui refetch — fails. The initial load already succeeded, so the + # failure lands on the refetch, not first paint. + node.fail_history(1) + # Click #1 — the REAL rewind on the SECOND user row: POSTs (the + # authoritative rewind commits server-side), the server emits clear_ui, + # and its refetch 500s. The stale transcript survives (#890 + # guard-before-wipe) and the _historyStale latch stays SET. + if not cdp.evaluate("window.__clickRewind(1)"): + raise AssertionError("rewind-failed-window: second-row rewind button missing") + if not _poll_until(lambda: node.rewind_requests == 1, 5, 0.05): + raise AssertionError("rewind-failed-window: first rewind never POSTed") + if not _poll_until(lambda: node.history_fail_remaining == 0, 15): + raise AssertionError("rewind-failed-window: forced /history failure never fired") + # Backend proof the failure actually happened (never scripted absence). + assert node.history_fail_remaining == 0, "rewind-failed-window: fail budget not consumed" + # Hold the bounded retry's /history open. The retry is a fixed 2s + # timer; delaying its fetch (the next /history to ARRIVE) defers the + # latch's only clear site — replayHistory on a SUCCESSFUL render — to + # ~failure+5s, a wide, CDP-speed-independent window for the gated click + # below. Armed AFTER the failure (so the FIRST refetch still fails + # FAST, keeping detection prompt) and ~1.8s BEFORE the retry fires. + # This is the determinism shape the E3 spec invites; a bare 500ms bound + # would also hold (the click lands ~300ms after the failure, well + # before the 2s retry), but the delay removes the race entirely — the + # latch provably cannot clear until we release, so the closed-phase + # checks below never overlap the heal. + node.delay_history(3000) + # CLOSED-PHASE — the failed-fetch aftermath. Wait for the failed + # refetch to fully settle: it releases the transient quiesce + # (_replayQueue -> null) while the _historyStale latch SURVIVES. This + # is the crux the scenario exists for — a pre-latch gate keyed on the + # quiesce would now be OPEN; only the latch holds it. + if not _poll_until( + lambda: cdp.evaluate( + "window.__pane._replayQueue == null && window.__pane._historyStale === true" + ), + 5, + 0.05, + ): + raise AssertionError( + "rewind-failed-window: failed fetch did not settle to latch-held/quiesce-released" + ) + # The stale transcript is intact — the failed fetch wiped nothing. + stale_rows = cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") + if stale_rows != 3: + raise AssertionError( + f"rewind-failed-window: failed fetch did not preserve the transcript " + f"(user rows={stale_rows}, expected 3)" + ) + # Click #2 — rewind on the FIRST user row while the latch is set: the + # #890 gate (busy || _historyStale) must return before POSTing. + if not cdp.evaluate("window.__clickRewind(0)"): + raise AssertionError("rewind-failed-window: first-row rewind button missing") + # The gate leaves no positive edge (a POST that never happens), so + # confirm the NON-occurrence over a bounded window: a failed gate's + # /rewind is NOT delayed and would land within ~200ms. This is the + # assertion that regresses to 2 on pre-latch code (with posts2). + _poll_until(lambda: node.rewind_requests != 1, 1.5, 0.05) + closed_posts = node.rewind_requests + # HEAL-PHASE — the bounded retry fires at ~2s (pane idle/turn-free), its + # held /history completes (~failure+5s) and rebuilds the rewound + # transcript: index 1 of 3 user rows rewinds 2 turns => ONE user row + # (same arithmetic as run_rewind_window). Poll to a deadline rather + # than sleeping the 2s timer + 3s hold. + healed = _poll_until( + lambda: ( + cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") == 1 + ), + 10, + 0.2, + ) + # The retry re-fetched: init load (1) + failed refetch (2) + retry (3). + # ">= 2" is the spec floor (matching the fail-refetch sibling); the + # 3-user-rows -> 1-user-row DOM transition above is the load-bearing + # proof the retry RENDERED — a stale transcript can only shrink via a + # successful /history render. Load-bearing counter — do not let + # history_requests drop back to write-only. + assert node.history_requests >= 2, ( + f"rewind-failed-window: retry did not re-fetch (history_requests={node.history_requests})" + ) + # Release the hold now that the heal landed — the reopen's own clear_ui + # refetch must not be delayed. + node.delay_history(0) + # REOPEN-PHASE — the healing render cleared the latch, reopening the + # gate. A rewind on the remaining user row is now legitimate and must + # land with a FRESH count (rewind_requests -> 2). On a heal failure the + # verdict is already lost; skip the click and stamp the observed counts. + if healed: + if not cdp.evaluate("window.__clickRewind(0)"): + raise AssertionError("rewind-failed-window: healed-row rewind button missing") + _poll_until(lambda: node.rewind_requests == 2, 8, 0.05) + posts = node.rewind_requests + print(f" rewind-failed-window closed_posts={closed_posts} healed={healed} posts={posts}") + cdp.evaluate( + f"window.__verifyRewindFail({posts}, {closed_posts}, {'true' if healed else 'false'})" + ) + return _poll_title(cdp, 15) + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + +def run_stale_backstop(chrome: str) -> str: + """Scenario E4 — the ``_historyStale`` latch's TRANSPORT-FREE idle-edge + backstop (#890, the round-5 critical). The DOUBLE-failure sibling of E3: + a rewind's clear_ui refetch AND its one bounded 2s retry are BOTH forced + to 500 (``node.fail_history(2)``), so the latch cannot self-heal and + rewind/edit stay gated over the stale-but-real transcript. A plain send + — sends are deliberately NOT latch-gated (``sendMessage`` gates only on + ``busy``; the raw ``authFetch`` here bypasses even that) — runs the fourth + scripted ``final_text`` turn whose ORGANIC turn-settle idle edge fires the + backstop: a quiesced, same-token REST ``_refetchHistory``, deliberately + NOT ``_loadHistoryThenConnect`` (the old reload backstop drew the server's + synthetic ``state_change:idle`` on its fresh reconnect and re-triggered + itself — a zero-backoff reconnect/refetch storm). With the fault budget + exhausted the refetch succeeds and heals the rewound + sent transcript. + + THE r5 PROOF (both counted at the fault layer): ``events_requests`` is + UNCHANGED across the whole heal (``sse0`` — zero new EventSource + connections; the storm regression opens one per reconnect) and + ``history_requests`` grew by exactly ONE (the backstop's single fetch — + the plain fourth turn emits no clear_ui). Backend proofs: + ``rewind_requests`` 1 -> (gated) 1 -> 2, ``history_fail_remaining == 0``. + Every poll is deadline-bounded so a regressed looping backstop stamps a + clean FAILED, never a hang.""" + from tests._sse_recovery_server import final_text_script + + # Seed three turns and queue a FOURTH final_text (the sentinel-bearing + # turn the backstop-driving send below runs) — the shared helper keeps the + # seeding byte-identical to E2/E3 so the rewind arithmetic matches. + node, ws_id = _seed_three_completed_turns( + "browser-stale-backstop", + extra_scripts=(final_text_script(BACKSTOP_SENTINEL),), + ) + profile = Path(_scratch()) / "chrome-stale-backstop" + proc, cdp_port = _launch_chrome(chrome, profile) + cdp: CDP | None = None + try: + cdp = CDP(_page_ws_url(cdp_port)) + url = f"{node.base_url}/recovery?ws_id={ws_id}&scenario=stale-backstop" + _set_cookie_and_navigate(cdp, node.base_url, node.token, url) + # Wait for the initial /history to paint all three user rows. This + # load MUST succeed, so arm the forced failures only AFTERWARDS. + if not _poll_until( + lambda: ( + cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") == 3 + ), + 20, + 0.2, + ): + raise AssertionError("stale-backstop: three user rows never rendered") + # Arm TWO forced /history 500s: the rewind's clear_ui refetch AND its + # one bounded 2s retry both fail, so the latch cannot self-heal and + # ONLY the organic idle-edge backstop can clear it. + node.fail_history(2) + # Click #1 — the REAL rewind on the SECOND user row: POSTs (the + # authoritative rewind commits server-side to ONE user turn — index 1 + # of 3 rewinds 2), the server emits clear_ui, and its refetch 500s + # (fault 2 -> 1). + if not cdp.evaluate("window.__clickRewind(1)"): + raise AssertionError("stale-backstop: second-row rewind button missing") + if not _poll_until(lambda: node.rewind_requests == 1, 5, 0.05): + raise AssertionError("stale-backstop: first rewind never POSTed") + # Both the clear_ui refetch AND the 2s retry must fire and fail (fault + # 2 -> 1 -> 0): history_fail_remaining == 0 proves both consumed. The + # retry fires ~2s after the first failure (pane idle, turn-free), so a + # 20s deadline covers it comfortably. + if not _poll_until(lambda: node.history_fail_remaining == 0, 20): + raise AssertionError( + "stale-backstop: the two forced /history failures never both fired" + ) + # Backend proof the failures actually happened (never scripted absence). + assert node.history_fail_remaining == 0, "stale-backstop: fail budget not consumed" + # Aftermath: the failed refetches released the transient quiesce + # (_replayQueue -> null) while the _historyStale latch SURVIVES — the + # backstop's precondition (and the idle-edge guard is !_replayQueue). + if not _poll_until( + lambda: cdp.evaluate( + "window.__pane._replayQueue == null && window.__pane._historyStale === true" + ), + 5, + 0.05, + ): + raise AssertionError( + "stale-backstop: aftermath did not settle to latch-held/quiesce-released" + ) + # The stale transcript is intact — the failed fetches wiped nothing. + stale_rows = cdp.evaluate("window.__pane.messagesEl.querySelectorAll('.msg.user').length") + if stale_rows != 3: + raise AssertionError( + f"stale-backstop: failed fetches did not preserve the transcript " + f"(user rows={stale_rows}, expected 3)" + ) + # Click #2 — rewind on the FIRST user row while the latch is set: the + # #890 gate (busy || _historyStale) must return before POSTing. The + # non-occurrence is confirmed over a bounded window (a broken gate's + # /rewind is not delayed and would land within ~200ms). + if not cdp.evaluate("window.__clickRewind(0)"): + raise AssertionError("stale-backstop: first-row rewind button missing") + _poll_until(lambda: node.rewind_requests != 1, 1.5, 0.05) + gated_posts = node.rewind_requests # must still be 1 (latch gated it) + # Baselines captured the instant BEFORE the send: the heal must add + # exactly ZERO SSE opens and exactly ONE /history fetch relative here. + events_baseline = node.events_requests + history_baseline = node.history_requests + # Drive a plain send via in-page authFetch — sends are NOT latch-gated + # (see docstring), so this reaches the server, runs the fourth scripted + # turn (BACKSTOP_SENTINEL), and its ORGANIC turn-settle idle edge fires + # the backstop. + _send_in_page(cdp, "fourth turn") + # HEAL: the fourth turn settles -> idle edge -> quiesced REST refetch + # (fault exhausted) succeeds -> replayHistory rebuilds the rewound (ONE + # user turn) + sent (a second) transcript to TWO user rows, SENTINEL + # present, latch cleared. The 3-user-rows -> 2-user-rows transition is + # the load-bearing proof the backstop RENDERED — a stale transcript can + # only change via a successful /history render. Deadline-bounded so a + # regressed looping backstop times out to a clean FAILED, never a hang. + healed = _poll_until( + lambda: cdp.evaluate( + "window.__pane.messagesEl.querySelectorAll('.msg.user').length === 2 " + "&& window.__pane._historyStale === false " + "&& (window.__pane.messagesEl.textContent||'').includes(" + + json.dumps(BACKSTOP_SENTINEL) + + ")" + ), + 20, + 0.2, + ) + # THE r5 DELTAS, captured BEFORE the reopen click's own clear_ui + # refetch so the arithmetic is exactly the backstop's: + # - events_delta MUST be 0: the backstop is a REST _refetchHistory, so + # it opens ZERO EventSource connections (a reload backstop's + # connectSSE would bump events_requests by one per reconnect — the + # round-5 storm). The EventSource opened at initial load is already + # folded into the baseline. + # - history_delta MUST be 1: the plain fourth turn emits no clear_ui, + # so the ONLY /history in the send+heal window is the backstop fetch. + events_delta = node.events_requests - events_baseline + history_delta = node.history_requests - history_baseline + # REOPEN: the healing render cleared the latch, so a rewind on a + # remaining user row is legitimate and lands (rewind_requests -> 2). + # On a heal failure the verdict is already lost; skip the click and + # stamp the observed counts. + if healed: + if not cdp.evaluate("window.__clickRewind(0)"): + raise AssertionError("stale-backstop: healed-row rewind button missing") + _poll_until(lambda: node.rewind_requests == 2, 8, 0.05) + posts = node.rewind_requests + print( + f" stale-backstop gated_posts={gated_posts} healed={healed} " + f"events_delta={events_delta} history_delta={history_delta} posts={posts}" + ) + cdp.evaluate( + "window.__verifyStaleBackstop(" + f"{'true' if healed else 'false'}, " + f"{events_delta}, {history_delta}, {gated_posts}, {posts})" + ) + return _poll_title(cdp, 15) + finally: + if cdp is not None: + cdp.close() + _kill(proc) + node.stop() + + def _wait_state(node: Any, ws_id: str, state: str, timeout: float) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: @@ -926,9 +1862,20 @@ def main() -> None: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument( "--scenario", - choices=["storm", "restart", "coord-restart", "both", "all"], + choices=[ + "storm", + "restart", + "coord-restart", + "fail-refetch", + "stale-ref-reload", + "rewind-window", + "rewind-failed-window", + "stale-backstop", + "both", + "all", + ], default="all", - help="'both' = A+B (legacy alias); 'all' adds the coordinator scenario", + help="'both' = A+B (legacy alias); 'all' runs every scenario", ) ap.add_argument("--keep-open", type=int, metavar="PORT", help="serve the storm page, no CDP") args = ap.parse_args() @@ -955,6 +1902,26 @@ def main() -> None: verdict = run_coord_restart(chrome) print(f"scenario C (coord): {verdict}") failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("fail-refetch", "all"): + verdict = run_fail_refetch(chrome) + print(f"scenario D (failref): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("stale-ref-reload", "all"): + verdict = run_stale_ref(chrome) + print(f"scenario E1 (staleref): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("rewind-window", "all"): + verdict = run_rewind_window(chrome) + print(f"scenario E2 (rewindwin): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("rewind-failed-window", "all"): + verdict = run_rewind_failed_window(chrome) + print(f"scenario E3 (rewindfail): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 + if args.scenario in ("stale-backstop", "all"): + verdict = run_stale_backstop(chrome) + print(f"scenario E4 (stalebackstop): {verdict}") + failures += 0 if verdict.startswith("RECOVERY-READY") else 1 raise SystemExit(1 if failures else 0) diff --git a/tests/_sse_recovery_server.py b/tests/_sse_recovery_server.py index a4370dcbc..b9f0d0a55 100644 --- a/tests/_sse_recovery_server.py +++ b/tests/_sse_recovery_server.py @@ -203,7 +203,64 @@ def session_factory( self._port = int(self._sock.getsockname()[1]) self._sock.listen(128) - self._server = uvicorn.Server(uvicorn.Config(self._app, log_level="warning", lifespan="on")) + # -- fault injection (public knobs below) ---------------------------- + # In-process arming: the Tier-2 runner holds this RecoveryServer and + # arms a knob, THEN drives the browser request that consumes it. + # Single-writer by construction — the runner never arms a knob while + # the loop thread is mid-consume — and CPython makes each int read / + # write atomic, so these need no lock even though the uvicorn loop + # thread increments/decrements them while the runner thread reads. + self.history_requests = 0 + self.rewind_requests = 0 + # Per-ws SSE connection opens (``GET …/events`` — the EventSource the + # pane's connectSSE builds). A TRANSPORT-FREE heal (the #890 idle-edge + # staleness backstop, a quiesced REST refetch) must leave this FLAT; a + # reload-based backstop would bump it once per reconnect (the round-5 + # storm). Same lock-free single-writer int discipline as above. + self.events_requests = 0 + self._history_fail_remaining = 0 + self._history_delay_ms = 0 + # A thin pure-ASGI fault layer wrapping the REAL app (the production + # app itself is untouched): count + optionally delay/fail + # ``GET …/history``, count ``POST …/rewind``, count each per-ws SSE + # connection open (``GET …/events``), forward everything else (SSE + # bodies, /send, lifespan, static) verbatim. + production_app = self._app + + async def _fault_app(scope: dict[str, Any], receive: Any, send: Any) -> None: + if scope.get("type") == "http": + path = scope.get("path", "") + method = scope.get("method", "") + if path.endswith("/history") and method == "GET": + self.history_requests += 1 + if self._history_delay_ms > 0: + await asyncio.sleep(self._history_delay_ms / 1000.0) + if self._history_fail_remaining > 0: + self._history_fail_remaining -= 1 + await send( + { + "type": "http.response.start", + "status": 500, + "headers": [(b"content-type", b"application/json")], + } + ) + await send({"type": "http.response.body", "body": b'{"error": "injected"}'}) + return + elif path.endswith("/rewind") and method == "POST": + self.rewind_requests += 1 + elif path.endswith("/events") and method == "GET": + # Per-ws SSE connection open — count it (readable on + # RecoveryServer) and forward the long-lived stream + # verbatim below. Uniquely the per-ws stream: the global + # lane is ``…/events/global`` (ends ``/global``), and the + # route the pane's EventSource hits is + # ``…/workstreams/{ws_id}/events`` (session_routes). + self.events_requests += 1 + await production_app(scope, receive, send) + + self._server = uvicorn.Server( + uvicorn.Config(_fault_app, log_level="warning", lifespan="on") + ) self._thread = threading.Thread( target=self._serve, name=f"uvicorn-recovery-{self._port}", daemon=True ) @@ -342,6 +399,29 @@ def fetch_history(self, ws_id: str) -> dict[str, Any]: result: dict[str, Any] = r.json() return result + # -- fault-injection knobs ----------------------------------------------- + # Armed in-process by the Tier-2 runner (single writer at a time — see + # __init__). A plain int is deliberate: CPython makes the loop thread's + # increment/decrement and the runner thread's read each atomic, and the + # arm-then-consume ordering means they never race. + + def fail_history(self, count: int) -> None: + """Make the next ``count`` ``GET …/history`` responses a 500 — the + failed refetch the #890 guard-before-wipe must survive.""" + self._history_fail_remaining = count + + def delay_history(self, ms: int) -> None: + """Hold each ``GET …/history`` ``ms`` ms before forwarding (0 + clears). Opens the clear_ui-refetch quiesce window that the row + affordance gate (``busy || _historyStale``) must close.""" + self._history_delay_ms = ms + + @property + def history_fail_remaining(self) -> int: + """Unconsumed forced-failure budget — 0 proves the armed failure + actually fired (assert backend state, never scripted absence).""" + return self._history_fail_remaining + # -- teardown ------------------------------------------------------------ def stop(self) -> None: diff --git a/tests/test_interactive_pane_js.py b/tests/test_interactive_pane_js.py index 21ff3b6de..4acac280f 100644 --- a/tests/test_interactive_pane_js.py +++ b/tests/test_interactive_pane_js.py @@ -292,7 +292,7 @@ def test_rebuild_quiesces_live_events_and_releases_agent_tracking() -> None: ) assert "!this.currentAssistantEl && !this.currentReasoningEl" in body replay = body.index("replayHistory(messages) {") - seg = body[replay : replay + 1600] + seg = body[replay : replay + 2600] for line in ( "this._resetStreamingRefs();", "this._clearAgentTracking();", @@ -319,12 +319,166 @@ def test_rebuild_quiesces_live_events_and_releases_agent_tracking() -> None: # idle edge) instead of dropping it — skipping left the lost-event gap # unrepaired for the rest of the session. assert "this._pendingTruncatedResync = true;" in body - # The refetch FAILURE branch resets streaming refs too — it never reaches - # replayHistory, and stale refs there streamed the retried generation's - # first segment into a detached bubble. - fail = body.index("Failure path never reaches replayHistory") - assert "this._resetStreamingRefs();" in body[fail : fail + 700], ( - "the refetch failure branch must reset streaming refs" + # The refetch FAILURE branch is a DOM/ref no-op (#890): the full + # guard-before-wipe contract — failure branch, clear_ui, the + # resumability-gated reload reset, and the affordance gates — is + # pinned in test_interactive_refetch_failure_preserves_the_pane. + + +def test_interactive_refetch_failure_preserves_the_pane() -> None: + """A FAILED /history fetch must leave the transcript, streaming refs, + and repair-intent state untouched on EVERY re-render route (#890 — + the interactive mirror of coord's #882 G3 guard-before-wipe, pinned + there by test_coordinator_refetch_failure_preserves_the_pane). The + wipe + resets live in replayHistory, reached only on success: + + - the clear_ui case must NOT pre-wipe (pre-#890 a failed fetch + during a rewind blanked the highest-traffic pane on a live + stream); + - the _refetchHistory failure branch must be empty except for the + quiesce release (no showEmptyState — the stray hint below stale + content was the resync-route wart; no ref reset); + - the failed-first-paint placeholder survives ONLY via the factory + connect() pre-seed, which must stay ahead of the load call; + - _loadHistoryThenConnect must reset streaming refs UNLESS a + truncation resync is armed (the armed route's cursor replay + resumes the mid-jitter bubble; every non-resumable flavor — + ws switch, first paint, idle edge, unarmed re-auth reload — + resets, or a stale ref would concatenate the next turn into the + old bubble); + - the row-level mutating affordances (rewind / edit / + edit-and-resend) must gate on the _historyStale latch alongside + busy — the latch spans clear_ui arrival through the next + SUCCESSFUL render, covering the fetch window AND the failed-fetch + aftermath (a quiesce-based gate reopened on the failure exit and + let a second rewind over-rewind off the stale DOM); + - the latch heals TRANSPORT-FREE: one turn-free bounded retry from + the clear_ui .then, and a quiesced same-token refetch at organic + idle edges as the double-failure backstop. Neither may touch + the stream — a reload's fresh reconnect draws the server's + synthetic state_change:idle back into the backstop's own trigger + (the round-5 zero-backoff reconnect storm). + """ + body = _INTERACTIVE.read_text(encoding="utf-8") + + # clear_ui: no pre-wipe before the fetch, and the quiesce must be + # armed BEFORE the fetch (queued live events land in the rebuilt — + # or, on failure, the stale-but-real — pane, never the void). + cl = body.index('case "clear_ui":') + cl_seg = body[cl : body.index("break;", cl)] + assert "this._refetchHistory(this.wsId, token)" in cl_seg + assert "this.messagesEl.replaceChildren();" not in cl_seg, ( + "clear_ui must not pre-wipe the transcript (#890)" + ) + assert "this._resetStreamingRefs();" not in cl_seg, ( + "clear_ui must not pre-reset streaming refs (#890)" + ) + assert cl_seg.index("this._beginReplayQuiesce(token);") < cl_seg.index( + "this._refetchHistory(" + ), "clear_ui must arm the quiesce BEFORE the fetch" + + # Failure branch: quiesce release only. + fail = body.index("Failed fetch = DOM + ref + repair-intent no-op") + fail_seg = body[fail : fail + 1100] + assert "this._endReplayQuiesce(token);" in fail_seg + assert "this.showEmptyState();" not in fail_seg, ( + "a failed fetch must not append an empty-state hint (#890)" + ) + assert "this._resetStreamingRefs();" not in fail_seg + # The failure branch must RESOLVE, never throw/reject: the clear_ui + # .then dispatches the queued edit-and-resend after a failed fetch + # too (the rewind already committed server-side) — a throw here + # would route to .catch and strand the resend. + assert "throw " not in fail_seg + assert "Promise.reject" not in fail_seg + + # Success path still owns the wipe + resets. + replay = body.index("replayHistory(messages) {") + replay_seg = body[replay : replay + 2600] + assert "this.messagesEl.replaceChildren();" in replay_seg + assert "this._resetStreamingRefs();" in replay_seg + + # Factory pre-seed ahead of the load — the failed-first-paint + # placeholder's only remaining producer. + conn = body.index("Load-bearing pre-seed (#890)") + conn_seg = body[conn : conn + 600] + seed = conn_seg.index("pane.showEmptyState();") + load = conn_seg.index("pane._loadHistoryThenConnect(wsId);") + assert seed < load, "connect() must seed the empty-state BEFORE the fetch" + + # Resumability-gated ref reset in _loadHistoryThenConnect: refs + # survive a reload only when an armed truncation cursor lets the + # reconnect resume into them; every other flavor (ws switch, + # unarmed re-auth reload) resets. + lh = body.index("_loadHistoryThenConnect(wsId) {") + lh_seg = body[lh : body.index("async _refetchHistory(", lh)] + assert "if (this._truncatedFromCursor == null) this._resetStreamingRefs();" in lh_seg, ( + "reload must reset streaming refs unless a truncation resync is armed (#890)" + ) + + # The mutating row affordances gate on the staleness latch alongside + # busy — scoped per method so a gate migrating off one affordance + # cannot hide inside a file-wide occurrence count. + for sig in ( + "_rewindToMessage(msgEl) {", + "_startEdit(msgEl, originalText) {", + "_editAndResend(msgEl, newText) {", + ): + m = body.index(sig) + assert "if (this.busy || this._historyStale) return;" in body[m : m + 400], ( + f"missing busy || _historyStale gate in {sig!r} (#890)" + ) + + # Latch lifecycle: set at clear_ui arrival BEFORE the quiesce is + # armed (so no event can interleave between restructure-signal and + # gate-close); cleared ONLY in replayHistory's render, which also + # cancels the pending failure retry; the retry is scheduled from + # the clear_ui .then (bounded by construction — a retry's own + # failure cannot re-schedule) and fire-time-gated turn-free; the + # idle edge backstops a double failure via else-if BEHIND the + # truncated branch (whose own render heals the latch too). + assert cl_seg.index("this._historyStale = true;") < cl_seg.index( + "this._beginReplayQuiesce(token);" + ), "the staleness latch must be set before the quiesce is armed" + assert "this._historyStale = false;" in replay_seg, ( + "replayHistory must clear the staleness latch (its only clear site)" + ) + assert "clearTimeout(this._staleRetryTimer);" in replay_seg + assert "this._staleRetryTimer = setTimeout(" in cl_seg + assert "!this.currentAssistantEl &&" in cl_seg, ( + "the clear_ui failure retry must be turn-free-gated" + ) + assert "} else if (this._historyStale && !this._replayQueue) {" in body, ( + "the idle edge must backstop the latch behind the truncated branch, " + "skipping edges with a quiesced fetch already in flight" + ) + # The backstop must be TRANSPORT-FREE: a quiesced same-token refetch, + # never _loadHistoryThenConnect — the reload's fresh reconnect draws + # the server's synthetic state_change:idle back into this branch's + # own trigger (the round-5 storm). + backstop = body.index("} else if (this._historyStale && !this._replayQueue) {") + backstop_seg = body[backstop : backstop + 2200] + assert "this._refetchHistory(this.wsId, staleToken);" in backstop_seg, ( + "the staleness backstop must heal via a quiesced REST refetch" + ) + assert "this._loadHistoryThenConnect(" not in backstop_seg, ( + "the staleness backstop must never touch the transport (#890 r5)" + ) + # The retry yields to an in-flight quiesce (no same-token stomp). + retry = cl_seg.index("this._staleRetryTimer = setTimeout(") + assert "!this._replayQueue &&" in cl_seg[retry : retry + 700], ( + "the clear_ui retry must yield to an in-flight quiesced fetch" + ) + + # Terminal teardown must cancel the failure retry: destroy() bumps + # no token and clears no latch, so an armed timer would otherwise + # pass its fire-time guards ~2s post-destroy and replayHistory into + # the detached pane. (disconnectSSE deliberately does NOT cancel + # it — transport-only reconnects keep the pending heal intent.) + dest = body.index("destroy() {") + dest_seg = body[dest : dest + 1600] + assert "clearTimeout(pane._staleRetryTimer);" in dest_seg, ( + "destroy() must cancel the clear_ui failure retry (#890)" ) diff --git a/turnstone/console/static/coordinator/coordinator.js b/turnstone/console/static/coordinator/coordinator.js index f161248b8..3401bf05d 100644 --- a/turnstone/console/static/coordinator/coordinator.js +++ b/turnstone/console/static/coordinator/coordinator.js @@ -1174,8 +1174,10 @@ function createCoordinatorPane(root, wsId, opts) { if (preview.length > 600) preview = preview.slice(0, 600) + "…"; } else if (argsRaw) { // Malformed JSON or non-object args — show the raw payload - // truncated. Matches the interactive replay's substring(0, 100) - // fallback at ui/static/app.js Pane.replayHistory. + // truncated. Matches the interactive replay's fallback in + // shared_static/interactive.js replayHistory (the live parity + // source; the old ui/static/app.js Pane.replayHistory moved + // there in the L-shell step-5a lift). header = name; preview = argsRaw.length > 200 ? argsRaw.slice(0, 200) + "…" : argsRaw; } @@ -5291,6 +5293,15 @@ function createCoordinatorPane(root, wsId, opts) { } function _rewindToMessage(msgEl) { + // KNOWN GAP (#894, do not re-derive): from clear_ui arrival until + // the next SUCCESSFUL refetchHistory render — the fetch window AND + // the failed-fetch aftermath — the stale transcript is visible + // with busy false, so this DOM count can over-rewind. Port + // interactive.js's #890 shape: a transcript-staleness latch set at + // clear_ui, cleared only by the full render, gating the mutating + // affordances (a plain refetch-in-flight flag is NOT enough — it + // reopens on the failed exit, the bug interactive hit). The + // retry leg needs a quiesce-free variant here. if (busy) return; const userMsgs = messagesEl.querySelectorAll(".msg.user"); const idx = Array.prototype.indexOf.call(userMsgs, msgEl); diff --git a/turnstone/shared_static/interactive.js b/turnstone/shared_static/interactive.js index 3b5780ece..10cec0208 100644 --- a/turnstone/shared_static/interactive.js +++ b/turnstone/shared_static/interactive.js @@ -267,6 +267,40 @@ class Pane { // resync no longer quiesces: it tears the stream down first, so no // live events exist during its rebuild.) this._replayQueue = null; + // Transcript-staleness latch (#890): TRUE = the visible transcript + // no longer matches the server's conversation STRUCTURE. Set at + // clear_ui arrival (the server just restructured) and on a ws + // reassignment (near-vacuous under one-Pane-per-ws, kept + // defensively); cleared in exactly ONE place — replayHistory's + // full render, the moment the DOM matches the server again. NOT + // tied to the quiesce: the quiesce tracks event buffering and + // releases on a FAILED fetch too, which left a window where a + // rewind/edit computed turn counts off the stale DOM (over-rewind). + // Row-level MUTATING affordances (rewind / edit / edit-and-resend) + // gate on this alongside ``busy``; a new mutating affordance must + // adopt the same gate. Route-L reloads deliberately do NOT set + // it: first paint renders into an empty pane (rewind early-returns + // on idx < 0), the truncated resync self-heals via its armed + // cursor, and the onLogin re-auth reload is count-preserving for + // rewind purposes except a server-side advance during the outage — + // where a stale count UNDER-rewinds (non-destructive) rather than + // over-rewinds. Accepted residual, ruled: after a DOUBLE fetch + // failure the latch persists and rewind/edit stay closed until the + // next idle-edge heal (any turn end) or reload — safe-closed + // strictly beats destructive-open, and a reconnect snapshot must + // NOT clear it (a snapshot heals only the in-flight turn, never + // the committed structure the latch tracks). + this._historyStale = false; + // Single bounded retry for a failed clear_ui refetch (transient + // 500s cluster in restart windows). Scheduled ONCE per clear_ui + // from its .then when the latch survived the fetch; fire-time + // guards: token still current, latch still set, and TURN-FREE + // (never replaceChildren a live bubble — the same rule the + // truncated resync's deferred branch follows; a busy pane defers + // to the idle-edge heal instead). Superseded by any newer load + // (token bump) and cancelled in replayHistory's supersession + // block. + this._staleRetryTimer = null; // Hot-path caches — all invalidated by _clearAgentTracking/replayHistory. // _nearBottom mirrors the scroller position via a passive scroll listener // (no per-token geometry reads); the two Maps make per-event row/stream @@ -1649,6 +1683,38 @@ class Pane { // same-ws reload keeps it armed until a successful full render // (replayHistory clears it) so a failed fetch can retry below. if (this.wsId !== wsId) this._truncatedFromCursor = null; + // A reassignment also invalidates the staleness latch's referent — + // whatever transcript is visible belongs to the OLD ws, so any + // DOM-derived count is wrong until the new ws renders. Set, not + // clear: replayHistory's render releases it. (Near-vacuous under + // one-Pane-per-ws; kept defensively with the other wsId!==wsId + // branches.) + if (this.wsId !== wsId) this._historyStale = true; + // Streaming-ref reset, gated on RESUMABILITY rather than ws + // identity: refs may survive a reload only when the reconnect can + // resume into them — same ws AND an armed truncation cursor (the + // ring replay continues the mid-jitter bubble; resetting there + // orphaned it beside a duplicate). Every other flavor resets: a + // ws SWITCH (the drop above just nulled the cursor — old-ws refs + // must not survive into the new ws's stream), a first paint or + // idle-edge reload (refs already null; no-op), and an UNARMED + // same-ws reload (the onLogin re-auth fan-out): its reconnect is + // cursorless, a turn that completed during the outage sends no + // resuming snapshot, and a stale non-null ref would concatenate + // the NEXT turn's content into the old bubble at its old DOM + // position. + // + // KNOWN RESIDUAL (ruled, do not re-derive): the armed-case + // preservation adds one route into the pre-existing cross-client + // in_progress_snapshot stale-ref secondary — a turn that starts + // streaming during the resync jitter leaves these refs non-null, + // and a reconnect snapshot reflecting a NEWER turn reuses the + // stale bubble (transient; heals at the next idle-edge resync). + // Deferred to the both-clients snapshot sweep: resetting armed + // refs here kills the mid-jitter continuity contract, and + // patching the snapshot handler in ONE client diverges parity + // (coordinator.js's handler is identical). + if (this._truncatedFromCursor == null) this._resetStreamingRefs(); this._replayQueue = null; this._clearAgentTracking(); this._pendingTruncatedResync = false; @@ -1740,14 +1806,19 @@ class Pane { this._endReplayQuiesce(token); } } else { - // Failure path never reaches replayHistory — reset the streaming refs - // here too, or the flushed backlog and resumed live events would paint - // into the subtree clear_ui already wiped. It also leaves - // _truncatedFromCursor SET (only replayHistory clears it), which is - // what arms the connect chokepoint's retry for a failed truncated - // resync. - this._resetStreamingRefs(); - this.showEmptyState(); + // Failed fetch = DOM + ref + repair-intent no-op (#890, the G3 + // guard-before-wipe ported from coordinator.js refetchHistory). + // The prior transcript stays on screen: no wipe ever ran (see + // the clear_ui case), and appending an empty-state hint below + // real content was the old resync-route wart. The streaming + // refs stay untouched — null per the turn-free-at-clear_ui + // invariant, or validly attached on the resync route, where + // resetting them orphaned a mid-jitter turn's bubble and let + // the reconnect snapshot build a duplicate beside it. + // _truncatedFromCursor stays SET (only replayHistory clears + // it), which is what arms the connect chokepoint's retry for a + // failed truncated resync. Only the quiesce must release, or + // a queued clear_ui backlog would wedge the pane. this._endReplayQuiesce(token); } } @@ -2008,6 +2079,40 @@ class Pane { this._pendingTruncatedResync = false; this._recordTruncatedGap(); this._loadHistoryThenConnect(this.wsId); + } else if (this._historyStale && !this._replayQueue) { + // Staleness-latch backstop (#890): a clear_ui refetch and + // its one bounded retry both failed, so the transcript + // still doesn't match the server and rewind/edit are + // latch-closed. The turn just settled — refetch now. + // + // TRANSPORT-FREE BY DESIGN (ruled, do not "upgrade" this + // to _loadHistoryThenConnect): the heal must never touch + // the stream. A reload's fresh reconnect draws the + // server's synthetic state_change:idle, which lands back + // in THIS branch — with /history down that was a + // zero-backoff disconnect/refetch/reconnect storm against + // a recovering node (the round-5 critical). A quiesced + // REST refetch emits zero SSE events, so the trigger edge + // is only ever organic (turn-settle-driven, no server + // heartbeats) and the loop is structurally impossible. + // Stream death has its own owners (EventSource native + // auto-reconnect, the host recovery beat, the truncated + // resync) — the old reload's reconnect was redundant. + // The !_replayQueue guard skips the edge when a quiesced + // fetch is already in flight. The else keeps the + // truncated branch's reload from doubling up (its render + // clears the latch too). On another failure the latch + // persists and the next ORGANIC settle retries. Accepted + // liveness lag: if /history recovers while the pane sits + // idle untouched, rewind/edit stay closed until the next + // organic settle — strictly safer than the storm. + // + // Fire-and-forget (no .catch) is deliberate: no composer state + // rides this heal to un-strand (that is the clear_ui caller's + // .catch), so a render throw stays loud, as in the load path. + const staleToken = this._historyLoadToken; + this._beginReplayQuiesce(staleToken); + this._refetchHistory(this.wsId, staleToken); } // Only steal focus if this is the active pane and no approval pending. if (this._host.isFocused(this) && !this.pendingApproval) { @@ -2226,28 +2331,87 @@ class Pane { case "clear_ui": { // Conversation was structurally reset (rewind / retry / resume / - // open / fork). Empty the pane for immediate feedback, then - // re-render from REST and dispatch any queued edit-and-resend - // once the (possibly truncated) history lands. The resend keys - // off this signal rather than an inline history SSE event. Capture - // the load token AND the ws identity: a ws SWITCH mid-flight - // discards both the re-render and the resend (no cross-ws send), - // but a SAME-ws supersession — the jittered truncated resync - // bumps _historyLoadToken through _loadHistoryThenConnect — must - // still fire the resend: the superseding load renders the same - // post-rewind history, and silently dropping the queued edit - // stranded the composer busy with a message that never sent. + // open / fork). Re-render from REST and dispatch any queued + // edit-and-resend once the (possibly truncated) history lands. + // The resend keys off this signal rather than an inline history + // SSE event. + // + // No pre-wipe (#890, the guard-before-wipe shape coord shipped + // in #882's G3): the wipe + streaming-ref reset live in + // replayHistory, reached only on a SUCCESSFUL fetch — so a + // failed /history keeps the prior transcript on screen. + // Stale-but-real beats a blank pane on a live stream, and + // /history failures cluster in exactly the restart windows + // that emit clear_ui replays. The quiesce stays armed across + // the fetch either way: queued live events replay into the + // rebuilt pane on success, or onto the stale-but-real one on + // failure — safe because clear_ui emitters are turn-free at + // emission (see the _resyncTimer field comment), so the + // surviving streaming refs are null and replayed events build + // fresh rows below the stale transcript. The stale rows stay + // VISIBLE but their mutating affordances gate on the + // _historyStale latch (set below, cleared only by a successful + // replayHistory render — see the field comment), so a turn + // count computed off the stale DOM can't reach the server + // during the fetch OR after a failed one. A failed fetch + // schedules one turn-free retry (the .then below); the + // idle-edge heal backstops a double failure. Coord's cosmetic + // [data-busy] grey-out stays a deferred parity item (#890 PR). + // + // Capture the load token AND the ws identity: a ws SWITCH + // mid-flight discards both the re-render and the resend (no + // cross-ws send), but a SAME-ws supersession — the jittered + // truncated resync bumps _historyLoadToken through + // _loadHistoryThenConnect — must still fire the resend: the + // superseding load renders the same post-rewind history, and + // silently dropping the queued edit stranded the composer busy + // with a message that never sent. On a FAILED fetch the + // resend still fires (the rewind already committed + // server-side); its bubble lands on the stale transcript and + // the rewound truth arrives with the next successful refetch — + // coord's same ruling. const token = this._historyLoadToken; const editWs = this.wsId; + this._historyStale = true; this._beginReplayQuiesce(token); - this.messagesEl.replaceChildren(); - this._resetStreamingRefs(); this._refetchHistory(this.wsId, token) .then(() => { + // Failed fetch (the latch survived — only replayHistory + // clears it): schedule the ONE bounded retry. Scheduled + // here rather than in the failure branch so a retry's own + // failure cannot re-schedule (bounded by construction); + // fire-time guards make it a no-op once superseded or + // while a turn streams (the idle-edge heal owns the busy + // case). + if (this._historyStale && token === this._historyLoadToken) { + if (this._staleRetryTimer) clearTimeout(this._staleRetryTimer); + this._staleRetryTimer = setTimeout(() => { + this._staleRetryTimer = null; + if ( + token === this._historyLoadToken && + this._historyStale && + !this._replayQueue && + !this.busy && + !this.currentAssistantEl && + !this.currentReasoningEl + ) { + // !_replayQueue: yield to an in-flight quiesced + // fetch (the idle-edge backstop shares this token) + // instead of stomping its queue for a same-token + // double-render. Fire-and-forget like the backstop: + // no composer state to strand here, so a render throw is + // left loud/uncaught (see the backstop's note). + this._beginReplayQuiesce(token); + this._refetchHistory(this.wsId, token); + } + }, 2000); + } if (token !== this._historyLoadToken && this.wsId !== editWs) { - // Cross-ws supersession: release the latch + busy so the - // composer recovers; the edit belongs to a pane state that - // no longer exists. + // Cross-ws supersession: drop the pending edit + release + // busy so the composer recovers; the edit belongs to a pane + // state that no longer exists. (_historyStale is NOT touched + // here — it clears only on a successful replayHistory render; + // the composer send path never consults the latch.) this._pendingEditSend = null; this.setBusy(false); return; @@ -2836,7 +3000,7 @@ class Pane { } _rewindToMessage(msgEl) { - if (this.busy) return; + if (this.busy || this._historyStale) return; // Count how many user messages come at or after this one. Bare // ``.msg.user`` is intentional: system-nudge markers carry that // class and the server's _find_turn_boundaries counts them as @@ -2849,7 +3013,7 @@ class Pane { } _startEdit(msgEl, originalText) { - if (this.busy) return; + if (this.busy || this._historyStale) return; // Save current child nodes for cancel restoration const savedNodes = []; while (msgEl.firstChild) { @@ -2911,7 +3075,7 @@ class Pane { } _editAndResend(msgEl, newText) { - if (this.busy) return; + if (this.busy || this._historyStale) return; // Count turns to rewind (from this message onward). Bare // ``.msg.user`` matches the server's turn semantics — see // _rewindToMessage. @@ -2970,13 +3134,15 @@ class Pane { } _resetStreamingRefs() { - // Null every ref that can point into a wiped subtree, so the next event - // creates fresh targets instead of writing invisibly into detached - // nodes. Called wherever the transcript DOM is (or is about to be) - // replaced — replayHistory, the clear_ui immediate wipe, and the - // refetch-FAILURE path (which shows the empty state without ever - // reaching replayHistory; leaving refs stale there made the retried - // generation's whole first segment stream into a detached bubble). + // Null every ref that can point into a wiped or non-resumable + // subtree, so the next event creates fresh targets instead of + // writing invisibly into detached nodes (or into a superseded + // bubble). Callers (#890 shape): replayHistory's full render (the + // subtree was just replaced) and _loadHistoryThenConnect's + // non-resumable reloads (no armed truncation cursor — the + // reconnect cannot resume into the surviving refs). The clear_ui + // pre-wipe and the refetch-failure call are GONE: a failed fetch + // preserves the pane, and preserved DOM keeps valid refs. this.currentAssistantEl = null; this.currentAssistantBodyEl = null; this.currentReasoningEl = null; @@ -3015,6 +3181,14 @@ class Pane { clearTimeout(this._resyncTimer); this._resyncTimer = null; } + // The render also restores structural truth — clear the staleness + // latch (its ONLY clear site) and cancel any pending clear_ui + // failure retry, which exists to produce exactly this render. + this._historyStale = false; + if (this._staleRetryTimer) { + clearTimeout(this._staleRetryTimer); + this._staleRetryTimer = null; + } // The rebuild just orphaned any in-flight streaming targets — reset them, // and release the agent-card/orphan maps whose entries now point at // replaced subtrees (detached-DOM retention). @@ -4884,6 +5058,11 @@ function createInteractivePane(root, wsId, opts) { active = true; if (!connected) { connected = true; + // Load-bearing pre-seed (#890): _refetchHistory's failure path + // no longer appends an empty-state, so a FAILED first paint + // shows this placeholder only because it is seeded here, + // before the fetch. Removing it would leave a genuinely blank + // pane when the first /history fails. pane.showEmptyState(); pane._loadHistoryThenConnect(wsId); } @@ -4914,6 +5093,16 @@ function createInteractivePane(root, wsId, opts) { clearTimeout(recoverTimer); recoverTimer = null; } + // The clear_ui failure retry survives everything EXCEPT a token + // bump — and destroy() bumps nothing, so an armed retry would + // fire ~2s post-destroy, pass its guards (latch still set, refs + // null), and replayHistory into the detached pane. Cancel it + // here, terminal-only: disconnectSSE must NOT cancel it — + // transport-only reconnects keep the pending heal intent. + if (pane._staleRetryTimer) { + clearTimeout(pane._staleRetryTimer); + pane._staleRetryTimer = null; + } pane.disconnectSSE(); // The document-level visibilitychange listener holds a strong ref // to the pane — leaving it registered would both leak the pane and