Report liveness while a provider call is in flight (Trace ABI v8)

.github/workflows/ci.yml

@@ -140,7 +140,7 @@ jobs:
           PY
       - name: Distributions bundle the exact event ABI corpus (#144)
         run: |
-          for f in trace-v7-execution.ndjson trace-v7-lifecycle.ndjson runner-v10-refusal.ndjson; do
+          for f in trace-v8-execution.ndjson trace-v8-lifecycle.ndjson runner-v10-refusal.ndjson; do
             unzip -l dist/*.whl | grep -q "droste/testing/fixtures/$f" || {
               echo "wheel is missing droste/testing/fixtures/$f"; exit 1;
             }
@@ -154,21 +154,21 @@ jobs:
 
           from droste.testing import (
               runner_v10_refusal_ndjson,
-              trace_v7_execution_ndjson,
-              trace_v7_lifecycle_ndjson,
+              trace_v8_execution_ndjson,
+              trace_v8_lifecycle_ndjson,
           )
 
           source = Path(sys.argv[1])
-          assert trace_v7_execution_ndjson() == (source / "trace-v7-execution.ndjson").read_bytes()
-          assert trace_v7_lifecycle_ndjson() == (source / "trace-v7-lifecycle.ndjson").read_bytes()
+          assert trace_v8_execution_ndjson() == (source / "trace-v8-execution.ndjson").read_bytes()
+          assert trace_v8_lifecycle_ndjson() == (source / "trace-v8-lifecycle.ndjson").read_bytes()
           assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes()
           PY
           sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')"
           tar xzf dist/droste-*.tar.gz -C "$tmp"
-          cmp src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \
-            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson"
-          cmp src/droste/testing/fixtures/trace-v7-execution.ndjson \
-            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-execution.ndjson"
+          cmp src/droste/testing/fixtures/trace-v8-lifecycle.ndjson \
+            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v8-lifecycle.ndjson"
+          cmp src/droste/testing/fixtures/trace-v8-execution.ndjson \
+            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v8-execution.ndjson"
           cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \
             "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson"
 
@@ -319,7 +319,7 @@ jobs:
           PY
       - name: Distributions bundle the exact event ABI corpus (#144)
         run: |
-          for f in trace-v7-execution.ndjson trace-v7-lifecycle.ndjson runner-v10-refusal.ndjson; do
+          for f in trace-v8-execution.ndjson trace-v8-lifecycle.ndjson runner-v10-refusal.ndjson; do
             unzip -l dist/*.whl | grep -q "droste/testing/fixtures/$f" || {
               echo "wheel is missing droste/testing/fixtures/$f"; exit 1;
             }
@@ -333,21 +333,21 @@ jobs:
 
           from droste.testing import (
               runner_v10_refusal_ndjson,
-              trace_v7_execution_ndjson,
-              trace_v7_lifecycle_ndjson,
+              trace_v8_execution_ndjson,
+              trace_v8_lifecycle_ndjson,
           )
 
           source = Path(sys.argv[1])
-          assert trace_v7_execution_ndjson() == (source / "trace-v7-execution.ndjson").read_bytes()
-          assert trace_v7_lifecycle_ndjson() == (source / "trace-v7-lifecycle.ndjson").read_bytes()
+          assert trace_v8_execution_ndjson() == (source / "trace-v8-execution.ndjson").read_bytes()
+          assert trace_v8_lifecycle_ndjson() == (source / "trace-v8-lifecycle.ndjson").read_bytes()
           assert runner_v10_refusal_ndjson() == (source / "runner-v10-refusal.ndjson").read_bytes()
           PY
           sdist_root="$(tar tzf dist/droste-*.tar.gz | sed -n '1s#/.*##p')"
           tar xzf dist/droste-*.tar.gz -C "$tmp"
-          cmp src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \
-            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-lifecycle.ndjson"
-          cmp src/droste/testing/fixtures/trace-v7-execution.ndjson \
-            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v7-execution.ndjson"
+          cmp src/droste/testing/fixtures/trace-v8-lifecycle.ndjson \
+            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v8-lifecycle.ndjson"
+          cmp src/droste/testing/fixtures/trace-v8-execution.ndjson \
+            "$tmp/$sdist_root/src/droste/testing/fixtures/trace-v8-execution.ndjson"
           cmp src/droste/testing/fixtures/runner-v10-refusal.ndjson \
             "$tmp/$sdist_root/src/droste/testing/fixtures/runner-v10-refusal.ndjson"
 

.github/workflows/release.yml

@@ -80,8 +80,8 @@ jobs:
           staging="droste-relay-$GITHUB_REF_NAME"
           mkdir -p "$staging/conformance" relay-dist
           cp src/droste/substrates/_relay/*.ts pyodide/README.md "$staging/"
-          cp src/droste/testing/fixtures/trace-v7-execution.ndjson \
-            src/droste/testing/fixtures/trace-v7-lifecycle.ndjson \
+          cp src/droste/testing/fixtures/trace-v8-execution.ndjson \
+            src/droste/testing/fixtures/trace-v8-lifecycle.ndjson \
             src/droste/testing/fixtures/runner-v10-refusal.ndjson \
             "$staging/conformance/"
           printf '%s %s\n' "$GITHUB_REF_NAME" "$GITHUB_SHA" > "$staging/DROSTE_VERSION"

UPGRADING.md

@@ -29,10 +29,10 @@ value and never schema-checks it. A provider that raises is reported through a
 `RuntimeWarning` and the checkpoint carries `payload: null` — a checkpoint can
 never fail a run.
 
-Hosts must accept Trace ABI 7 and update to the `trace_v7_*` conformance
-fixtures (`droste.testing.trace_v7_lifecycle_ndjson` /
-`trace_v7_execution_ndjson`, backing `trace-v7-lifecycle.ndjson` /
-`trace-v7-execution.ndjson`). Scaffold manifests report `abis.trace: 7`, so
+Hosts must accept Trace ABI 7 and update to the `trace_v8_*` conformance
+fixtures (`droste.testing.trace_v8_lifecycle_ndjson` /
+`trace_v8_execution_ndjson`, backing `trace-v8-lifecycle.ndjson` /
+`trace-v8-execution.ndjson`). Scaffold manifests report `abis.trace: 7`, so
 every manifest id changes; pinned ids must be re-derived. Strict v6 readers
 reject the new event and the new version, so this is an atomic consumer
 migration.

docs/trace-abi.md

@@ -230,12 +230,12 @@ and sdist. Python consumers load them through package resources:
 ```python
 from droste.testing import (
     runner_v10_refusal_ndjson,
-    trace_v7_execution_ndjson,
-    trace_v7_lifecycle_ndjson,
+    trace_v8_execution_ndjson,
+    trace_v8_lifecycle_ndjson,
 )
 
-execution_lines = trace_v7_execution_ndjson().splitlines()
-event_lines = trace_v7_lifecycle_ndjson().splitlines()
+execution_lines = trace_v8_execution_ndjson().splitlines()
+event_lines = trace_v8_lifecycle_ndjson().splitlines()
 pre_admission_refusal = runner_v10_refusal_ndjson()
 ```
 

examples/pyodide-host/e2e_test.ts

@@ -33,7 +33,7 @@ const RUNNER_REFUSAL_FIXTURE = new URL(
   import.meta.url,
 );
 const TRACE_LIFECYCLE_FIXTURE = new URL(
-  "../../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson",
+  "../../src/droste/testing/fixtures/trace-v8-lifecycle.ndjson",
   import.meta.url,
 );
 const TEST_BUDGET = {

pyodide/event_channel_probe.ts

@@ -4,7 +4,7 @@ const mode = Deno.args[0];
 const channel = eventChannelFromEnvironment();
 const fixture = await Deno.readTextFile(
   new URL(
-    "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson",
+    "../src/droste/testing/fixtures/trace-v8-lifecycle.ndjson",
     import.meta.url,
   ),
 );

pyodide/event_channel_test.ts

@@ -8,7 +8,7 @@ import {
 import { isRlmEvent } from "../src/droste/substrates/_relay/events.ts";
 
 const TRACE_LIFECYCLE_FIXTURE = new URL(
-  "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson",
+  "../src/droste/testing/fixtures/trace-v8-lifecycle.ndjson",
   import.meta.url,
 );
 

pyodide/events_test.ts

@@ -37,6 +37,7 @@ const BODIES: Record<string, Record<string, unknown>> = {
   },
   execution_error: { iteration: 1, error_type: "ValueError", message: "bad" },
   reasoning_delta: { text: "thinking" },
+  heartbeat: { elapsed_ms: 15000 },
   subcall: {
     phase: "start",
     call_id: "call-1",
@@ -169,7 +170,7 @@ function wire(
     run_id: "run-1",
     seq: 1,
     timestamp: "2026-07-14T00:00:00Z",
-    version: 7,
+    version: 8,
     persistence_class: persistence ?? PERSISTENCE_BY_TYPE[type],
     depth: 0,
     ...body,
@@ -383,7 +384,7 @@ Deno.test("successful output beginning ERROR remains an output event", () => {
 
 Deno.test("Python and relay accept the same execution golden NDJSON", async () => {
   const fixture = new URL(
-    "../src/droste/testing/fixtures/trace-v7-execution.ndjson",
+    "../src/droste/testing/fixtures/trace-v8-execution.ndjson",
     import.meta.url,
   );
   const lines = (await Deno.readTextFile(fixture)).trim().split("\n");
@@ -427,7 +428,7 @@ Deno.test("Python and relay accept the same execution golden NDJSON", async () =
 
 Deno.test("Python and relay accept the same lifecycle golden NDJSON", async () => {
   const fixture = new URL(
-    "../src/droste/testing/fixtures/trace-v7-lifecycle.ndjson",
+    "../src/droste/testing/fixtures/trace-v8-lifecycle.ndjson",
     import.meta.url,
   );
   const lines = (await Deno.readTextFile(fixture)).trim().split("\n");
@@ -609,6 +610,7 @@ Deno.test("vocabulary matches the engine's emitters", () => {
       "code",
       "done",
       "execution_error",
+      "heartbeat",
       "extract",
       "iteration_start",
       "llm_response",

pyodide/heartbeat_test.ts

@@ -0,0 +1,108 @@
+// Liveness while a provider call is in flight.
+//
+// A host's no-progress watchdog cannot tell "blocked on the network" from
+// "wedged in WASM" by silence alone, and was killing healthy runs whose subcall
+// simply took a while. Batch calls make this worst: they post to
+// `/responses/batch`, which does not match the `/responses` streaming suffix, so
+// they emit no `reasoning_delta` either — nothing at all between the subcall's
+// start and its completion.
+//
+// The heartbeat is emitted from the relay rather than the engine on purpose.
+// Pyodide runs on this same thread, so a timer here cannot fire while generated
+// code spins in WASM — which is exactly when a watchdog SHOULD fire. It ticks
+// only while the event loop is free, i.e. only when the process is healthy.
+//
+// Run: deno test --allow-read --allow-env pyodide/heartbeat_test.ts
+import { assert, assertEquals } from "jsr:@std/assert@1";
+
+const { isRlmEvent, PERSISTENCE_BY_TYPE, RLM_EVENT_TYPES } = await import(
+  "../src/droste/substrates/_relay/events.ts"
+);
+
+function wire(body: Record<string, unknown>): string {
+  return JSON.stringify({
+    type: "heartbeat",
+    version: 8,
+    run_id: "run-1",
+    seq: 4,
+    timestamp: "2026-08-03T00:00:00Z",
+    depth: 0,
+    persistence_class: "transient",
+    ...body,
+  });
+}
+
+Deno.test("the relay forwards a heartbeat", () => {
+  assert(isRlmEvent(wire({ elapsed_ms: 15_000 })));
+  assert(isRlmEvent(wire({ elapsed_ms: 0 })));
+});
+
+Deno.test("a heartbeat carries liveness and nothing else", () => {
+  // Content would make it retainable and turn a liveness ping into a privacy
+  // question. The body is exactly one non-negative reading.
+  assert(!isRlmEvent(wire({ elapsed_ms: 15_000, text: "leaked" })));
+  assert(!isRlmEvent(wire({})));
+  assert(!isRlmEvent(wire({ elapsed_ms: -1 })));
+  assert(!isRlmEvent(wire({ elapsed_ms: "15000" })));
+});
+
+Deno.test("heartbeats are transient, so they never reach a run record", () => {
+  assertEquals(PERSISTENCE_BY_TYPE.heartbeat, "transient");
+  assert(RLM_EVENT_TYPES.has("heartbeat"));
+});
+
+Deno.test("the timer only ticks while the event loop is free", async () => {
+  // The property the whole design rests on. Reproduced here with the same
+  // shape the relay uses: an interval alongside awaited work.
+  const ticks: number[] = [];
+  const started = Date.now();
+  const timer = setInterval(() => ticks.push(Date.now() - started), 20);
+  try {
+    // Awaiting yields, so the timer runs — a healthy provider wait.
+    await new Promise((resolve) => setTimeout(resolve, 120));
+    assert(
+      ticks.length >= 3,
+      `expected ticks while awaiting, got ${ticks.length}`,
+    );
+
+    // Blocking the thread is what a wedged Pyodide execution looks like from
+    // here. No tick may land during it, which is what lets a watchdog still
+    // (correctly) detect a wedge.
+    const before = ticks.length;
+    const spinUntil = Date.now() + 120;
+    while (Date.now() < spinUntil) { /* occupy the single thread */ }
+    assertEquals(
+      ticks.length,
+      before,
+      "a blocked thread must not be able to report itself alive",
+    );
+  } finally {
+    clearInterval(timer);
+  }
+});
+
+// The shape a real subcall actually produced, captured from a live 75-second
+// provider call. Pinned verbatim because the first implementation wrapped only
+// `fetch` — which a streamed response resolves as soon as headers arrive — and
+// so reported nothing for exactly the wait it exists to cover. The mechanism
+// test above passed anyway, because it proved the timer worked rather than that
+// it spanned the right window. Zero heartbeats in a 4428-delta live run is what
+// actually caught it.
+Deno.test("a heartbeat a live subcall produced is forwarded", () => {
+  const observed = {
+    type: "heartbeat",
+    elapsed_ms: 15003,
+    run_id: "ca4fd03d-5532-4255-9669-86165c4bb0a3",
+    parent_run_id: "f6708912-b58d-4e3d-b0e9-a2c28d608635",
+    depth: 1,
+    seq: 2,
+    timestamp: "2026-08-03T21:46:54.805Z",
+    version: 8,
+    persistence_class: "transient",
+  };
+
+  assert(isRlmEvent(JSON.stringify(observed)));
+  // depth 1 is a subcall — the calls that run long and, when batched, stream
+  // nothing at all. That is the case this whole change exists for.
+  assertEquals(observed.depth, 1);
+});

src/droste/execution/progress.py

@@ -40,6 +40,7 @@
         "output",  # {iteration, stdout, calls_made, answer_ready, answer_content_chars}
         "execution_error",  # {iteration, error_type, message} — a step failed; repair may follow
         "reasoning_delta",  # relay-side {text}, from streamed /responses
+        "heartbeat",  # relay-side {elapsed_ms} — a provider call is still in flight
         "subcall",  # broker-correlated subcall lifecycle facts
         "repair",  # discriminated repair lifecycle facts
         "extract",  # discriminated terminal extraction lifecycle facts

src/droste/execution/trace.py

@@ -17,7 +17,7 @@
 from typing import Any, Callable, Mapping
 from uuid import uuid4
 
-TRACE_ABI_VERSION = 7
+TRACE_ABI_VERSION = 8
 
 
 class PersistenceClass(str, Enum):
@@ -44,7 +44,9 @@ class PersistenceClass(str, Enum):
         "checkpoint",
     }
 )
-TRANSIENT_EVENT_TYPES = frozenset({"startup", "progress", "reasoning_delta", "usage_progress"})
+TRANSIENT_EVENT_TYPES = frozenset(
+    {"startup", "progress", "reasoning_delta", "usage_progress", "heartbeat"}
+)
 
 PERSISTENCE_BY_TYPE: Mapping[str, PersistenceClass] = MappingProxyType(
     {
@@ -90,6 +92,9 @@ class PersistenceClass(str, Enum):
             {},
         ),
         "reasoning_delta": ({"text": str}, {}),
+        # Pure liveness, carrying no content: the relay is still waiting on a
+        # provider call. `elapsed_ms` is diagnostic only, never a budget input.
+        "heartbeat": ({"elapsed_ms": int}, {}),
         "subcall": (
             {
                 "phase": str,

src/droste/substrates/_relay/events.ts

@@ -15,6 +15,7 @@ export const RLM_EVENT_TYPES = new Set<string>([
   "output", // {iteration, stdout, calls_made, answer_ready, answer_content_chars}
   "execution_error", // {iteration, error_type, message} — a step failed; repair may follow (#35)
   "reasoning_delta", // {text} — emitted relay-side from streamed /responses
+  "heartbeat", // {elapsed_ms} — relay-side liveness while a provider call is in flight
   "subcall", // broker-correlated start/progress/completion/failure
   "repair", // discriminated repair start/completion/failure
   "extract", // discriminated extract start/completion/failure
@@ -33,6 +34,7 @@ export const PERSISTENCE_BY_TYPE: Readonly<Record<string, string>> = {
   startup: "transient",
   progress: "transient",
   reasoning_delta: "transient",
+  heartbeat: "transient",
   usage_progress: "transient",
   iteration_start: "configurable",
   llm_response: "configurable",
@@ -284,6 +286,11 @@ function validBody(type: string, body: Record<string, unknown>): boolean {
       return exactBody(body, ["iteration", "error_type", "message"]) &&
         integerField("iteration") && stringField("error_type") &&
         stringField("message");
+    case "heartbeat":
+      // Liveness only. Carries no content, so nothing to validate beyond the
+      // envelope and a non-negative elapsed reading.
+      return exactBody(body, ["elapsed_ms"]) && integerField("elapsed_ms") &&
+        Number(body.elapsed_ms) >= 0;
     case "reasoning_delta":
       return exactBody(body, ["text"]) && stringField("text");
     case "subcall":
@@ -431,7 +438,7 @@ export function isRlmEvent(line: string): boolean {
         Number.isInteger(o.seq) && o.seq > 0 &&
         typeof o.timestamp === "string" &&
         /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(o.timestamp) &&
-        o.version === 7 &&
+        o.version === 8 &&
         o.persistence_class === PERSISTENCE_BY_TYPE[o.type] &&
         Number.isInteger(o.depth) && o.depth >= 0 &&
         (o.depth === 0