The working set stops being a blob every submit overwrites (v0.5.0)

migrations/015_target_facts.sql

@@ -0,0 +1,50 @@
+-- 015: Established facts get their own append-only table.
+--
+-- WHY. `targets.state` is a single JSONB blob that submitResult REPLACED
+-- wholesale (`UPDATE targets SET state = $2`). Three consequences, all of which
+-- showed up on firstproof-c4:
+--
+--   1. An agent adding one fact had to reconstruct the entire working set or
+--      clobber it, so what was already established could be lost by any submit.
+--   2. Debris became sticky. A note about one attempt's formatting failure, and
+--      an auto-written salvage from a timed-out run, were copied forward by
+--      every later agent — because dropping a key it did not understand was
+--      indistinguishable from destroying a real result.
+--   3. The 64 KB cap truncated by keeping the TAIL, so the first thing dropped
+--      as state grew was the accumulated head: the established facts.
+--
+-- Facts are the part that must never be lost or rewritten, so they move out of
+-- the blob and into rows: append-only, provenance-carrying, and unbounded by
+-- the blob cap. `state` keeps its job of holding the *mutable* working set
+-- (cursors, current phase, scratch), which is now merged per key rather than
+-- replaced (see mergeStateUpdate).
+--
+-- Append-only, with retraction rather than deletion: a fact recorded in error
+-- is superseded on the record, never erased, so the history of what was
+-- believed stays auditable. Same principle as the ledger.
+
+CREATE TABLE target_facts (
+  id BIGSERIAL PRIMARY KEY,
+  target_id UUID NOT NULL REFERENCES targets(id) ON DELETE CASCADE,
+  -- The claim, in the agent's own words. Prose on purpose: what counts as a
+  -- fact about an open conjecture is not enumerable in advance.
+  claim TEXT NOT NULL,
+  -- The contribution that established it. Nullable because an admin may seed a
+  -- fact when curating a target, before any volunteer has worked it.
+  established_by BIGINT REFERENCES contributions(id),
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  -- Superseded, not deleted. Set when a later contribution shows the claim was
+  -- wrong; the row stays and stops counting as current.
+  retracted_at TIMESTAMPTZ,
+  retracted_reason TEXT,
+  CHECK (length(claim) > 0)
+);
+
+-- The feed reads one target's facts oldest-first; the id tiebreak keeps that
+-- order total (created_at alone ties when several land in one transaction).
+CREATE INDEX idx_target_facts_target ON target_facts (target_id, id);
+
+-- Idempotence. A replayed submit, or two agents independently establishing the
+-- same thing, must not stack duplicate rows. md5 rather than the raw claim so
+-- the index stays small regardless of how long a claim runs.
+CREATE UNIQUE INDEX uq_target_facts_claim ON target_facts (target_id, md5(claim));

package.json

@@ -1,6 +1,6 @@
 {
   "name": "givework",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "type": "module",
   "description": "Volunteer your AI agent's spare capacity to open mathematics",
   "license": "Apache-2.0",

src/executor.ts

@@ -684,7 +684,13 @@ export const RESULT_JSON_SCHEMA = {
     },
     state_update: {
       type: 'object',
-      description: "Replacement for the target's compacted working set",
+      description:
+        "Updates to the target's working set, MERGED per key — send only the keys you " +
+        'changed, never a copy of the whole thing. Two reserved keys: `facts` is an array ' +
+        'of short claims this run established (appended permanently, never overwritten by ' +
+        'a later run — put durable results here, not in ordinary keys), and `$retract` is ' +
+        'an array of key names to delete when scaffolding from an earlier attempt is no ' +
+        'longer true. Omitting a key leaves it untouched; only $retract removes one.',
     },
     artifact_uri: { type: 'string' },
     code_contribution: {
@@ -1466,11 +1472,13 @@ const SALVAGE_ARTIFACT_CHARS = 16_000;
  *    assistant events when any arrived, else a chars/4 token estimate over the
  *    prompt we sent plus whatever text came back (the prompt was certainly
  *    processed, so the floor is 1 cent — "free" would be a lie).
- *  - The STATE is merged, not replaced. state_update overwrites the target's
- *    compacted working set, and a timeout's fragments must never clobber the
- *    accumulated frontier — so the salvage rides in under a `timeout_salvage`
- *    key beside the existing state. If there is no partial text, or the current
- *    state isn't a mergeable object, no state_update is sent at all.
+ *  - The STATE update carries only `timeout_salvage`. Merging is the server's
+ *    job now (mergeStateUpdate), so a timeout's fragments cannot clobber the
+ *    accumulated frontier no matter what else is on the target. This used to
+ *    spread the prior state back over itself to protect it, which worked but
+ *    re-published every key the run never touched — that is how a stale note
+ *    outlives the attempt that wrote it. If there is no partial text, no
+ *    state_update is sent at all.
  *  - Whatever text was captured is preserved in full (bounded) as the
  *    contribution's inline result/artifact, so the next agent can continue
  *    rather than restart. If truly nothing was captured, the contribution still
@@ -1512,21 +1520,20 @@ function salvageTimedOutRun(
         ? 'Partial output was captured and is attached for the next agent to continue from.'
         : 'No partial output could be captured before the kill; a full window produced nothing visible — consider a smaller chunk or a decomposition.');
 
-  // Merge-don't-clobber: only extend a plain-object state, never replace it.
-  const prior = task.target_state;
-  const mergeable = prior == null || (typeof prior === 'object' && !Array.isArray(prior));
-  const state_update =
-    salvage && mergeable
-      ? {
-          ...(prior as Record<string, unknown> | null | undefined),
-          timeout_salvage: {
-            task_id: task.task_id,
-            elapsed_ms: err.elapsedMs,
-            source,
-            partial: salvage.slice(0, SALVAGE_STATE_CHARS),
-          },
-        }
-      : undefined;
+  // Send ONLY our own key. The server merges per key (mergeStateUpdate), so
+  // re-sending the prior state to avoid clobbering it — which is what this used
+  // to do — is both unnecessary and actively harmful: echoing back keys this
+  // agent never looked at is how stale scaffolding got copied forward forever.
+  const state_update = salvage
+    ? {
+        timeout_salvage: {
+          task_id: task.task_id,
+          elapsed_ms: err.elapsedMs,
+          source,
+          partial: salvage.slice(0, SALVAGE_STATE_CHARS),
+        },
+      }
+    : undefined;
 
   return {
     result: {

src/operations.ts

@@ -932,6 +932,81 @@ function boundedProposal(proposed: unknown): unknown {
   };
 }
 
+/** Reserved keys in a state_update — routed, never stored in the blob. */
+const STATE_FACTS_KEY = 'facts';
+const STATE_RETRACT_KEY = '$retract';
+
+export interface ParsedStateUpdate {
+  /** The merged working set to store on targets.state. */
+  state: Record<string, unknown>;
+  /** Claims to append to target_facts. */
+  facts: string[];
+  /** Working-set keys the agent explicitly dropped. */
+  retracted: string[];
+}
+
+/**
+ * Merge a state_update into the existing working set instead of replacing it.
+ *
+ * The old behaviour was `UPDATE targets SET state = $2` — last writer wins over
+ * the whole blob. That forced every agent to reconstruct the entire working set
+ * from scratch just to add one key, so any submit could silently destroy what
+ * earlier work had established, and any key an agent did not recognise got
+ * copied forward forever because dropping it looked exactly like vandalism.
+ *
+ * Now:
+ *   - unknown keys are merged per key, so adding one thing cannot drop another;
+ *   - `facts: [...]` is routed to the append-only target_facts table, out of
+ *     reach of both the blob cap and the next writer;
+ *   - `$retract: ["key", ...]` is the ONLY way to remove a working-set key, so
+ *     dropping stale scaffolding is a deliberate, attributable act rather than
+ *     a side effect of omission. This is what lets an agent clean up after a
+ *     dead end without an admin editing the field by hand.
+ *
+ * A non-object update (scalar, array, null) replaces wholesale, as before —
+ * targets seeded with a bare cursor keep working untouched.
+ */
+export function mergeStateUpdate(existing: unknown, update: unknown): ParsedStateUpdate {
+  const isPlain = (v: unknown): v is Record<string, unknown> =>
+    !!v && typeof v === 'object' && !Array.isArray(v);
+  if (!isPlain(update)) {
+    return { state: update as Record<string, unknown>, facts: [], retracted: [] };
+  }
+
+  // Facts may be given as plain strings or as {claim} objects — an agent
+  // writing structured output shouldn't have to guess which we take.
+  const rawFacts = update[STATE_FACTS_KEY];
+  const facts: string[] = [];
+  if (Array.isArray(rawFacts)) {
+    for (const f of rawFacts) {
+      const claim = typeof f === 'string' ? f : isPlain(f) ? String(f.claim ?? '') : '';
+      const trimmed = claim.trim();
+      if (trimmed) facts.push(trimmed);
+    }
+  }
+
+  const rawRetract = update[STATE_RETRACT_KEY];
+  const retracted = Array.isArray(rawRetract)
+    ? rawRetract.filter((k): k is string => typeof k === 'string' && k.length > 0)
+    : [];
+
+  // A truncation marker is a tombstone the platform wrote, not a working set an
+  // agent built, so the next real update REPLACES it rather than merging into
+  // it. Merging would pin `truncated`/`note`/`tail` in place permanently — a
+  // fresh species of exactly the sticky debris this function exists to end.
+  const isTombstone = isPlain(existing) && existing.truncated === true && 'tail' in existing;
+  const base = isPlain(existing) && !isTombstone ? { ...existing } : {};
+  for (const [k, v] of Object.entries(update)) {
+    if (k === STATE_FACTS_KEY || k === STATE_RETRACT_KEY) continue;
+    base[k] = v;
+  }
+  for (const k of retracted) delete base[k];
+  // Never let a retired reserved key survive in the stored blob.
+  delete base[STATE_FACTS_KEY];
+  delete base[STATE_RETRACT_KEY];
+  return { state: base, facts, retracted };
+}
+
 /**
  * Bound a state_update to MAX_STATE_BYTES by truncating instead of rejecting.
  * By the time a submit carries an oversized state the volunteer's tokens are
@@ -1426,13 +1501,10 @@ export async function submitResult(
   const summary = typeof opts.summary === 'string' ? opts.summary.slice(0, MAX_SUMMARY_CHARS) : '';
   // An oversized state is truncated, never rejected: by submit time the spend
   // is real, and failing the submit would lose both the work and the booking.
-  let stateUpdate = opts.stateUpdate;
-  let stateTruncated = false;
-  if (stateUpdate !== undefined) {
-    const bounded = boundedStateUpdate(stateUpdate);
-    stateUpdate = bounded.state;
-    stateTruncated = bounded.truncated;
-  }
+  // NOT bounded here any more: the update is merged into the existing working
+  // set inside the transaction (which needs the current row), so the size cap
+  // has to apply to the merged result, not to the fragment on its own.
+  const stateUpdate = opts.stateUpdate;
   // A non-terminal contribution returns the task to the pool, so `result` has
   // nowhere to live on the task row. Preserve it as the contribution's inline
   // artifact (unless the agent already supplied one) rather than dropping the
@@ -1719,11 +1791,33 @@ export async function submitResult(
 
     // 6. Refresh the target's compacted working set, if the agent supplied one
     //    (bounded above — an oversized update was truncated, not rejected).
-    if (stateUpdate !== undefined) {
+    //    Merged per key, not replaced: see mergeStateUpdate. The targets row is
+    //    locked FOR UPDATE first so two submits landing on the same conjecture
+    //    can't read-modify-write over each other — without it, merging would
+    //    reintroduce the very lost-update it exists to prevent.
+    let stateTruncated = false;
+    if (stateUpdate !== undefined && targetId) {
+      const cur = await client.query<{ state: unknown }>(
+        `SELECT state FROM targets WHERE id = $1 FOR UPDATE`,
+        [targetId],
+      );
+      const parsed = mergeStateUpdate(cur.rows[0]?.state, stateUpdate);
+      const bounded = boundedStateUpdate(parsed.state);
+      stateTruncated = bounded.truncated;
       await client.query(`UPDATE targets SET state = $2 WHERE id = $1`, [
         targetId,
-        JSON.stringify(stateUpdate),
+        JSON.stringify(bounded.state),
       ]);
+      // Facts are append-only and deduped by claim, so a replayed submit adds
+      // nothing and two agents establishing the same thing collapse to one row.
+      for (const claim of parsed.facts) {
+        await client.query(
+          `INSERT INTO target_facts (target_id, claim, established_by)
+           VALUES ($1, $2, $3)
+           ON CONFLICT (target_id, md5(claim)) DO NOTHING`,
+          [targetId, claim, contrib.rows[0].id],
+        );
+      }
     }
 
     // 7. Funnel: on THIS connection, under a savepoint — see checkoutTask. The
@@ -2446,6 +2540,43 @@ export interface TargetProgress {
   metrics: TargetProgressMetrics;
   /** The newest page of the feed; page the rest via listTargetContributions. */
   recent_contributions: TargetContribution[];
+  /**
+   * What this conjecture has actually established, oldest first. Append-only
+   * and out of `state`, so no later submit can overwrite it.
+   */
+  facts: TargetFact[];
+  /**
+   * What is left to do, DERIVED from the task graph on every read — never
+   * stored, so it cannot go stale. This is what a hand-maintained "NEXT: …"
+   * field in `state` was standing in for.
+   */
+  next_steps: TargetNextSteps;
+}
+
+export interface TargetFact {
+  id: number;
+  claim: string;
+  /** The contribution that established it, or null for an admin-seeded fact. */
+  established_by: number | null;
+  created_at: string;
+  /** Superseded rather than deleted; retracted facts stay on the record. */
+  retracted_at: string | null;
+  retracted_reason: string | null;
+}
+
+export interface TargetNextSteps {
+  /** Open tasks a volunteer could claim right now. */
+  claimable: { id: string; title: string; kind: string; max_cost_cents: number }[];
+  /** Submitted, waiting on verification or review — work in flight, not lost. */
+  awaiting_verification: number;
+  /**
+   * Tasks whose lock expired: claimed, never finished, back in the pool. A
+   * large number here is the signal that something is systematically failing
+   * (the C4 timeout pile), which no prose field would ever have surfaced.
+   */
+  expired: number;
+  /** Nothing claimable and nothing in flight — the target needs decomposition. */
+  stalled: boolean;
 }
 
 /** One row of a conjecture's public contribution feed. */
@@ -2529,6 +2660,7 @@ export async function getTargetProgress(slug: string): Promise<TargetProgress |
     [t.id],
   );
   const recent = await contributionPage(t.id, PROGRESS_FEED_PAGE, 0);
+  const [facts, nextSteps] = await Promise.all([listTargetFacts(t.id), deriveNextSteps(t.id)]);
   const mr = m.rows[0];
   return {
     slug: t.slug,
@@ -2552,6 +2684,73 @@ export async function getTargetProgress(slug: string): Promise<TargetProgress |
       last_activity_at: mr.last_activity_at ? new Date(mr.last_activity_at).toISOString() : null,
     },
     recent_contributions: recent,
+    facts,
+    next_steps: nextSteps,
+  };
+}
+
+/** A target's established facts, oldest first — retracted ones included, marked. */
+async function listTargetFacts(targetId: string): Promise<TargetFact[]> {
+  const { rows } = await query<{
+    id: string;
+    claim: string;
+    established_by: string | null;
+    created_at: string | Date;
+    retracted_at: string | Date | null;
+    retracted_reason: string | null;
+  }>(
+    `SELECT id, claim, established_by, created_at, retracted_at, retracted_reason
+       FROM target_facts WHERE target_id = $1 ORDER BY id`,
+    [targetId],
+  );
+  return rows.map((r) => ({
+    id: Number(r.id),
+    claim: r.claim,
+    established_by: r.established_by === null ? null : Number(r.established_by),
+    created_at: new Date(r.created_at).toISOString(),
+    retracted_at: r.retracted_at ? new Date(r.retracted_at).toISOString() : null,
+    retracted_reason: r.retracted_reason,
+  }));
+}
+
+/**
+ * What is left to do, computed from the task graph rather than stored.
+ *
+ * The whole reason a target ever carried a hand-written "NEXT: propose a
+ * decomposition of …" was that nothing derived it. A stored directive is stale
+ * the moment the work it describes lands, and then someone has to go and edit
+ * the field — forever, once per conjecture per turn of work. This cannot go
+ * stale, because there is nothing to keep up to date.
+ */
+async function deriveNextSteps(targetId: string): Promise<TargetNextSteps> {
+  const { rows } = await query<{
+    id: string;
+    title: string;
+    kind: string;
+    max_cost_cents: string;
+    status: string;
+  }>(
+    `SELECT id, title, kind::text AS kind, max_cost_cents, status::text AS status
+       FROM tasks
+      WHERE target_id = $1 AND status IN ('open', 'submitted', 'locked', 'expired')
+      ORDER BY max_cost_cents, created_at`,
+    [targetId],
+  );
+  const claimable = rows
+    .filter((r) => r.status === 'open')
+    .map((r) => ({
+      id: r.id,
+      title: r.title,
+      kind: r.kind,
+      max_cost_cents: Number(r.max_cost_cents),
+    }));
+  const awaiting = rows.filter((r) => r.status === 'submitted').length;
+  const inFlight = rows.filter((r) => r.status === 'locked').length;
+  return {
+    claimable,
+    awaiting_verification: awaiting,
+    expired: rows.filter((r) => r.status === 'expired').length,
+    stalled: claimable.length === 0 && awaiting === 0 && inFlight === 0,
   };
 }