fix(hub): orphaned agents on a dead host no longer block its deletion

docs/reference/api-overview.md

@@ -99,7 +99,7 @@ Detail: [`hub-agents.md`](hub-agents.md), [ADR-003](../decisions/003-a2a-relay-r
 |---|---|---|
 | POST | `/v1/teams/{team}/hosts` | Register a host (host-runner) |
 | GET | `/v1/teams/{team}/hosts` | List hosts |
-| GET / DELETE | `/v1/teams/{team}/hosts/{host}` | Get / delete host |
+| GET / DELETE | `/v1/teams/{team}/hosts/{host}` | Get / delete host (delete reaps orphaned agents when the host is offline; 409 when it is online with live agents) |
 | POST | `/v1/teams/{team}/hosts/{host}/heartbeat` | Heartbeat with build metadata + capabilities |
 | GET | `/v1/teams/{team}/hosts/{host}/commands` | Pull queued out-of-band commands |
 | PATCH | `/v1/teams/{team}/hosts/{host}/ssh_hint` | Update non-secret SSH hint |

docs/reference/openapi.yaml

@@ -781,9 +781,19 @@ paths:
       tags: [Hosts]
       operationId: deleteHost
       summary: Delete a host (admin)
+      description: >-
+        Deletes the host row. Agents still assigned to it are handled by
+        whether the host is reachable: if the host is `online` with live
+        agents the delete is refused with 409 (its host-runner can stop them
+        properly first); if the host is `offline` those agents are orphaned by
+        definition, so they are reaped to `crashed` — sessions paused, bearer
+        tokens revoked — and the delete proceeds. Agents already in a terminal
+        status (`terminated`, `crashed`, `failed`) never block a delete.
       responses:
         "204": { description: Deleted }
         "404": { $ref: "#/components/responses/NotFound" }
+        "409":
+          description: Host is online and still has live agents
 
   /v1/teams/{team}/hosts/{host}/heartbeat:
     parameters:

hub/internal/server/agent_status.go

@@ -0,0 +1,46 @@
+package server
+
+// Agent terminal statuses — ONE definition, because this predicate decides
+// whether an agent still counts as alive and had drifted into two spellings.
+//
+// The lifecycle (docs/reference/glossary.md, ADR-009) is
+//
+//	pending → running → terminated | crashed | failed → archived
+//
+// so all three of terminated/crashed/failed are ends of the line:
+//
+//   - terminated — the operator stopped it (or a swap replaced it);
+//   - crashed    — the process died under it (the host-runner's reconcile
+//     loop reports this when a pane is gone or a driver is missing);
+//   - failed     — it stopped itself with an error.
+//
+// They differ only in *why* the process ended, never in *whether* it did, so
+// any query asking "is this agent still alive" must treat them identically.
+// Before this constant existed, handleDeleteHost listed only two of the
+// three, which made a crashed agent — a corpse by every other query's
+// reckoning — permanently block deletion of the host it died on.
+//
+// archived_at is a separate axis: archiving is only reachable from a
+// terminal status (handleArchiveAgent), so a live-agent check never needs it.
+const (
+	// sqlAgentNotTerminal is the WHERE-clause fragment for "this agent is
+	// still alive". Concatenate it into a query; it names no bind
+	// parameters and no table qualifier, so it composes anywhere `agents`
+	// is the only table in scope.
+	sqlAgentNotTerminal = `status NOT IN ('terminated','crashed','failed')`
+)
+
+// agentTerminalStatuses is the same set as Go values, for callers deciding in
+// code rather than in SQL. Keep in lockstep with sqlAgentNotTerminal —
+// TestAgentTerminalStatusesAgreeWithSQL asserts they do.
+var agentTerminalStatuses = []string{"terminated", "crashed", "failed"}
+
+// isAgentTerminal reports whether an agent status means the process is over.
+func isAgentTerminal(status string) bool {
+	for _, s := range agentTerminalStatuses {
+		if status == s {
+			return true
+		}
+	}
+	return false
+}

hub/internal/server/handlers_agents.go

@@ -151,7 +151,7 @@ func (s *Server) handleListAgents(w http.ResponseWriter, r *http.Request) {
 	case live:
 		q += " AND status IN ('running','idle','paused')"
 	case !includeTerminated:
-		q += " AND status NOT IN ('terminated','failed','crashed')"
+		q += " AND " + sqlAgentNotTerminal
 	}
 	if pid := r.URL.Query().Get("project_id"); pid != "" {
 		q += " AND project_id = ?"
@@ -397,20 +397,7 @@ func (s *Server) handlePatchAgent(w http.ResponseWriter, r *http.Request) {
 		s.applyAgentTerminationEffects(r.Context(), team, id, "agent stopped via PATCH", false)
 	} else if in.Status != nil &&
 		(*in.Status == "crashed" || *in.Status == "failed") {
-		_, _ = s.writeDB.ExecContext(r.Context(), `
-			UPDATE sessions
-			   SET status = 'paused', last_active_at = ?
-			 WHERE team_id = ?
-			   AND current_agent_id = ?
-			   AND status = 'active'`,
-			NowUTC(), team, id)
-		_, _ = auth.RevokeAgentTokens(r.Context(), s.writeDB, id, NowUTC())
-		// Fold + stamp the run digest now (#118 §4). The operator stop path
-		// finalizes via stopSessionInternal; a crash/failure flows through
-		// here instead, so without this the first Insight open after the crash
-		// pays the full O(n) backfill. finalizeDigestOutcome brings the digest
-		// current off the read path.
-		s.finalizeDigestOutcome(r.Context(), team, id)
+		s.applyAgentCrashEffects(r.Context(), team, id)
 	}
 	// ADR-029 D-3: auto-derive the linked task's status from the
 	// agent's terminal transition. Most-recent-spawn drives; older
@@ -466,6 +453,87 @@ func (s *Server) applyAgentTerminationEffects(ctx context.Context, team, id, rea
 	s.finalizeDigestOutcome(ctx, team, id)
 }
 
+// applyAgentCrashEffects runs the side-effects of an agent reaching a
+// terminal status it did NOT choose — 'crashed' or 'failed'. Unlike the
+// operator's stop (applyAgentTerminationEffects) there is no host command to
+// enqueue and no agent.terminate audit row: nobody asked for this, the
+// process simply ended. What remains is cleaning up after it — pause the
+// session that now points at nothing, revoke the bearer the dead process
+// held, and seal the run digest so the next Insight open doesn't pay an O(n)
+// backfill (#118 §4).
+//
+// Extracted from handlePatchAgent so the orphan reaper produces the identical
+// aftermath; a second copy would drift, and a crash whose tokens outlive it
+// is a security bug, not a cosmetic one. The caller owns flipping
+// agents.status and calling deriveTaskStatusFromAgent (ADR-029 D-3).
+func (s *Server) applyAgentCrashEffects(ctx context.Context, team, id string) {
+	_, _ = s.writeDB.ExecContext(ctx, `
+		UPDATE sessions
+		   SET status = 'paused', last_active_at = ?
+		 WHERE team_id = ?
+		   AND current_agent_id = ?
+		   AND status = 'active'`,
+		NowUTC(), team, id)
+	_, _ = auth.RevokeAgentTokens(ctx, s.writeDB, id, NowUTC())
+	s.finalizeDigestOutcome(ctx, team, id)
+}
+
+// liveAgent is the minimum an orphan sweep needs: who to reap, and what to
+// call them in the audit record.
+type liveAgent struct {
+	ID     string
+	Handle string
+}
+
+// liveAgentsOnHost lists the agents assigned to a host whose process has not
+// ended. Used by the host-delete guard to decide between refusing and
+// reaping.
+func (s *Server) liveAgentsOnHost(ctx context.Context, team, host string) ([]liveAgent, error) {
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT id, handle FROM agents
+		 WHERE team_id = ? AND host_id = ?
+		   AND `+sqlAgentNotTerminal, team, host)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []liveAgent
+	for rows.Next() {
+		var a liveAgent
+		if err := rows.Scan(&a.ID, &a.Handle); err != nil {
+			return nil, err
+		}
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+
+// reapOrphanedAgent flips one agent to 'crashed' and runs the crash
+// aftermath. Called when the host an agent lived on is being deleted while
+// offline: the process is unreachable by construction, so leaving the row
+// claiming 'running' would assert something the hub cannot possibly know.
+//
+// 'crashed' rather than 'terminated' on purpose — nobody stopped this agent,
+// its floor fell away. The distinction is load-bearing downstream: ADR-029
+// D-3 derives task.status='blocked' from a crash (work interrupted, worth
+// retrying) but 'cancelled' from a terminate with no result (work abandoned).
+func (s *Server) reapOrphanedAgent(ctx context.Context, team, id string) error {
+	res, err := s.writeDB.ExecContext(ctx, `
+		UPDATE agents SET status = 'crashed', terminated_at = ?
+		 WHERE team_id = ? AND id = ?
+		   AND `+sqlAgentNotTerminal, NowUTC(), team, id)
+	if err != nil {
+		return err
+	}
+	// Lost a race with a real termination — the aftermath already ran.
+	if n, _ := res.RowsAffected(); n == 0 {
+		return nil
+	}
+	s.applyAgentCrashEffects(ctx, team, id)
+	_ = s.deriveTaskStatusFromAgent(ctx, team, id, "crashed")
+	return nil
+}
+
 // handleStopAgent is POST /v1/teams/{team}/agents/{agent}/stop — the
 // RESUMABLE kill. Kills the agent and flips its session to `paused`, so
 // `agents.resume` can respawn it. The principal's "Stop session" does
@@ -1484,7 +1552,7 @@ func (s *Server) DoSpawn(ctx context.Context, team string, in spawnIn) (spawnOut
 			UPDATE agents
 			   SET status = 'terminated', terminated_at = ?
 			 WHERE team_id = ? AND id = ?
-			   AND status NOT IN ('terminated','failed','crashed')`,
+			   AND `+sqlAgentNotTerminal,
 			now, team, priorAgentID); err != nil {
 			return spawnOut{}, http.StatusInternalServerError, err
 		}

hub/internal/server/handlers_general_steward.go

@@ -185,7 +185,7 @@ func (s *Server) findRunningGeneralSteward(ctx context.Context, team string) (st
 	err := s.db.QueryRowContext(ctx, `
 		SELECT id FROM agents
 		WHERE team_id = ? AND kind = ?
-		  AND status NOT IN ('terminated','crashed','failed')
+		  AND `+sqlAgentNotTerminal+`
 		  AND terminated_at IS NULL
 		ORDER BY created_at DESC
 		LIMIT 1`,