DOS-825: reclaim wedged todo dispatches

packages/views/tsconfig.json

@@ -2,6 +2,7 @@
   "extends": "@multica/tsconfig/react-library.json",
   "compilerOptions": {
     "outDir": "dist",
+    "rootDir": ".",
     "paths": {
       "@claud-ometer/envelope": ["../../../Claud-ometer/src/lib/command/envelope.ts"]
     }

server/cmd/server/runtime_sweeper.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 	"time"
 
@@ -60,6 +61,17 @@ const (
 	// ticks and 500 rows/tick we drain 60k rows/hour worst case — plenty
 	// of headroom for the documented backlog without monopolising DB CPU.
 	queuedExpireBatchSize = 500
+	// todoDispatchReclaimBatchSize caps the recovery pass that re-enqueues
+	// todo issues whose prior task failed or vanished. The pass runs every
+	// sweeper tick, so a modest cap repairs wedges quickly without flooding
+	// daemons after an outage.
+	todoDispatchReclaimBatchSize = 50
+	// todoDispatchReclaimCooldownSeconds prevents infinite re-enqueue loops
+	// by waiting 15 minutes after a task failure before reclaiming it.
+	todoDispatchReclaimCooldownSeconds = 15 * 60.0
+	// todoDispatchReclaimMaxAttempts is the breaker limit. If an issue fails
+	// this many times, it stops being reclaimed and is marked blocked.
+	todoDispatchReclaimMaxAttempts = 5
 )
 
 // runRuntimeSweeper periodically marks runtimes as offline if their
@@ -85,6 +97,7 @@ func runRuntimeSweeper(ctx context.Context, queries *db.Queries, liveness handle
 			sweepStaleRuntimes(ctx, queries, liveness, taskSvc, bus)
 			sweepStaleTasks(ctx, queries, taskSvc, bus)
 			sweepExpiredQueuedTasks(ctx, queries, taskSvc)
+			sweepTodoDispatchReclaim(ctx, queries, taskSvc)
 			gcRuntimes(ctx, queries, bus)
 		}
 	}
@@ -285,6 +298,97 @@ func sweepExpiredQueuedTasks(ctx context.Context, queries *db.Queries, taskSvc *
 	taskSvc.HandleFailedTasks(ctx, failedTasks)
 }
 
+type dispatchReclaimStats struct {
+	Scanned   int
+	Reclaimed int
+	Routed    int
+	Failed    int
+	Skipped   map[string]int
+}
+
+func (s *dispatchReclaimStats) skip(reason string) {
+	if s.Skipped == nil {
+		s.Skipped = make(map[string]int)
+	}
+	s.Skipped[reason]++
+}
+
+// sweepTodoDispatchReclaim repairs todo issues whose dispatch path wedged after
+// a failed or missing task row. The candidate list is a bounded picker only; the
+// service revalidates and enqueues under an issue lock so recovery never cancels
+// a task that raced in after the sweep began.
+func sweepTodoDispatchReclaim(ctx context.Context, queries *db.Queries, taskSvc *service.TaskService) dispatchReclaimStats {
+	stats := dispatchReclaimStats{Skipped: make(map[string]int)}
+	if taskSvc == nil {
+		stats.skip("task_service_missing")
+		return stats
+	}
+
+	candidates, err := queries.ListTodoDispatchReclaimCandidates(ctx, db.ListTodoDispatchReclaimCandidatesParams{
+		MaxPerTick:   todoDispatchReclaimBatchSize,
+		CooldownSecs: todoDispatchReclaimCooldownSeconds,
+	})
+	if err != nil {
+		stats.Failed++
+		slog.Warn("todo dispatch reclaim: failed to list candidates", "error", err)
+		return stats
+	}
+	stats.Scanned = len(candidates)
+
+	for _, c := range candidates {
+		issueID := c.ID
+		issueKey := util.UUIDToString(issueID)
+		targetAgentID := util.UUIDToString(c.TargetAgentID)
+
+		if c.FailedTasksCount >= todoDispatchReclaimMaxAttempts {
+			stats.skip("breaker_tripped")
+			_, blockErr := queries.UpdateIssueStatus(ctx, db.UpdateIssueStatusParams{
+				ID:          issueID,
+				Status:      "blocked",
+				WorkspaceID: c.WorkspaceID,
+			})
+			if blockErr != nil {
+				slog.Warn("todo dispatch reclaim: failed to block issue", "issue_id", issueKey, "error", blockErr)
+			} else {
+				slog.Info("todo dispatch reclaim: issue tripped breaker and blocked", "issue_id", issueKey, "failed_tasks", c.FailedTasksCount)
+			}
+			continue
+		}
+
+		_, isSquadRoute, err := taskSvc.RecoverTodoDispatch(ctx, issueID, todoDispatchReclaimCooldownSeconds)
+		if errors.Is(err, service.ErrTodoDispatchReclaimNotEligible) {
+			stats.skip("candidate_changed")
+			continue
+		}
+		if err != nil {
+			stats.Failed++
+			slog.Warn("todo dispatch reclaim: recovery enqueue failed",
+				"issue_id", issueKey,
+				"target_agent_id", targetAgentID,
+				"is_squad_route", c.IsSquadRoute,
+				"error", err,
+			)
+			continue
+		}
+		if isSquadRoute {
+			stats.Routed++
+		} else {
+			stats.Reclaimed++
+		}
+	}
+
+	if stats.Scanned > 0 || stats.Reclaimed > 0 || stats.Routed > 0 || stats.Failed > 0 {
+		slog.Info("todo dispatch reclaim: completed",
+			"scanned", stats.Scanned,
+			"reclaimed", stats.Reclaimed,
+			"routed", stats.Routed,
+			"skipped", stats.Skipped,
+			"failed", stats.Failed,
+		)
+	}
+	return stats
+}
+
 // broadcastFailedTasks is preserved as a thin shim for the integration tests
 // in this package. New call sites should use TaskService.HandleFailedTasks
 // directly so the side effects (event broadcast, agent reconcile, issue