fix(backfill): gate the open-data crawl on its own segments, not a clock every job bumps

src/db/repositories.ts

@@ -5408,17 +5408,6 @@ export async function countRecentMergedPullRequests(env: Env, fullName: string):
   return Number(row?.count ?? 0);
 }
 
-export async function listContributorRecentMergedPullRequests(env: Env, login: string): Promise<RecentMergedPullRequestRecord[]> {
-  const db = getDb(env.DB);
-  const rows = await db
-    .select()
-    .from(recentMergedPullRequests)
-    .where(loginMatches(recentMergedPullRequests.authorLogin, login))
-    .orderBy(desc(recentMergedPullRequests.mergedAt))
-    .limit(1000);
-  return rows.map(toRecentMergedPullRequestRecord);
-}
-
 export async function upsertContributor(env: Env, contributor: ContributorRecord): Promise<void> {
   const db = getDb(env.DB);
   await db

src/github/backfill.ts

@@ -10,6 +10,7 @@ import {
   getPullRequestDetailSyncState,
   listRepoGithubTotalsSnapshotHistory,
   getRepoSyncSegment,
+  listRepoSyncSegments,
   getRepoSyncState,
   listOpenIssueNumbers,
   listOpenPullRequests,
@@ -327,23 +328,56 @@ const DEFAULT_LIMITS: BackfillLimits = {
 
 const FRESH_SYNC_MS = 6 * 60 * 60 * 1000;
 const ERROR_BACKOFF_MS = 60 * 60 * 1000;
+/** The segments the open-data fan-out dispatches -- and therefore the only ones whose freshness may gate it. */
+const OPEN_DATA_SEGMENTS: readonly BackfillSegmentName[] = ["labels", "open_issues", "open_pull_requests", "recent_merged_pull_requests"];
+const OPEN_DATA_BACKFILL_SKIPPED_METRIC = "loopover_github_open_data_backfill_skipped_total";
+
+/** Age of the OPEN-DATA crawl itself: the oldest completion across the four segments the fan-out below actually
+ *  dispatches. A segment that has never run (no row, or an unparseable completedAt) is infinitely old, so a repo
+ *  still owed its first crawl of any one segment is never treated as fresh. Measures ATTEMPTS, not successes --
+ *  a segment that ran minutes ago and came back `waiting_rate_limit` is not owed another attempt yet, which is
+ *  exactly the re-sync storm #4497 set out to stop. */
+function openDataCrawlAgeMs(segments: RepoSyncSegmentRecord[], nowMs: number): number {
+  const completedBySegment = new Map(segments.map((segment) => [segment.segment, segment.completedAt]));
+  let ageMs = 0;
+  for (const name of OPEN_DATA_SEGMENTS) {
+    const completedMs = Date.parse(completedBySegment.get(name) ?? "");
+    if (!Number.isFinite(completedMs)) return Number.POSITIVE_INFINITY;
+    ageMs = Math.max(ageMs, nowMs - completedMs);
+  }
+  return ageMs;
+}
 
 /** Shared freshness/error-backoff decision (#4497): a repo whose last sync is either a fresh success (within
  *  FRESH_SYNC_MS) or a recent error (within ERROR_BACKOFF_MS) should be skipped rather than re-synced, unless
  *  the caller explicitly forces a refresh. Returns null when a sync should proceed (never synced, no completed
  *  timestamp, or the existing sync is stale enough to redo). Shared by backfillRegisteredRepositories (the
  *  admin-endpoint/test path) and enqueueRepositoryOpenDataBackfill (the real scheduled-cron path) so both
  *  respect the SAME cadence -- previously only the former checked this, so the scheduled path re-synced every
- *  registered repo every 30 minutes forever regardless of freshness or a permanent error state. */
+ *  registered repo every 30 minutes forever regardless of freshness or a permanent error state.
+ *
+ *  The fresh-success window is measured on the open-data SEGMENTS' own completions, not on
+ *  repo_sync_state.lastCompletedAt (#10193). That column is a repo-wide clock that
+ *  refreshRepoSyncStateFromSegments rewrites at the end of EVERY segment write path, including two that run far
+ *  more often than FRESH_SYNC_MS and dispatch no open-data work: the ~2-minute re-gate sweep's own
+ *  open_pull_requests refresh (queue/processors.ts refreshOpenPullRequestsForScheduledSweep, which bypasses this
+ *  gate with force) and the backfill-pr-details follow-on it enqueues. Reading it here made the gate
+ *  self-perpetuating -- it never aged past the window, so labels/open_issues/recent_merged_pull_requests went
+ *  undispatched from the moment #4529 shipped. Only recent_merged_pull_requests showed it, because it is the one
+ *  open-data table with no webhook or sweep writer to mask the frozen crawl. The error-backoff window still reads
+ *  lastCompletedAt: a repo-level error state is repo-wide by nature. */
 function syncFreshnessSkipReason(
   syncState: RepoSyncStateRecord | null,
+  openDataSegments: RepoSyncSegmentRecord[],
   force: boolean | undefined,
 ): { freshSuccess: boolean; recentError: boolean } | null {
   if (force || !syncState?.lastCompletedAt || syncState.status === "never_synced") return null;
-  const ageMs = Date.now() - Date.parse(syncState.lastCompletedAt);
+  const nowMs = Date.now();
+  const stateAgeMs = nowMs - Date.parse(syncState.lastCompletedAt);
   const freshSuccess =
-    (syncState.status === "success" || syncState.status === "partial" || syncState.status === "capped") && Number.isFinite(ageMs) && ageMs < FRESH_SYNC_MS;
-  const recentError = syncState.status === "error" && Number.isFinite(ageMs) && ageMs < ERROR_BACKOFF_MS;
+    (syncState.status === "success" || syncState.status === "partial" || syncState.status === "capped") &&
+    openDataCrawlAgeMs(openDataSegments, nowMs) < FRESH_SYNC_MS;
+  const recentError = syncState.status === "error" && Number.isFinite(stateAgeMs) && stateAgeMs < ERROR_BACKOFF_MS;
   return freshSuccess || recentError ? { freshSuccess, recentError } : null;
 }
 const SEGMENT_PAGE_BUDGET: Record<BackfillMode, number> = { light: 2, full: 10, resume: 10 };
@@ -447,8 +481,8 @@ export async function backfillRegisteredRepositories(
         warnings,
       };
     }
-    const syncState = await getRepoSyncState(env, repo.fullName);
-    const skipReason = syncFreshnessSkipReason(syncState, options.force);
+    const [syncState, syncSegments] = await Promise.all([getRepoSyncState(env, repo.fullName), listRepoSyncSegments(env, repo.fullName)]);
+    const skipReason = syncFreshnessSkipReason(syncState, syncSegments, options.force);
     if (skipReason && syncState) {
       return {
         repoFullName: repo.fullName,
@@ -482,9 +516,22 @@ export async function enqueueRepositoryOpenDataBackfill(
   // gate -- this is the path the real scheduled cron actually dispatches through (see that function's own
   // routing comment), which previously had NO freshness/error-backoff check at all and re-synced every
   // registered repo every 30 minutes forever, backing off neither for a fresh success nor a permanent error.
-  const previous = await getRepoSyncState(env, repo.fullName);
-  const skipReason = syncFreshnessSkipReason(previous, options.force);
+  const [previous, syncSegments] = await Promise.all([getRepoSyncState(env, repo.fullName), listRepoSyncSegments(env, repo.fullName)]);
+  const skipReason = syncFreshnessSkipReason(previous, syncSegments, options.force);
   if (skipReason && previous) {
+    // #10193: the skip used to be reported ONLY as a warnings[] string, which this function's cron caller
+    // (queue/job-dispatch.ts's backfill-registered-repos case) discards -- so a repo whose open-data crawl was
+    // skipped on every 30-minute tick for three weeks left no log line, metric, or audit trail anywhere, and the
+    // dead crawl was found only by noticing the table it feeds had stopped growing. A starved crawl is now visible.
+    incr(OPEN_DATA_BACKFILL_SKIPPED_METRIC, { reason: skipReason.freshSuccess ? "fresh_success" : "recent_error" });
+    console.log(
+      JSON.stringify({
+        event: "open_data_backfill_skipped",
+        repoFullName: repo.fullName,
+        reason: skipReason.freshSuccess ? "fresh_success" : "recent_error",
+        openDataCrawlAgeMs: openDataCrawlAgeMs(syncSegments, Date.now()),
+      }),
+    );
     return {
       ok: true,
       repoFullName: repo.fullName,
@@ -518,9 +565,8 @@ export async function enqueueRepositoryOpenDataBackfill(
     lastCompletedAt: previous?.lastCompletedAt,
     warnings: previous?.warnings ?? [],
   });
-  const segments: BackfillSegmentName[] = ["labels", "open_issues", "open_pull_requests", "recent_merged_pull_requests"];
   await Promise.all(
-    segments.map((segment, index) =>
+    OPEN_DATA_SEGMENTS.map((segment, index) =>
       env.JOBS.send(
         { type: "backfill-repo-segment", requestedBy: options.requestedBy, repoFullName: repo.fullName, ...repoInstallationPayload(repo), segment, mode, ...(options.force === undefined ? {} : { force: options.force }) },
         { delaySeconds: index * 15 },

src/selfhost/metrics.ts

@@ -147,6 +147,13 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["loopover_review_end_to_end_latency_seconds", { help: "Real end-to-end review latency in seconds, from the PR's current head SHA becoming ready for review (open + non-draft) to this pass's comment publish -- distinct from a single queue job's own claim-to-completion latency_ms, this spans every queueing/deferral wait in between.", type: "histogram" }],
   ["loopover_github_branch_protection_permission_denied_total", { help: "GitHub branch-protection reads denied by permissions.", type: "counter" }],
   ["loopover_github_pull_request_files_fetch_total", { help: "GitHub pull-request file fetch attempts.", type: "counter" }],
+  [
+    "loopover_github_open_data_backfill_skipped_total",
+    {
+      help: "Scheduled open-data backfills skipped by the freshness/error-backoff gate, by reason (fresh_success/recent_error). A repo whose crawl is genuinely starved shows a sustained fresh_success rate with no matching segment progress (#10193).",
+      type: "counter",
+    },
+  ],
   ["loopover_pr_state_cache_total", { help: "Pull-request state cache outcomes.", type: "counter" }],
   ["loopover_ci_state_cache_total", { help: "CI-state snapshot cache outcomes.", type: "counter" }],
   ["loopover_ops_anomaly_total", { help: "Ops anomaly scan detections (review burst / review failure burst), by repo and kind.", type: "counter" }],