feat(F192): evidence-source prerequisite gate for eval scheduler

docs/harness-feedback/eval-domains/eval-a2a.yaml

@@ -502,7 +502,7 @@ threadPolicy:
 legacyScheduledTaskIds: [] # Cleaned 2026-06-01 — see migrations/2026-06-01-eval-a2a-legacy-task-cleanup.md
 handoffTargetResolver:
   featureId: F167
-  ownerCatId: opus-47
+  ownerCatId: opus
   threadLookup: feature-thread
 sla:
   acknowledgeHours: 24

packages/api/src/index.ts

@@ -5910,6 +5910,9 @@ async function main(): Promise<void> {
   );
   // N-day factory is in its own module (split from eval-domain-daily for file-size limit)
   const { createEvalDomainNDaySpec } = await import('./infrastructure/harness-eval/domain/eval-domain-nday.js');
+  const { createTelemetryEvidencePrereqProbe } = await import(
+    './infrastructure/harness-eval/domain/eval-domain-evidence-gate.js'
+  );
   const { getOwnerUserId } = await import('./config/cat-config-loader.js');
   // cloud R6 P2 (PR-2) + memory wire-up: mirror the same wired set the
   // eval-hub.ts route computes (Object.keys(verdictGenerators)). Bootstrap-time
@@ -5978,6 +5981,10 @@ async function main(): Promise<void> {
     return ok;
   };
 
+  const evidencePrereqProbe = createTelemetryEvidencePrereqProbe({
+    otelEnabled: () => telemetryHandle.getMetricsText !== null,
+  });
+
   const evalScheduleOpts = {
     harnessFeedbackRoot: resolve(repoRoot, 'docs', 'harness-feedback'),
     threadStore,
@@ -5986,6 +5993,7 @@ async function main(): Promise<void> {
     redis: redisClient ?? undefined,
     wiredPublishDomains,
     publishPrereqProbe,
+    evidencePrereqProbe,
   };
   taskRunnerV2.register(createEvalDomainDailySpec(evalScheduleOpts));
   taskRunnerV2.register(createEvalDomainWeeklySpec(evalScheduleOpts));

packages/api/src/infrastructure/harness-eval/domain/eval-domain-daily.ts

@@ -18,6 +18,11 @@ import type { TaskSpec_P1 } from '../../scheduler/types.js';
 import { buildEvalCatInvocation } from '../eval-cat-invocation.js';
 import { ensureEvalDomainThreads } from '../hub/eval-hub-thread-ensure.js';
 import { inventoryLegacyTasks, type LegacyScheduledTaskLike } from '../legacy-task-cleanup.js';
+import {
+  buildEvidencePrereqSkippedMessage,
+  type EvidencePrereqProbe,
+  evaluateEvidencePrereq,
+} from './eval-domain-evidence-gate.js';
 import { getEvalCatOverride } from './eval-domain-override.js';
 import {
   type EvalDomainRegistryEntry,
@@ -42,6 +47,15 @@ export interface EvalDomainScheduleOpts {
    * → legacy default (all known-wireable domains get publish instructions in invocation).
    */
   wiredPublishDomains?: ReadonlySet<EvalDomainRegistryEntry['domainId']>;
+  /**
+   * Pre-invocation evidence-source prerequisite probe.
+   *
+   * This runs before publishPrereqProbe because evidence production is upstream
+   * of verdict publishing. If the source adapter cannot produce fresh evidence,
+   * the scheduler posts a skip notice to the domain thread and does not invoke
+   * the eval cat.
+   */
+  evidencePrereqProbe?: EvidencePrereqProbe;
   /**
    * Direction B (clowder-ai#923 fix): pre-invocation prerequisite probe.
    *
@@ -197,6 +211,20 @@ function createEvalDomainSpec(config: EvalDomainSpecConfig): TaskSpec_P1<EvalDom
           );
         }
 
+        if (config.evidencePrereqProbe) {
+          const evidencePrereq = await evaluateEvidencePrereq(config.evidencePrereqProbe, domain);
+          if (!evidencePrereq.ok) {
+            if (ctx.deliver) {
+              await ctx.deliver({
+                threadId: domain.systemThreadId,
+                content: buildEvidencePrereqSkippedMessage(domain, evidencePrereq.reason),
+                userId: 'scheduler',
+              });
+            }
+            return;
+          }
+        }
+
         // Direction B (clowder-ai#923 fix): publish-prereq gate.
         // Runs BEFORE legacy-gate / override / buildEvalCatInvocation so that a runtime
         // missing publish-verdict prerequisites never invokes the eval cat. The cat

packages/api/src/infrastructure/harness-eval/domain/eval-domain-evidence-gate.ts

@@ -0,0 +1,104 @@
+/**
+ * Evidence-source prerequisite gate for scheduled eval domains.
+ *
+ * Verdict provenance: eval:a2a build verdict
+ * `2026-07-07-eval-a2a-reeval-telemetry-still-disabled-build` (PR #19).
+ *
+ * Direction B's `publishPrereqProbe` (eval-domain-daily.ts) answers "can this
+ * runtime ACCEPT a published verdict?". This gate answers the upstream
+ * question: "can the domain's evidence source PRODUCE evidence at all?"
+ *
+ * eval:a2a consumes `f167-runtime-eval` artifacts derived from live OTel
+ * telemetry. When `TELEMETRY_HMAC_SALT` is unset in a non-dev environment,
+ * `initTelemetry()` disables OTel at boot — no fresh snapshots can exist, and
+ * invoking the eval cat burns a full LLM session to re-conclude "telemetry
+ * still disabled" (the 2026-06-30 → 2026-07-07 series produced daily
+ * near-identical verdicts). Failing closed BEFORE invocation posts a
+ * zero-LLM-cost skip notice to the domain's own system thread instead,
+ * keeping the gap visible without the burn.
+ *
+ * OTel init state is fixed for the process lifetime (the salt is read at
+ * boot), so a boolean thunk wired from bootstrap is a complete input — no
+ * re-probing or caching needed.
+ */
+
+import type { EvalDomainRegistryEntry } from './eval-domain-registry.js';
+
+/** Minimal domain projection the gate needs — keeps probes trivial to test. */
+export type EvidenceGateDomain = Pick<EvalDomainRegistryEntry, 'domainId' | 'sourceAdapter'>;
+
+export type EvidencePrereqResult = { ok: true } | { ok: false; reason: string };
+
+export type EvidencePrereqProbe = (domain: EvidenceGateDomain) => EvidencePrereqResult | Promise<EvidencePrereqResult>;
+
+/**
+ * Source adapters whose evidence pipeline hard-requires live OTel telemetry.
+ * Registry `sourceAdapter` is a free slug (see eval-domain-registry.ts), so
+ * the adapter → prerequisite mapping lives here, next to the probe.
+ */
+const TELEMETRY_BACKED_ADAPTERS: ReadonlySet<string> = new Set(['f167-runtime-eval']);
+
+export function isTelemetryBackedAdapter(sourceAdapter: string): boolean {
+  return TELEMETRY_BACKED_ADAPTERS.has(sourceAdapter);
+}
+
+/**
+ * Probe factory. Bootstrap wires `otelEnabled: () => !!telemetryHandle.getMetricsText`
+ * — the same init-state signal `GET /api/telemetry/health` reports as
+ * `otelEnabled` (routes/telemetry.ts Phase K note: actual init state, not an
+ * env-var proxy). Non-telemetry-backed adapters always pass through.
+ */
+export function createTelemetryEvidencePrereqProbe(opts: {
+  otelEnabled: () => boolean;
+  /** Override the reason text; defaults to the health route's disabledReason derivation. */
+  disabledReason?: () => string;
+}): EvidencePrereqProbe {
+  return (domain) => {
+    if (!isTelemetryBackedAdapter(domain.sourceAdapter)) return { ok: true };
+    if (opts.otelEnabled()) return { ok: true };
+    const reason =
+      opts.disabledReason?.() ??
+      (process.env.OTEL_SDK_DISABLED === 'true'
+        ? 'OTel disabled by OTEL_SDK_DISABLED=true'
+        : 'OTel disabled at boot: HMAC salt validation failed (TELEMETRY_HMAC_SALT not configured)');
+    return { ok: false, reason };
+  };
+}
+
+/** Fail-closed evaluation: a probe that throws is treated as "evidence unavailable". */
+export async function evaluateEvidencePrereq(
+  probe: EvidencePrereqProbe,
+  domain: EvidenceGateDomain,
+): Promise<EvidencePrereqResult> {
+  try {
+    return await Promise.resolve(probe(domain));
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return { ok: false, reason: `evidence prereq probe threw: ${message}` };
+  }
+}
+
+/**
+ * Stable-header skip notice posted to the domain's OWN system thread when the
+ * cron fails closed. Header format mirrors `buildPublishPrereqSkippedMessage`
+ * so eval-domain readers / log scrubbers can grep both skip classes uniformly.
+ */
+export function buildEvidencePrereqSkippedMessage(domain: EvidenceGateDomain, reason: string): string {
+  return [
+    `## Eval Domain: ${domain.domainId} — SKIPPED (evidence source unavailable)`,
+    '',
+    "The scheduled eval was skipped because this domain's evidence source",
+    `(\`${domain.sourceAdapter}\`) cannot produce evidence on this runtime:`,
+    '',
+    `> ${reason}`,
+    '',
+    'Why this matters: invoking the eval cat without a live evidence source',
+    'burns a full LLM session to re-conclude the same telemetry gap every fire',
+    '(see the eval:a2a 2026-06-30 → 2026-07-07 verdict series). The fail-closed',
+    'skip keeps the gap visible in this thread at zero LLM cost.',
+    '',
+    'Next action: configure a non-empty `TELEMETRY_HMAC_SALT` for the API',
+    'runtime and restart it (OTel initializes at boot), or set `enabled: false`',
+    "in this domain's registry YAML to pause the schedule intentionally.",
+  ].join('\n');
+}

packages/api/src/infrastructure/harness-eval/domain/eval-domain-nday.ts

@@ -20,6 +20,7 @@ import {
   type EvalDomainScheduleOpts,
   evaluatePublishPrereq,
 } from './eval-domain-daily.js';
+import { buildEvidencePrereqSkippedMessage, evaluateEvidencePrereq } from './eval-domain-evidence-gate.js';
 import { getEvalCatOverride } from './eval-domain-override.js';
 import {
   type EvalDomainRegistryEntry,
@@ -155,6 +156,20 @@ export function createEvalDomainNDaySpec(opts: EvalDomainScheduleOpts): TaskSpec
           );
         }
 
+        if (opts.evidencePrereqProbe) {
+          const evidencePrereq = await evaluateEvidencePrereq(opts.evidencePrereqProbe, domain);
+          if (!evidencePrereq.ok) {
+            if (ctx.deliver) {
+              await ctx.deliver({
+                threadId: domain.systemThreadId,
+                content: buildEvidencePrereqSkippedMessage(domain, evidencePrereq.reason),
+                userId: 'scheduler',
+              });
+            }
+            return;
+          }
+        }
+
         // Direction B publish-prereq gate (same as daily/weekly spec)
         if (opts.publishPrereqProbe) {
           const prereqOk = await evaluatePublishPrereq(opts.publishPrereqProbe, domain.domainId);