diff --git a/docs/harness-feedback/eval-domains/eval-a2a.yaml b/docs/harness-feedback/eval-domains/eval-a2a.yaml index 6ff4c15b62..606c9e4f13 100644 --- a/docs/harness-feedback/eval-domains/eval-a2a.yaml +++ b/docs/harness-feedback/eval-domains/eval-a2a.yaml @@ -502,7 +502,7 @@ threadPolicy: legacyScheduledTaskIds: [] # Cleaned 2026-06-01 — see migrations/2026-06-01-eval-a2a-legacy-task-cleanup.md handoffTargetResolver: featureId: F167 - ownerCatId: opus-47 + ownerCatId: opus threadLookup: feature-thread sla: acknowledgeHours: 24 diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 96e351dffe..802a79426e 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -5910,6 +5910,9 @@ async function main(): Promise { ); // N-day factory is in its own module (split from eval-domain-daily for file-size limit) const { createEvalDomainNDaySpec } = await import('./infrastructure/harness-eval/domain/eval-domain-nday.js'); + const { createTelemetryEvidencePrereqProbe } = await import( + './infrastructure/harness-eval/domain/eval-domain-evidence-gate.js' + ); const { getOwnerUserId } = await import('./config/cat-config-loader.js'); // cloud R6 P2 (PR-2) + memory wire-up: mirror the same wired set the // eval-hub.ts route computes (Object.keys(verdictGenerators)). Bootstrap-time @@ -5978,6 +5981,10 @@ async function main(): Promise { return ok; }; + const evidencePrereqProbe = createTelemetryEvidencePrereqProbe({ + otelEnabled: () => telemetryHandle.getMetricsText !== null, + }); + const evalScheduleOpts = { harnessFeedbackRoot: resolve(repoRoot, 'docs', 'harness-feedback'), threadStore, @@ -5986,6 +5993,7 @@ async function main(): Promise { redis: redisClient ?? undefined, wiredPublishDomains, publishPrereqProbe, + evidencePrereqProbe, }; taskRunnerV2.register(createEvalDomainDailySpec(evalScheduleOpts)); taskRunnerV2.register(createEvalDomainWeeklySpec(evalScheduleOpts)); diff --git a/packages/api/src/infrastructure/harness-eval/domain/eval-domain-daily.ts b/packages/api/src/infrastructure/harness-eval/domain/eval-domain-daily.ts index 612b4b1e4c..d6da14fbc2 100644 --- a/packages/api/src/infrastructure/harness-eval/domain/eval-domain-daily.ts +++ b/packages/api/src/infrastructure/harness-eval/domain/eval-domain-daily.ts @@ -18,6 +18,11 @@ import type { TaskSpec_P1 } from '../../scheduler/types.js'; import { buildEvalCatInvocation } from '../eval-cat-invocation.js'; import { ensureEvalDomainThreads } from '../hub/eval-hub-thread-ensure.js'; import { inventoryLegacyTasks, type LegacyScheduledTaskLike } from '../legacy-task-cleanup.js'; +import { + buildEvidencePrereqSkippedMessage, + type EvidencePrereqProbe, + evaluateEvidencePrereq, +} from './eval-domain-evidence-gate.js'; import { getEvalCatOverride } from './eval-domain-override.js'; import { type EvalDomainRegistryEntry, @@ -42,6 +47,15 @@ export interface EvalDomainScheduleOpts { * → legacy default (all known-wireable domains get publish instructions in invocation). */ wiredPublishDomains?: ReadonlySet; + /** + * Pre-invocation evidence-source prerequisite probe. + * + * This runs before publishPrereqProbe because evidence production is upstream + * of verdict publishing. If the source adapter cannot produce fresh evidence, + * the scheduler posts a skip notice to the domain thread and does not invoke + * the eval cat. + */ + evidencePrereqProbe?: EvidencePrereqProbe; /** * Direction B (clowder-ai#923 fix): pre-invocation prerequisite probe. * @@ -197,6 +211,20 @@ function createEvalDomainSpec(config: EvalDomainSpecConfig): TaskSpec_P1; + +export type EvidencePrereqResult = { ok: true } | { ok: false; reason: string }; + +export type EvidencePrereqProbe = (domain: EvidenceGateDomain) => EvidencePrereqResult | Promise; + +/** + * Source adapters whose evidence pipeline hard-requires live OTel telemetry. + * Registry `sourceAdapter` is a free slug (see eval-domain-registry.ts), so + * the adapter → prerequisite mapping lives here, next to the probe. + */ +const TELEMETRY_BACKED_ADAPTERS: ReadonlySet = new Set(['f167-runtime-eval']); + +export function isTelemetryBackedAdapter(sourceAdapter: string): boolean { + return TELEMETRY_BACKED_ADAPTERS.has(sourceAdapter); +} + +/** + * Probe factory. Bootstrap wires `otelEnabled: () => !!telemetryHandle.getMetricsText` + * — the same init-state signal `GET /api/telemetry/health` reports as + * `otelEnabled` (routes/telemetry.ts Phase K note: actual init state, not an + * env-var proxy). Non-telemetry-backed adapters always pass through. + */ +export function createTelemetryEvidencePrereqProbe(opts: { + otelEnabled: () => boolean; + /** Override the reason text; defaults to the health route's disabledReason derivation. */ + disabledReason?: () => string; +}): EvidencePrereqProbe { + return (domain) => { + if (!isTelemetryBackedAdapter(domain.sourceAdapter)) return { ok: true }; + if (opts.otelEnabled()) return { ok: true }; + const reason = + opts.disabledReason?.() ?? + (process.env.OTEL_SDK_DISABLED === 'true' + ? 'OTel disabled by OTEL_SDK_DISABLED=true' + : 'OTel disabled at boot: HMAC salt validation failed (TELEMETRY_HMAC_SALT not configured)'); + return { ok: false, reason }; + }; +} + +/** Fail-closed evaluation: a probe that throws is treated as "evidence unavailable". */ +export async function evaluateEvidencePrereq( + probe: EvidencePrereqProbe, + domain: EvidenceGateDomain, +): Promise { + try { + return await Promise.resolve(probe(domain)); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { ok: false, reason: `evidence prereq probe threw: ${message}` }; + } +} + +/** + * Stable-header skip notice posted to the domain's OWN system thread when the + * cron fails closed. Header format mirrors `buildPublishPrereqSkippedMessage` + * so eval-domain readers / log scrubbers can grep both skip classes uniformly. + */ +export function buildEvidencePrereqSkippedMessage(domain: EvidenceGateDomain, reason: string): string { + return [ + `## Eval Domain: ${domain.domainId} — SKIPPED (evidence source unavailable)`, + '', + "The scheduled eval was skipped because this domain's evidence source", + `(\`${domain.sourceAdapter}\`) cannot produce evidence on this runtime:`, + '', + `> ${reason}`, + '', + 'Why this matters: invoking the eval cat without a live evidence source', + 'burns a full LLM session to re-conclude the same telemetry gap every fire', + '(see the eval:a2a 2026-06-30 → 2026-07-07 verdict series). The fail-closed', + 'skip keeps the gap visible in this thread at zero LLM cost.', + '', + 'Next action: configure a non-empty `TELEMETRY_HMAC_SALT` for the API', + 'runtime and restart it (OTel initializes at boot), or set `enabled: false`', + "in this domain's registry YAML to pause the schedule intentionally.", + ].join('\n'); +} diff --git a/packages/api/src/infrastructure/harness-eval/domain/eval-domain-nday.ts b/packages/api/src/infrastructure/harness-eval/domain/eval-domain-nday.ts index dd1627609d..5e87dfb881 100644 --- a/packages/api/src/infrastructure/harness-eval/domain/eval-domain-nday.ts +++ b/packages/api/src/infrastructure/harness-eval/domain/eval-domain-nday.ts @@ -20,6 +20,7 @@ import { type EvalDomainScheduleOpts, evaluatePublishPrereq, } from './eval-domain-daily.js'; +import { buildEvidencePrereqSkippedMessage, evaluateEvidencePrereq } from './eval-domain-evidence-gate.js'; import { getEvalCatOverride } from './eval-domain-override.js'; import { type EvalDomainRegistryEntry, @@ -155,6 +156,20 @@ export function createEvalDomainNDaySpec(opts: EvalDomainScheduleOpts): TaskSpec ); } + if (opts.evidencePrereqProbe) { + const evidencePrereq = await evaluateEvidencePrereq(opts.evidencePrereqProbe, domain); + if (!evidencePrereq.ok) { + if (ctx.deliver) { + await ctx.deliver({ + threadId: domain.systemThreadId, + content: buildEvidencePrereqSkippedMessage(domain, evidencePrereq.reason), + userId: 'scheduler', + }); + } + return; + } + } + // Direction B publish-prereq gate (same as daily/weekly spec) if (opts.publishPrereqProbe) { const prereqOk = await evaluatePublishPrereq(opts.publishPrereqProbe, domain.domainId); diff --git a/packages/api/test/harness-eval/eval-domain-evidence-gate.test.js b/packages/api/test/harness-eval/eval-domain-evidence-gate.test.js new file mode 100644 index 0000000000..069a965707 --- /dev/null +++ b/packages/api/test/harness-eval/eval-domain-evidence-gate.test.js @@ -0,0 +1,234 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { describe, it, mock } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { createEvalDomainDailySpec } from '../../dist/infrastructure/harness-eval/domain/eval-domain-daily.js'; +import { + createTelemetryEvidencePrereqProbe, + evaluateEvidencePrereq, +} from '../../dist/infrastructure/harness-eval/domain/eval-domain-evidence-gate.js'; +import { createEvalDomainNDaySpec } from '../../dist/infrastructure/harness-eval/domain/eval-domain-nday.js'; + +const repoHarnessFeedbackRoot = fileURLToPath(new URL('../../../../docs/harness-feedback', import.meta.url)); + +/** + * Evidence-source prereq gate (eval:a2a build verdict + * `2026-07-07-eval-a2a-reeval-telemetry-still-disabled-build`, PR #19). + * + * Bug class: the scheduled eval fires on a runtime whose OTel telemetry is + * disabled (TELEMETRY_HMAC_SALT unset → initTelemetry() returned null handles). + * The `f167-runtime-eval` source cannot produce fresh snapshots, yet the eval + * cat is invoked anyway and burns a full LLM session to re-conclude "telemetry + * still disabled" — every day (2026-06-30 → 2026-07-07 verdict series). The + * gate fails closed BEFORE invocation and posts a zero-LLM-cost skip notice + * to the domain's own system thread instead. + */ +describe('eval-domain evidence-source prereq gate (eval:a2a PR #19)', () => { + const mkCtx = () => { + const deliverMock = mock.fn(async () => 'msg_evidence'); + const triggerMock = mock.fn(); + return { + deliverMock, + triggerMock, + ctx: { assignedCatId: null, deliver: deliverMock, invokeTrigger: { trigger: triggerMock } }, + }; + }; + + describe('createTelemetryEvidencePrereqProbe', () => { + it('telemetry-backed adapter + OTel disabled → not ok, reason points at salt', () => { + const probe = createTelemetryEvidencePrereqProbe({ otelEnabled: () => false }); + const result = probe({ domainId: 'eval:a2a', sourceAdapter: 'f167-runtime-eval' }); + assert.equal(result.ok, false); + assert.ok( + result.reason.includes('TELEMETRY_HMAC_SALT'), + `default reason must name the missing salt env var (got: ${result.reason})`, + ); + }); + + it('telemetry-backed adapter + OTel enabled → ok', () => { + const probe = createTelemetryEvidencePrereqProbe({ otelEnabled: () => true }); + const result = probe({ domainId: 'eval:a2a', sourceAdapter: 'f167-runtime-eval' }); + assert.equal(result.ok, true); + }); + + it('non-telemetry adapter passes through even when OTel is disabled', () => { + const probe = createTelemetryEvidencePrereqProbe({ otelEnabled: () => false }); + const result = probe({ domainId: 'eval:sop', sourceAdapter: 'sop-trace-eval' }); + assert.equal(result.ok, true, 'gate must only constrain telemetry-backed source adapters'); + }); + + it('OTEL_SDK_DISABLED=true → reason names the env toggle, not the salt', () => { + const prev = process.env.OTEL_SDK_DISABLED; + process.env.OTEL_SDK_DISABLED = 'true'; + try { + const probe = createTelemetryEvidencePrereqProbe({ otelEnabled: () => false }); + const result = probe({ domainId: 'eval:a2a', sourceAdapter: 'f167-runtime-eval' }); + assert.equal(result.ok, false); + assert.ok(result.reason.includes('OTEL_SDK_DISABLED'), `got: ${result.reason}`); + } finally { + if (prev === undefined) delete process.env.OTEL_SDK_DISABLED; + else process.env.OTEL_SDK_DISABLED = prev; + } + }); + }); + + it('bootstrap wires the telemetry init state into every scheduled eval spec', () => { + const indexSource = readFileSync(new URL('../../src/index.ts', import.meta.url), 'utf8'); + + assert.match(indexSource, /createTelemetryEvidencePrereqProbe/); + assert.match( + indexSource, + /otelEnabled:\s*\(\)\s*=>\s*telemetryHandle\.getMetricsText\s*!==\s*null/, + 'the probe must observe the actual boot-time telemetry handle, not an env proxy', + ); + assert.match( + indexSource, + /const evalScheduleOpts = \{[\s\S]*?evidencePrereqProbe,[\s\S]*?\};/, + 'shared daily, weekly, and N-day schedule options must include the evidence probe', + ); + }); + + describe('evaluateEvidencePrereq', () => { + it('probe throw → fail-closed not-ok with reason', async () => { + const result = await evaluateEvidencePrereq( + () => { + throw new Error('synthetic evidence probe failure'); + }, + { domainId: 'eval:a2a', sourceAdapter: 'f167-runtime-eval' }, + ); + assert.equal(result.ok, false); + assert.ok(result.reason.includes('synthetic evidence probe failure')); + }); + }); + + describe('daily spec integration', () => { + async function getA2aItem(spec) { + const gateResult = await spec.admission.gate(); + const item = gateResult.workItems.find((w) => w.subjectKey === 'eval:a2a'); + assert.ok(item, 'eval:a2a must be a registered daily domain'); + return item; + } + + it('probe not-ok → SKIPPED notice in domain thread, cat never invoked', async () => { + const spec = createEvalDomainDailySpec({ + harnessFeedbackRoot: repoHarnessFeedbackRoot, + defaultUserId: 'default-user', + evidencePrereqProbe: () => ({ ok: false, reason: 'OTel disabled at boot: HMAC salt validation failed' }), + }); + const item = await getA2aItem(spec); + const { deliverMock, triggerMock, ctx } = mkCtx(); + + await spec.run.execute(item.signal, item.subjectKey, ctx); + + assert.equal(triggerMock.mock.callCount(), 0, 'trigger must NOT fire when evidence source is down'); + assert.equal(deliverMock.mock.callCount(), 1); + const call = deliverMock.mock.calls[0].arguments[0]; + assert.equal(call.threadId, 'thread_eval_a2a', 'skip notice must stay in the domain system thread'); + assert.equal(call.userId, 'scheduler'); + assert.ok(call.content.includes('SKIPPED (evidence source unavailable)'), 'stable header for grep/dedup'); + assert.ok(call.content.includes('HMAC salt validation failed'), 'notice must carry the probe reason'); + assert.ok(call.content.includes('TELEMETRY_HMAC_SALT'), 'notice must state the actionable next step'); + }); + + it('probe ok → normal invocation proceeds', async () => { + const spec = createEvalDomainDailySpec({ + harnessFeedbackRoot: repoHarnessFeedbackRoot, + defaultUserId: 'default-user', + evidencePrereqProbe: () => ({ ok: true }), + }); + const item = await getA2aItem(spec); + const { deliverMock, triggerMock, ctx } = mkCtx(); + + await spec.run.execute(item.signal, item.subjectKey, ctx); + + assert.equal(triggerMock.mock.callCount(), 1, 'trigger fires when evidence source is healthy'); + assert.equal(deliverMock.mock.callCount(), 1); + assert.ok(!deliverMock.mock.calls[0].arguments[0].content.includes('SKIPPED')); + }); + + it('probe throws → fail-closed skip, no crash, no LLM call', async () => { + const spec = createEvalDomainDailySpec({ + harnessFeedbackRoot: repoHarnessFeedbackRoot, + defaultUserId: 'default-user', + evidencePrereqProbe: () => { + throw new Error('synthetic gate crash'); + }, + }); + const item = await getA2aItem(spec); + const { deliverMock, triggerMock, ctx } = mkCtx(); + + await spec.run.execute(item.signal, item.subjectKey, ctx); + + assert.equal(triggerMock.mock.callCount(), 0); + assert.equal(deliverMock.mock.callCount(), 1); + assert.ok(deliverMock.mock.calls[0].arguments[0].content.includes('SKIPPED (evidence source unavailable)')); + }); + + it('both gates failing → evidence-source message wins (upstream-first ordering)', async () => { + const spec = createEvalDomainDailySpec({ + harnessFeedbackRoot: repoHarnessFeedbackRoot, + defaultUserId: 'default-user', + evidencePrereqProbe: () => ({ ok: false, reason: 'OTel disabled at boot' }), + publishPrereqProbe: () => false, + }); + const item = await getA2aItem(spec); + const { deliverMock, triggerMock, ctx } = mkCtx(); + + await spec.run.execute(item.signal, item.subjectKey, ctx); + + assert.equal(triggerMock.mock.callCount(), 0); + assert.equal(deliverMock.mock.callCount(), 1, 'exactly one skip notice, not two'); + const content = deliverMock.mock.calls[0].arguments[0].content; + assert.ok(content.includes('evidence source unavailable'), 'evidence gate runs before publish gate'); + assert.ok(!content.includes('publish prereq missing')); + }); + + it('evidence probe ok + publish gate still enforced → publish skip preserved', async () => { + const spec = createEvalDomainDailySpec({ + harnessFeedbackRoot: repoHarnessFeedbackRoot, + defaultUserId: 'default-user', + evidencePrereqProbe: () => ({ ok: true }), + publishPrereqProbe: () => false, + }); + const item = await getA2aItem(spec); + const { deliverMock, triggerMock, ctx } = mkCtx(); + + await spec.run.execute(item.signal, item.subjectKey, ctx); + + assert.equal(triggerMock.mock.callCount(), 0); + assert.equal(deliverMock.mock.callCount(), 1); + assert.ok(deliverMock.mock.calls[0].arguments[0].content.includes('publish prereq missing')); + }); + }); + + describe('nday spec integration', () => { + it('probe not-ok → skip notice, no trigger, no Redis last-dispatch write', async () => { + const redisSet = mock.fn(async () => 'OK'); + const redis = { get: mock.fn(async () => null), set: redisSet }; + const spec = createEvalDomainNDaySpec({ + harnessFeedbackRoot: repoHarnessFeedbackRoot, + defaultUserId: 'default-user', + redis, + evidencePrereqProbe: () => ({ ok: false, reason: 'OTel disabled at boot' }), + }); + + const gateResult = await spec.admission.gate(); + assert.equal(gateResult.run, true, 'registry must contain at least one N-day domain'); + const item = gateResult.workItems[0]; + const { deliverMock, triggerMock, ctx } = mkCtx(); + + await spec.run.execute(item.signal, item.subjectKey, ctx); + + assert.equal(triggerMock.mock.callCount(), 0, 'trigger must NOT fire when evidence source is down'); + assert.equal(deliverMock.mock.callCount(), 1); + const call = deliverMock.mock.calls[0].arguments[0]; + assert.equal(call.threadId, item.signal.systemThreadId); + assert.ok(call.content.includes('SKIPPED (evidence source unavailable)')); + assert.equal( + redisSet.mock.callCount(), + 0, + 'skip must NOT consume the N-day window — domain retries on next daily probe', + ); + }); + }); +});