From 779ace8797114bbca20d538e76bd9734c9dc11af Mon Sep 17 00:00:00 2001 From: 0xJMC Date: Fri, 26 Jun 2026 19:46:08 -0400 Subject: [PATCH] feat(validation): independent validation that certifies items into validated knowledge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provider validate flow (claim->fulfill->validate) updated the request but never certified the research items, and never checked the validator was anyone other than the fulfiller — so 'validated' meant 'self-approved' and produced no validated knowledge the validated_only/enterprise tier could return. New lib/validation.ts (shared, tested): - assertValidatorIndependent: a validator can't be the request's fulfiller or a contributor of the items (validator_is_the_fulfiller / validator_is_a_contributor). - certifyResearchItems: on accept, recompute the quality rubric and promote each fulfillment item to validated ONLY when it clears the enterprise floor (>=70), so a thumbs-up can't launder a thin item; logs a contributor_quality_events row. validate route now enforces independence + certifies the fulfillment items, and returns { certified, certifiedCount }. llms.txt + skill manifest document the independence rule. +8 unit tests (217 total). --- .../knowledge/requests/[id]/validate/route.ts | 47 +++++++- App/app/skill/manifest/route.ts | 1 + App/lib/validation.ts | 114 ++++++++++++++++++ App/public/llms.txt | 4 +- App/tests/validation.test.ts | 97 +++++++++++++++ 5 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 App/lib/validation.ts create mode 100644 App/tests/validation.test.ts diff --git a/App/app/knowledge/requests/[id]/validate/route.ts b/App/app/knowledge/requests/[id]/validate/route.ts index 95d7969..cb6d24c 100644 --- a/App/app/knowledge/requests/[id]/validate/route.ts +++ b/App/app/knowledge/requests/[id]/validate/route.ts @@ -1,10 +1,11 @@ import { withDb } from '../../../../../lib/db'; import { getProviderIdentity } from '../../../../../lib/providers'; import { publicKnowledgeRequest } from '../../../../../lib/requests'; +import { assertValidatorIndependent, certifyResearchItems, type CertifiedItem } from '../../../../../lib/validation'; export const dynamic = 'force-dynamic'; -type Params = { params: Promise<{ id: string }> }; +type Params = { params: Promise<{ id: string }> }; function unauthorized() { return Response.json({ ok: false, error: 'unauthorized' }, { status: 401 }); @@ -19,6 +20,7 @@ export async function POST(request: Request, { params }: Params) { const body = await request.json().catch(() => ({})); const identity = getProviderIdentity(request, body); if (!identity.agentId) return Response.json({ ok: false, error: 'x_agent_id_required' }, { status: 400 }); + const validatorAgentId = identity.agentId; // narrowed to string; survives the async closure const accepted = body.accepted !== false; const notes = String(body.notes || body.validation_notes || '').trim() || null; @@ -27,6 +29,31 @@ export async function POST(request: Request, { params }: Params) { const agent = await client.query(`SELECT id, status FROM agents WHERE id = $1`, [identity.agentId]); if (!agent.rowCount || agent.rows[0].status !== 'active') return { error: 'validator_agent_not_registered_or_inactive', status: 403 } as const; + // Load the request first — its fulfiller + items drive the independence + // guard and the item certification. + const reqRow = await client.query(`SELECT * FROM knowledge_requests WHERE id = $1`, [id]); + if (!reqRow.rowCount) return { error: 'request_not_found', status: 404 } as const; + const req = reqRow.rows[0]; + if (req.status !== 'fulfilled') return { error: 'request_not_fulfilled', status: 409, row: req } as const; + + const itemIds: string[] = Array.isArray(req.fulfillment_item_ids) ? req.fulfillment_item_ids.filter(Boolean) : []; + + // Independent validation: an accepting validator may not be the fulfiller or + // a contributor of the items being certified (no self-approval). + if (accepted) { + let contributorIds: Array = []; + if (itemIds.length) { + const c = await client.query(`SELECT contributor_agent_id FROM research_items WHERE id = ANY($1::text[])`, [itemIds]); + contributorIds = c.rows.map((r) => r.contributor_agent_id); + } + const indep = assertValidatorIndependent({ + validatorAgentId, + fulfilledByAgentId: req.fulfilled_by_agent_id, + contributorAgentIds: contributorIds, + }); + if (!indep.ok) return { error: indep.reason, status: 403, row: req } as const; + } + const result = await client.query( `UPDATE knowledge_requests SET status = $2, @@ -39,16 +66,28 @@ export async function POST(request: Request, { params }: Params) { [id, accepted ? 'validated' : 'rejected', identity.agentId, notes] ); if (!result.rowCount) { + // Raced past 'fulfilled' between our read and write. const existing = await client.query(`SELECT * FROM knowledge_requests WHERE id = $1`, [id]); - if (!existing.rowCount) return { error: 'request_not_found', status: 404 } as const; return { error: 'request_not_fulfilled', status: 409, row: existing.rows[0] } as const; } - return { row: result.rows[0] } as const; + + // Accepting a request promotes its fulfillment items to validated knowledge + // (independently certified) so the enterprise / validated_only tier returns them. + let certified: CertifiedItem[] = []; + if (accepted && itemIds.length) { + certified = await certifyResearchItems(client, { itemIds, validatorAgentId }); + } + return { row: result.rows[0], certified } as const; }); if ('error' in response) { return Response.json({ ok: false, error: response.error, request: response.row ? publicKnowledgeRequest(response.row) : undefined }, { status: response.status }); } - return Response.json({ ok: true, request: publicKnowledgeRequest(response.row) }); + return Response.json({ + ok: true, + request: publicKnowledgeRequest(response.row), + certified: response.certified, + certifiedCount: response.certified.filter((c) => c.validationStatus === 'validated').length, + }); } diff --git a/App/app/skill/manifest/route.ts b/App/app/skill/manifest/route.ts index acd0d0f..f52628d 100644 --- a/App/app/skill/manifest/route.ts +++ b/App/app/skill/manifest/route.ts @@ -72,6 +72,7 @@ export async function GET() { autoCreateOnQueryMiss: 'POST /skill/query creates an open knowledge request when coverage is insufficient unless createRequestOnMiss=false', statuses: ['open', 'claimed', 'fulfilled', 'validated', 'closed', 'rejected'], purpose: 'turn missing knowledge into research tasks that provider agents can claim, fulfill, validate, and index', + independentValidation: 'validate is independent — the validator cannot be the fulfiller or a contributor of the items; accepting an evidenced request certifies its items as validated knowledge (only those clearing the quality floor) so the validated_only/enterprise tier returns them', }, economics: { mode: 'credit', diff --git a/App/lib/validation.ts b/App/lib/validation.ts new file mode 100644 index 0000000..cdd6410 --- /dev/null +++ b/App/lib/validation.ts @@ -0,0 +1,114 @@ +/** + * Independent validation — the credibility core of the "validated knowledge" + * rail. Validating a request must (a) be done by an agent that is NOT the one + * that fulfilled it or contributed the items (no self-approval), and (b) actually + * promote the contributed research items to `validated` so the enterprise / + * validated_only tier can return them. Both halves live here so the request + * `validate` route and the admin quality tool share one implementation. + */ +import type { Client } from "pg"; + +import { assessKnowledgeQuality } from "./quality"; + +/** + * Guard: a validator may not certify their own work. Reject when the validator + * is the request's fulfiller or a contributor of any item being certified — + * otherwise "validated" would just mean "self-approved". Pure + testable. + */ +export function assertValidatorIndependent(input: { + validatorAgentId: string; + fulfilledByAgentId?: string | null; + contributorAgentIds?: Array; +}): { ok: true } | { ok: false; reason: string } { + const v = (input.validatorAgentId || "").trim().toLowerCase(); + if (!v) return { ok: false, reason: "validator_agent_required" }; + const fulfiller = (input.fulfilledByAgentId || "").trim().toLowerCase(); + if (fulfiller && fulfiller === v) return { ok: false, reason: "validator_is_the_fulfiller" }; + const contributors = (input.contributorAgentIds || []) + .map((c) => (c || "").trim().toLowerCase()) + .filter(Boolean); + if (contributors.includes(v)) return { ok: false, reason: "validator_is_a_contributor" }; + return { ok: true }; +} + +export type CertifiedItem = { + id: string; + validationStatus: "validated" | "pending"; + confidencePercent: number; + trustTier: string; + reasons: string[]; +}; + +/** + * Certify research items as independently validated. Recomputes the quality + * rubric with `validationStatus='validated'` (so the validated bump applies) and + * promotes each item to `validated` only when it clears the enterprise floor + * (score >= 70); weaker items stay `pending` with a reason, so a thumbs-up can't + * launder a thin item past the bar. Logs one `contributor_quality_events` row + * per item (who validated it + the assessment). Mirrors `/api/admin/quality`. + * + * The caller owns the independence check ([assertValidatorIndependent]) and the + * request-status transition; this only touches the items. + */ +export async function certifyResearchItems( + client: Client, + input: { itemIds: string[]; validatorAgentId: string }, +): Promise { + const ids = input.itemIds.filter(Boolean); + if (!ids.length) return []; + + const select = await client.query( + `SELECT id, title, summary, confidence, evidence, validation_status, contributor_agent_id, content_hash + FROM research_items WHERE id = ANY($1::text[])`, + [ids], + ); + + const out: CertifiedItem[] = []; + for (const row of select.rows) { + const assessment = assessKnowledgeQuality({ + title: row.title, + summary: row.summary, + confidence: row.confidence, + evidence: row.evidence, + validationStatus: "validated", + contributorAgentId: row.contributor_agent_id, + contentHash: row.content_hash, + }); + const finalStatus: CertifiedItem["validationStatus"] = assessment.score >= 70 ? "validated" : "pending"; + const reasons = finalStatus === "validated" + ? assessment.reasons.filter((r) => r !== "awaiting independent validation" && r !== "not independently validated") + : assessment.reasons; + + await client.query( + `UPDATE research_items + SET validation_status = $2, + quality_score = $3, + confidence_percent = $4, + trust_tier = $5, + quality_reasons = $6, + validated_at = CASE WHEN $2 = 'validated' THEN now() ELSE validated_at END, + updated_at = now() + WHERE id = $1`, + [row.id, finalStatus, assessment.score, assessment.confidencePercent, assessment.tier, reasons], + ); + await client.query( + `INSERT INTO contributor_quality_events (research_item_id, contributor_agent_id, event_type, score, metadata) + VALUES ($1, $2, $3, $4, $5)`, + [ + row.id, + row.contributor_agent_id || null, + finalStatus === "validated" ? "validated" : "assessed", + assessment.score, + { validatorAgentId: input.validatorAgentId, source: "request_validation", assessment }, + ], + ); + out.push({ + id: row.id, + validationStatus: finalStatus, + confidencePercent: assessment.confidencePercent, + trustTier: assessment.tier, + reasons, + }); + } + return out; +} diff --git a/App/public/llms.txt b/App/public/llms.txt index f48c8cc..938e7da 100644 --- a/App/public/llms.txt +++ b/App/public/llms.txt @@ -42,7 +42,7 @@ Privacy boundary: private organization records are ACL-protected and sensitive o You earn credits whenever knowledge you contributed is consumed by a paid query (value split equally across the contributing items' providers → attribution → credited to your `x-agent-wallet`). Two ways: -- **Answer open requests:** `GET /knowledge/requests?status=open` → `POST /knowledge/requests/{id}/claim` → `POST /knowledge/requests/{id}/fulfill` (attach research + evidence) → `POST /knowledge/requests/{id}/validate`. Lifecycle: open → claimed → fulfilled → validated → closed. +- **Answer open requests:** `GET /knowledge/requests?status=open` → `POST /knowledge/requests/{id}/claim` → `POST /knowledge/requests/{id}/fulfill` (attach research + evidence) → `POST /knowledge/requests/{id}/validate`. Lifecycle: open → claimed → fulfilled → validated → closed. **Validation is independent**: the validator can't be the fulfiller or a contributor of the items (`validator_is_the_fulfiller` / `validator_is_a_contributor` on a violation). Accepting an evidenced request **certifies its items as validated knowledge** (only those clearing the quality floor) so the `validated_only`/`enterprise` tier returns them. - **Submit research directly:** `POST /api/ingest/research` with `x-agent-id` + `x-agent-wallet` (contributor auth: bearer ingest token / agent id / org membership). Default visibility is private; public candidates are stored for review. Evidence is required by default — validated, evidenced items rank and earn more. ## Agent roles @@ -50,7 +50,7 @@ You earn credits whenever knowledge you contributed is consumed by a paid query - **Consumer:** queries public or organization-private context (pays per query). - **Requester:** turns missing context into an open request instead of guessing. - **Provider:** claims requests, submits sanitized + evidenced research, fulfills work, earns credits. -- **Validator:** reviews evidence and trust state before higher-confidence reuse. +- **Validator:** an *independent* agent (not the fulfiller/contributor) reviews the evidence and accepts — certifying the items into validated knowledge that higher tiers can trust. ## Easiest path: the plugin diff --git a/App/tests/validation.test.ts b/App/tests/validation.test.ts new file mode 100644 index 0000000..9cc0c79 --- /dev/null +++ b/App/tests/validation.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it, vi } from "vitest"; + +import { assertValidatorIndependent, certifyResearchItems } from "../lib/validation"; + +describe("assertValidatorIndependent", () => { + it("accepts a validator distinct from the fulfiller and contributors", () => { + expect( + assertValidatorIndependent({ validatorAgentId: "Vera", fulfilledByAgentId: "Pablo", contributorAgentIds: ["Pablo", "Carla"] }), + ).toEqual({ ok: true }); + }); + + it("rejects the fulfiller validating their own request (case-insensitive)", () => { + const r = assertValidatorIndependent({ validatorAgentId: "Pablo", fulfilledByAgentId: "pablo" }); + expect(r).toEqual({ ok: false, reason: "validator_is_the_fulfiller" }); + }); + + it("rejects a contributor of the items being certified", () => { + const r = assertValidatorIndependent({ validatorAgentId: "Carla", fulfilledByAgentId: "Pablo", contributorAgentIds: ["Carla"] }); + expect(r).toEqual({ ok: false, reason: "validator_is_a_contributor" }); + }); + + it("rejects an empty validator id", () => { + expect(assertValidatorIndependent({ validatorAgentId: " " }).ok).toBe(false); + }); + + it("tolerates null/undefined contributor entries", () => { + expect( + assertValidatorIndependent({ validatorAgentId: "Vera", fulfilledByAgentId: null, contributorAgentIds: [null, undefined, "Pablo"] }), + ).toEqual({ ok: true }); + }); +}); + +// Fake pg Client: serve the SELECT of items, capture UPDATE/INSERT. +function fakeClient(items: Array>) { + const updates: unknown[][] = []; + const events: unknown[][] = []; + const query = vi.fn(async (sql: string, paramsArg: unknown[] = []) => { + if (/SELECT id, title, summary/.test(sql)) return { rows: items, rowCount: items.length }; + if (/UPDATE research_items/.test(sql)) { updates.push(paramsArg); return { rows: [], rowCount: 1 }; } + if (/INSERT INTO contributor_quality_events/.test(sql)) { events.push(paramsArg); return { rows: [], rowCount: 1 }; } + return { rows: [], rowCount: 0 }; + }); + return { client: { query } as never, updates, events }; +} + +const strongItem = { + id: "item-strong", + title: "Base smart-wallet gas sponsorship", + summary: "A".repeat(140), // >= 120 chars + confidence: "high", + evidence: [ + { type: "url", url: "https://docs.base.org/x", verified: true }, + { type: "url", url: "https://basescan.org/tx/0xabc" }, + ], + validation_status: "pending", + contributor_agent_id: "Pablo", + content_hash: "deadbeef", +}; + +const weakItem = { + id: "item-weak", + title: "x", // too short + summary: "thin", // too thin + confidence: "low", + evidence: [], + validation_status: "pending", + contributor_agent_id: null, + content_hash: "", +}; + +describe("certifyResearchItems", () => { + it("promotes an evidenced, well-formed item to validated", async () => { + const { client, updates, events } = fakeClient([strongItem]); + const out = await certifyResearchItems(client, { itemIds: ["item-strong"], validatorAgentId: "Vera" }); + expect(out[0].validationStatus).toBe("validated"); + expect(out[0].confidencePercent).toBeGreaterThanOrEqual(70); + // the UPDATE wrote 'validated' and the event logged 'validated' by the validator + expect(updates[0][1]).toBe("validated"); + expect(events[0][2]).toBe("validated"); + expect((events[0][4] as { validatorAgentId: string }).validatorAgentId).toBe("Vera"); + // the "not independently validated" reason is stripped once certified + expect(out[0].reasons).not.toContain("not independently validated"); + }); + + it("keeps a thin item pending (a thumbs-up can't launder it past the floor)", async () => { + const { client, updates } = fakeClient([weakItem]); + const out = await certifyResearchItems(client, { itemIds: ["item-weak"], validatorAgentId: "Vera" }); + expect(out[0].validationStatus).toBe("pending"); + expect(out[0].confidencePercent).toBeLessThan(70); + expect(updates[0][1]).toBe("pending"); + }); + + it("returns [] for no item ids", async () => { + const { client } = fakeClient([]); + expect(await certifyResearchItems(client, { itemIds: [], validatorAgentId: "Vera" })).toEqual([]); + }); +});