From 0de1a4f2a98a359e07a52cf234e69f60e17044ca Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Mon, 20 Jul 2026 19:34:11 +0100 Subject: [PATCH 1/7] feat: add external evidence verifier CLI --- artifacts/verify/README.md | 33 + .../activity-personalization.v1.schema.json | 676 ++++++++++++++++++ artifacts/verify/broken-off.example.json | 134 ++++ artifacts/verify/passing-off.example.json | 107 +++ artifacts/verify/passing-on.example.json | 140 ++++ artifacts/verify/producer-template.mjs | 16 + package.json | 2 + src/verify/cli.ts | 396 ++++++++++ src/verify/examples.ts | 265 +++++++ src/verify/outcome.ts | 17 + src/verify/report.ts | 188 +++++ src/verify/schema.ts | 198 +++++ src/verify/verify.ts | 101 +++ tests/verify/cli.unit.ts | 368 ++++++++++ tests/verify/core.unit.ts | 326 +++++++++ 15 files changed, 2967 insertions(+) create mode 100644 artifacts/verify/README.md create mode 100644 artifacts/verify/activity-personalization.v1.schema.json create mode 100644 artifacts/verify/broken-off.example.json create mode 100644 artifacts/verify/passing-off.example.json create mode 100644 artifacts/verify/passing-on.example.json create mode 100644 artifacts/verify/producer-template.mjs create mode 100644 src/verify/cli.ts create mode 100644 src/verify/examples.ts create mode 100644 src/verify/outcome.ts create mode 100644 src/verify/report.ts create mode 100644 src/verify/schema.ts create mode 100644 src/verify/verify.ts create mode 100644 tests/verify/cli.unit.ts create mode 100644 tests/verify/core.unit.ts diff --git a/artifacts/verify/README.md b/artifacts/verify/README.md new file mode 100644 index 0000000..dbd6dd9 --- /dev/null +++ b/artifacts/verify/README.md @@ -0,0 +1,33 @@ +# PromiseProof external evidence scaffold + +This scaffold supports exactly one contract family: +`activity-personalization/v1`. + +Use `broken-off.example.json`, `passing-off.example.json`, and +`passing-on.example.json` as fixed-shape examples for producing a complete +external evidence bundle. `producer-template.mjs` is a small wrapper for a +complete `PromiseEvidence` JSON object; it does not collect or attest evidence. +Then verify one bundle: + +```text +npm run promiseproof -- verify --evidence --out +``` + +Or gate OFF and ON together: + +```text +npm run promiseproof -- gate --off --on --out +``` + +The OFF contract checks zero identifiable recommendation activity, a functional +contextual feed, and witnessed OFF persistence across UI, toggle, storage, and +backend state. The ON control checks correlated identifiable activity and a +functional behavioral feed. + +Evidence is externally supplied. PromiseProof does not attest how it was +collected. The deterministic PromiseProof evaluator alone evaluates a bundle +that passes strict validation. + +These examples prove externally supplied evidence ingestion for one contract +family. They do not prove arbitrary evidence collection or arbitrary promise +support. Article Atlas is a neutral synthetic example product. diff --git a/artifacts/verify/activity-personalization.v1.schema.json b/artifacts/verify/activity-personalization.v1.schema.json new file mode 100644 index 0000000..1228500 --- /dev/null +++ b/artifacts/verify/activity-personalization.v1.schema.json @@ -0,0 +1,676 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "schemaVersion": { + "type": "string", + "const": "1" + }, + "contractFamily": { + "type": "string", + "const": "activity-personalization/v1" + }, + "evidence": { + "type": "object", + "properties": { + "scenario": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "ui": { + "type": "object", + "properties": { + "preference": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "toggleChecked": { + "type": "boolean" + }, + "feedFunctional": { + "type": "boolean" + } + }, + "required": [ + "preference", + "toggleChecked", + "feedFunctional" + ], + "additionalProperties": false + }, + "storage": { + "type": "object", + "properties": { + "preference": { + "anyOf": [ + { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "preference" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "activityPayloads": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "eventType": { + "type": "string", + "const": "page_view" + }, + "itemId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "clientSequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "runId", + "userId", + "eventType", + "itemId", + "clientSequence", + "occurredAt" + ], + "additionalProperties": false + } + }, + "preferenceUpdates": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "targetUserId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "payload": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "preference": { + "type": "string", + "enum": [ + "on", + "off" + ] + } + }, + "required": [ + "runId", + "preference" + ], + "additionalProperties": false + } + }, + "required": [ + "targetUserId", + "payload" + ], + "additionalProperties": false + } + } + }, + "required": [ + "activityPayloads", + "preferenceUpdates" + ], + "additionalProperties": false + }, + "response": { + "type": "object", + "properties": { + "preferenceUpdates": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "preference": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "updatedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "receipt": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "preference" + }, + "receiptId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "receivedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "preference": { + "type": "string", + "enum": [ + "on", + "off" + ] + } + }, + "required": [ + "kind", + "receiptId", + "sequence", + "receivedAt", + "userId", + "preference" + ], + "additionalProperties": false + } + }, + "required": [ + "userId", + "preference", + "updatedAt", + "receipt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "preferenceUpdates" + ], + "additionalProperties": false + }, + "backend": { + "type": "object", + "properties": { + "preference": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "activityReceipts": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "activity" + }, + "service": { + "type": "string", + "const": "recommendation" + }, + "receiptId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "receivedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "payload": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "eventType": { + "type": "string", + "const": "page_view" + }, + "itemId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "clientSequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "runId", + "userId", + "eventType", + "itemId", + "clientSequence", + "occurredAt" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "service", + "receiptId", + "sequence", + "receivedAt", + "payload" + ], + "additionalProperties": false + } + }, + "recommendationReceipts": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "recommendation" + }, + "receiptId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "receivedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "source": { + "type": "string", + "enum": [ + "contextual", + "behavioral" + ] + }, + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "items": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "title": { + "type": "string", + "maxLength": 512 + }, + "description": { + "type": "string", + "maxLength": 2048 + }, + "eyebrow": { + "type": "string", + "maxLength": 512 + } + }, + "required": [ + "id", + "title", + "description", + "eyebrow" + ], + "additionalProperties": false + } + } + }, + "required": [ + "kind", + "receiptId", + "sequence", + "receivedAt", + "source", + "items" + ], + "additionalProperties": false + } + }, + "preferenceReceipts": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "preference" + }, + "receiptId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "receivedAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "userId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "preference": { + "type": "string", + "enum": [ + "on", + "off" + ] + } + }, + "required": [ + "kind", + "receiptId", + "sequence", + "receivedAt", + "userId", + "preference" + ], + "additionalProperties": false + } + } + }, + "required": [ + "preference", + "activityReceipts", + "recommendationReceipts", + "preferenceReceipts" + ], + "additionalProperties": false + }, + "recommendation": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "contextual", + "behavioral" + ] + }, + "itemIds": { + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + } + } + }, + "required": [ + "source", + "itemIds" + ], + "additionalProperties": false + }, + "timestamps": { + "type": "object", + "properties": { + "clientTimeline": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "event": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "\\S" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "detail": { + "type": "object", + "propertyNames": { + "type": "string", + "maxLength": 256 + }, + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "maxLength": 2048 + }, + { + "type": "number" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ] + } + } + }, + "required": [ + "sequence", + "event", + "timestamp" + ], + "additionalProperties": false + } + }, + "activityReceivedAt": { + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "preferenceReceivedAt": { + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "recommendationReceivedAt": { + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + } + }, + "required": [ + "clientTimeline", + "activityReceivedAt", + "preferenceReceivedAt", + "recommendationReceivedAt" + ], + "additionalProperties": false + }, + "journey": { + "type": "object", + "properties": { + "reloadObserved": { + "type": "boolean" + } + }, + "required": [ + "reloadObserved" + ], + "additionalProperties": false + } + }, + "required": [ + "scenario", + "runId", + "userId", + "ui", + "storage", + "request", + "response", + "backend", + "recommendation", + "timestamps", + "journey" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "contractFamily", + "evidence" + ], + "additionalProperties": false, + "title": "PromiseProof activity-personalization/v1 evidence bundle", + "description": "Externally supplied evidence for the single activity-personalization/v1 contract family." +} diff --git a/artifacts/verify/broken-off.example.json b/artifacts/verify/broken-off.example.json new file mode 100644 index 0000000..132a1cd --- /dev/null +++ b/artifacts/verify/broken-off.example.json @@ -0,0 +1,134 @@ +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "off", + "runId": "article-atlas-broken-off", + "userId": "article-atlas-reader-001", + "ui": { + "preference": "off", + "toggleChecked": false, + "feedFunctional": true + }, + "storage": { + "preference": "off" + }, + "request": { + "activityPayloads": [ + { + "runId": "article-atlas-broken-off", + "userId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + ], + "preferenceUpdates": [ + { + "targetUserId": "article-atlas-reader-001", + "payload": { + "runId": "article-atlas-broken-off", + "preference": "off" + } + } + ] + }, + "response": { + "preferenceUpdates": [ + { + "userId": "article-atlas-reader-001", + "preference": "off", + "updatedAt": "2026-01-15T12:00:00.000Z", + "receipt": { + "kind": "preference", + "receiptId": "article-atlas-broken-off-preference-receipt", + "sequence": 1, + "receivedAt": "2026-01-15T12:00:00.000Z", + "userId": "article-atlas-reader-001", + "preference": "off" + } + } + ] + }, + "backend": { + "preference": "off", + "activityReceipts": [ + { + "kind": "activity", + "service": "recommendation", + "receiptId": "article-atlas-broken-off-activity-receipt", + "sequence": 2, + "receivedAt": "2026-01-15T12:00:01.000Z", + "payload": { + "runId": "article-atlas-broken-off", + "userId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + } + ], + "recommendationReceipts": [ + { + "kind": "recommendation", + "receiptId": "article-atlas-broken-off-recommendation-receipt", + "sequence": 3, + "receivedAt": "2026-01-15T12:00:02.000Z", + "source": "contextual", + "items": [ + { + "id": "article-atlas-story-101", + "title": "A field guide to urban trees", + "description": "A synthetic Article Atlas recommendation.", + "eyebrow": "Article Atlas" + } + ] + } + ], + "preferenceReceipts": [ + { + "kind": "preference", + "receiptId": "article-atlas-broken-off-preference-receipt", + "sequence": 1, + "receivedAt": "2026-01-15T12:00:00.000Z", + "userId": "article-atlas-reader-001", + "preference": "off" + } + ] + }, + "recommendation": { + "source": "contextual", + "itemIds": [ + "article-atlas-story-101" + ] + }, + "timestamps": { + "clientTimeline": [ + { + "sequence": 1, + "event": "preference_restored", + "timestamp": "2026-01-15T12:00:00.000Z" + }, + { + "sequence": 2, + "event": "recommendations_loaded", + "timestamp": "2026-01-15T12:00:02.000Z" + } + ], + "activityReceivedAt": [ + "2026-01-15T12:00:01.000Z" + ], + "preferenceReceivedAt": [ + "2026-01-15T12:00:00.000Z" + ], + "recommendationReceivedAt": [ + "2026-01-15T12:00:02.000Z" + ] + }, + "journey": { + "reloadObserved": true + } + } +} diff --git a/artifacts/verify/passing-off.example.json b/artifacts/verify/passing-off.example.json new file mode 100644 index 0000000..6e92f86 --- /dev/null +++ b/artifacts/verify/passing-off.example.json @@ -0,0 +1,107 @@ +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "off", + "runId": "article-atlas-passing-off", + "userId": "article-atlas-reader-001", + "ui": { + "preference": "off", + "toggleChecked": false, + "feedFunctional": true + }, + "storage": { + "preference": "off" + }, + "request": { + "activityPayloads": [], + "preferenceUpdates": [ + { + "targetUserId": "article-atlas-reader-001", + "payload": { + "runId": "article-atlas-passing-off", + "preference": "off" + } + } + ] + }, + "response": { + "preferenceUpdates": [ + { + "userId": "article-atlas-reader-001", + "preference": "off", + "updatedAt": "2026-01-15T12:00:00.000Z", + "receipt": { + "kind": "preference", + "receiptId": "article-atlas-passing-off-preference-receipt", + "sequence": 1, + "receivedAt": "2026-01-15T12:00:00.000Z", + "userId": "article-atlas-reader-001", + "preference": "off" + } + } + ] + }, + "backend": { + "preference": "off", + "activityReceipts": [], + "recommendationReceipts": [ + { + "kind": "recommendation", + "receiptId": "article-atlas-passing-off-recommendation-receipt", + "sequence": 3, + "receivedAt": "2026-01-15T12:00:02.000Z", + "source": "contextual", + "items": [ + { + "id": "article-atlas-story-101", + "title": "A field guide to urban trees", + "description": "A synthetic Article Atlas recommendation.", + "eyebrow": "Article Atlas" + } + ] + } + ], + "preferenceReceipts": [ + { + "kind": "preference", + "receiptId": "article-atlas-passing-off-preference-receipt", + "sequence": 1, + "receivedAt": "2026-01-15T12:00:00.000Z", + "userId": "article-atlas-reader-001", + "preference": "off" + } + ] + }, + "recommendation": { + "source": "contextual", + "itemIds": [ + "article-atlas-story-101" + ] + }, + "timestamps": { + "clientTimeline": [ + { + "sequence": 1, + "event": "preference_restored", + "timestamp": "2026-01-15T12:00:00.000Z" + }, + { + "sequence": 2, + "event": "recommendations_loaded", + "timestamp": "2026-01-15T12:00:02.000Z" + } + ], + "activityReceivedAt": [], + "preferenceReceivedAt": [ + "2026-01-15T12:00:00.000Z" + ], + "recommendationReceivedAt": [ + "2026-01-15T12:00:02.000Z" + ] + }, + "journey": { + "reloadObserved": true + } + } +} diff --git a/artifacts/verify/passing-on.example.json b/artifacts/verify/passing-on.example.json new file mode 100644 index 0000000..b9b6909 --- /dev/null +++ b/artifacts/verify/passing-on.example.json @@ -0,0 +1,140 @@ +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "on", + "runId": "article-atlas-passing-on", + "userId": "article-atlas-reader-001", + "ui": { + "preference": "on", + "toggleChecked": true, + "feedFunctional": true + }, + "storage": { + "preference": "on" + }, + "request": { + "activityPayloads": [ + { + "runId": "article-atlas-passing-on", + "userId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + ], + "preferenceUpdates": [ + { + "targetUserId": "article-atlas-reader-001", + "payload": { + "runId": "article-atlas-passing-on", + "preference": "on" + } + } + ] + }, + "response": { + "preferenceUpdates": [ + { + "userId": "article-atlas-reader-001", + "preference": "on", + "updatedAt": "2026-01-15T12:00:00.000Z", + "receipt": { + "kind": "preference", + "receiptId": "article-atlas-passing-on-preference-receipt", + "sequence": 1, + "receivedAt": "2026-01-15T12:00:00.000Z", + "userId": "article-atlas-reader-001", + "preference": "on" + } + } + ] + }, + "backend": { + "preference": "on", + "activityReceipts": [ + { + "kind": "activity", + "service": "recommendation", + "receiptId": "article-atlas-passing-on-activity-receipt", + "sequence": 2, + "receivedAt": "2026-01-15T12:00:01.000Z", + "payload": { + "runId": "article-atlas-passing-on", + "userId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + } + ], + "recommendationReceipts": [ + { + "kind": "recommendation", + "receiptId": "article-atlas-passing-on-recommendation-receipt", + "sequence": 3, + "receivedAt": "2026-01-15T12:00:02.000Z", + "source": "behavioral", + "userId": "article-atlas-reader-001", + "items": [ + { + "id": "article-atlas-story-101", + "title": "A field guide to urban trees", + "description": "A synthetic Article Atlas recommendation.", + "eyebrow": "Article Atlas" + } + ] + } + ], + "preferenceReceipts": [ + { + "kind": "preference", + "receiptId": "article-atlas-passing-on-preference-receipt", + "sequence": 1, + "receivedAt": "2026-01-15T12:00:00.000Z", + "userId": "article-atlas-reader-001", + "preference": "on" + } + ] + }, + "recommendation": { + "source": "behavioral", + "itemIds": [ + "article-atlas-story-101" + ] + }, + "timestamps": { + "clientTimeline": [ + { + "sequence": 1, + "event": "preference_restored", + "timestamp": "2026-01-15T12:00:00.000Z" + }, + { + "sequence": 2, + "event": "activity_recorded", + "timestamp": "2026-01-15T12:00:01.000Z" + }, + { + "sequence": 3, + "event": "recommendations_loaded", + "timestamp": "2026-01-15T12:00:02.000Z" + } + ], + "activityReceivedAt": [ + "2026-01-15T12:00:01.000Z" + ], + "preferenceReceivedAt": [ + "2026-01-15T12:00:00.000Z" + ], + "recommendationReceivedAt": [ + "2026-01-15T12:00:02.000Z" + ] + }, + "journey": { + "reloadObserved": true + } + } +} diff --git a/artifacts/verify/producer-template.mjs b/artifacts/verify/producer-template.mjs new file mode 100644 index 0000000..c7d48da --- /dev/null +++ b/artifacts/verify/producer-template.mjs @@ -0,0 +1,16 @@ +import { readFile, writeFile } from "node:fs/promises"; + +const [evidenceFile = "promise-evidence.json", outputFile = "bundle.json"] = + process.argv.slice(2); +const evidence = JSON.parse(await readFile(evidenceFile, "utf8")); +const bundle = { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence, +}; + +await writeFile( + outputFile, + `${JSON.stringify(bundle, null, 2)}\n`, + { encoding: "utf8", flag: "wx" }, +); diff --git a/package.json b/package.json index f08fb1f..5e8f586 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,8 @@ "start:race": "cross-env NODE_ENV=production DEMO_MODE=initialization-race node dist/server.mjs", "start:propagation": "cross-env NODE_ENV=production DEMO_MODE=propagation-failure node dist/server.mjs", "typecheck": "tsc --noEmit", + "promiseproof": "tsx src/verify/cli.ts", + "test:external": "tsx --test tests/verify/core.unit.ts tests/verify/cli.unit.ts", "evidence:verify": "tsx scripts/evidence-verify.ts", "demo:rehearse": "tsx scripts/demo-rehearse.ts", "test:judge:unit": "tsx --test tests/judge/rehearsal.unit.ts", diff --git a/src/verify/cli.ts b/src/verify/cli.ts new file mode 100644 index 0000000..426316c --- /dev/null +++ b/src/verify/cli.ts @@ -0,0 +1,396 @@ +import { + lstat, + mkdir, + readFile, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +import { + brokenOffExample, + passingOffExample, + passingOnExample, + producerTemplate, + scaffoldReadme, +} from "./examples.js"; +import { + EXIT_CODE, + type ExternalOutcome, +} from "./outcome.js"; +import { + createGateReport, + createVerifyReport, + serializeGateReportMarkdown, + serializeReportJson, + serializeVerifyReportMarkdown, +} from "./report.js"; +import { + externalBundleJsonSchema, + MAX_INPUT_BYTES, +} from "./schema.js"; +import { + gateValidationIssues, + runGate, + verifyBundle, +} from "./verify.js"; + +const HELP = `PromiseProof external evidence verifier + +Usage: + npm run promiseproof -- init --out + npm run promiseproof -- verify --evidence --out + npm run promiseproof -- gate --off --on --out + npm run promiseproof -- --help + +Exit codes: + 0 PASS + 1 usage, I/O, or unexpected execution error + 2 BROKEN_PROMISE + 3 INVALID_EVIDENCE + +Supported contract family: activity-personalization/v1 +`; + +const REPORT_FILENAMES = ["report.json", "report.md"] as const; +const SCAFFOLD_FILES = { + "activity-personalization.v1.schema.json": () => + serializeJson(externalBundleJsonSchema()), + "broken-off.example.json": () => serializeJson(brokenOffExample), + "passing-off.example.json": () => serializeJson(passingOffExample), + "passing-on.example.json": () => serializeJson(passingOnExample), + "producer-template.mjs": () => producerTemplate, + "README.md": () => scaffoldReadme, +} as const; + +class UsageError extends Error {} +class InvalidEvidenceFileError extends Error {} + +interface CliIo { + readonly stdout: (message: string) => void; + readonly stderr: (message: string) => void; +} + +interface ParsedOptions { + readonly [name: string]: string; +} + +function serializeJson(value: unknown): string { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function parseOptions( + args: readonly string[], + allowed: readonly string[], +): ParsedOptions { + const options: Record = {}; + const allowedSet = new Set(allowed); + + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if ( + name === undefined || + value === undefined || + !name.startsWith("--") || + value.startsWith("--") + ) { + throw new UsageError("Options must be supplied as --name pairs."); + } + + const key = name.slice(2); + if (!allowedSet.has(key)) { + throw new UsageError(`Unknown option: ${name}`); + } + if (options[key] !== undefined) { + throw new UsageError(`Duplicate option: ${name}`); + } + options[key] = value; + } + + for (const required of allowed) { + if (options[required] === undefined) { + throw new UsageError(`Missing required option: --${required}`); + } + } + + return options; +} + +function safeOutputDirectory(rawPath: string): string { + if (rawPath.trim().length === 0) { + throw new UsageError("--out must not be empty."); + } + + const normalizedSegments = rawPath.replaceAll("\\", "/").split("/"); + if (normalizedSegments.includes("..")) { + throw new UsageError("--out must not contain path traversal."); + } + + const resolved = path.resolve(rawPath); + if (resolved === path.parse(resolved).root) { + throw new UsageError("--out must not be a filesystem root."); + } + return resolved; +} + +function safeChildPath(directory: string, filename: string): string { + if ( + path.basename(filename) !== filename || + filename === "." || + filename === ".." + ) { + throw new Error("Generated filename is unsafe."); + } + + const child = path.resolve(directory, filename); + if (path.dirname(child) !== directory) { + throw new Error("Generated path escaped the output directory."); + } + return child; +} + +async function ensureOutputDirectory(directory: string): Promise { + try { + const existing = await lstat(directory); + if (existing.isSymbolicLink() || !existing.isDirectory()) { + throw new Error("Output path must be a real directory."); + } + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + await mkdir(directory, { recursive: true }); + return; + } + throw error; + } +} + +async function refuseUnsafeReportTarget(target: string): Promise { + try { + const existing = await lstat(target); + if (existing.isSymbolicLink() || !existing.isFile()) { + throw new Error(`Refusing unsafe report target: ${path.basename(target)}`); + } + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + return; + } + throw error; + } +} + +async function writeReports( + outputDirectory: string, + json: string, + markdown: string, +): Promise { + await ensureOutputDirectory(outputDirectory); + const jsonPath = safeChildPath(outputDirectory, REPORT_FILENAMES[0]); + const markdownPath = safeChildPath(outputDirectory, REPORT_FILENAMES[1]); + await refuseUnsafeReportTarget(jsonPath); + await refuseUnsafeReportTarget(markdownPath); + await writeFile(jsonPath, json, "utf8"); + await writeFile(markdownPath, markdown, "utf8"); +} + +async function writeScaffold(outputDirectory: string): Promise { + await ensureOutputDirectory(outputDirectory); + const entries = Object.entries(SCAFFOLD_FILES); + const targets = entries.map(([filename]) => + safeChildPath(outputDirectory, filename), + ); + + for (const target of targets) { + try { + await lstat(target); + throw new Error( + `Refusing to overwrite existing scaffold file: ${path.basename(target)}`, + ); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + continue; + } + throw error; + } + } + + const created: string[] = []; + try { + for (const [filename, content] of entries) { + const target = safeChildPath(outputDirectory, filename); + await writeFile(target, content(), { encoding: "utf8", flag: "wx" }); + created.push(target); + } + } catch (error) { + await Promise.all( + created.map(async (target) => { + const { unlink } = await import("node:fs/promises"); + await unlink(target); + }), + ); + throw error; + } +} + +async function readExternalJson(filename: string): Promise { + let metadata; + try { + metadata = await stat(filename); + } catch { + throw new Error("Unable to read evidence file."); + } + + if (!metadata.isFile()) { + throw new Error("Evidence path must identify a regular file."); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new InvalidEvidenceFileError( + `: input exceeds ${MAX_INPUT_BYTES}-byte limit`, + ); + } + + const bytes = await readFile(filename); + if (bytes.byteLength > MAX_INPUT_BYTES) { + throw new InvalidEvidenceFileError( + `: input exceeds ${MAX_INPUT_BYTES}-byte limit`, + ); + } + + try { + return JSON.parse(bytes.toString("utf8")) as unknown; + } catch { + throw new InvalidEvidenceFileError(": malformed JSON"); + } +} + +function printInvalid(io: CliIo, issues: readonly string[]): number { + io.stderr("INVALID_EVIDENCE"); + for (const issue of [...issues].sort()) { + io.stderr(`- ${issue}`); + } + return EXIT_CODE.INVALID_EVIDENCE; +} + +function printOutcome(io: CliIo, outcome: ExternalOutcome): void { + if (outcome === "PASS") { + io.stdout("PASS — report.json and report.md written."); + } else if (outcome === "BROKEN_PROMISE") { + io.stdout("BROKEN_PROMISE — report.json and report.md written."); + } +} + +async function runInit(args: readonly string[], io: CliIo): Promise { + const options = parseOptions(args, ["out"]); + await writeScaffold(safeOutputDirectory(options.out!)); + io.stdout("Initialized activity-personalization/v1 scaffold (6 files)."); + return EXIT_CODE.PASS; +} + +async function runVerify(args: readonly string[], io: CliIo): Promise { + const options = parseOptions(args, ["evidence", "out"]); + const raw = await readExternalJson(options.evidence!); + const result = verifyBundle(raw); + if (result.outcome === "INVALID_EVIDENCE") { + return printInvalid(io, result.issues); + } + + const report = createVerifyReport(result); + await writeReports( + safeOutputDirectory(options.out!), + serializeReportJson(report), + serializeVerifyReportMarkdown(report), + ); + printOutcome(io, result.outcome); + return EXIT_CODE[result.outcome]; +} + +async function runGateCommand( + args: readonly string[], + io: CliIo, +): Promise { + const options = parseOptions(args, ["off", "on", "out"]); + const rawOff = await readExternalJson(options.off!); + const rawOn = await readExternalJson(options.on!); + const result = runGate(rawOff, rawOn); + if (result.outcome === "INVALID_EVIDENCE") { + return printInvalid(io, gateValidationIssues(result)); + } + + const report = createGateReport(result); + await writeReports( + safeOutputDirectory(options.out!), + serializeReportJson(report), + serializeGateReportMarkdown(report), + ); + printOutcome(io, result.outcome); + return EXIT_CODE[result.outcome]; +} + +export async function runCli( + args: readonly string[], + io: CliIo = { + stdout: (message) => console.log(message), + stderr: (message) => console.error(message), + }, +): Promise { + if (args.length === 0) { + io.stderr(HELP); + return EXIT_CODE.EXECUTION_ERROR; + } + if (args[0] === "--help" || args[0] === "-h") { + io.stdout(HELP); + return EXIT_CODE.PASS; + } + + const [command, ...commandArgs] = args; + try { + if (command === "init") { + return await runInit(commandArgs, io); + } + if (command === "verify") { + return await runVerify(commandArgs, io); + } + if (command === "gate") { + return await runGateCommand(commandArgs, io); + } + throw new UsageError(`Unknown command: ${String(command)}`); + } catch (error) { + if (error instanceof InvalidEvidenceFileError) { + return printInvalid(io, [error.message]); + } + if (error instanceof UsageError) { + io.stderr(`USAGE_ERROR: ${error.message}`); + io.stderr("Run npm run promiseproof -- --help for usage."); + return EXIT_CODE.EXECUTION_ERROR; + } + + io.stderr( + `EXECUTION_ERROR: ${error instanceof Error ? error.message : String(error)}`, + ); + return EXIT_CODE.EXECUTION_ERROR; + } +} + +const invokedPath = process.argv[1] + ? path.resolve(process.argv[1]) + : undefined; +if (invokedPath === fileURLToPath(import.meta.url)) { + process.exitCode = await runCli(process.argv.slice(2)); +} diff --git a/src/verify/examples.ts b/src/verify/examples.ts new file mode 100644 index 0000000..499014d --- /dev/null +++ b/src/verify/examples.ts @@ -0,0 +1,265 @@ +import type { ExternalBundle } from "./schema.js"; +import { + SUPPORTED_CONTRACT_FAMILY, + SUPPORTED_SCHEMA_VERSION, +} from "./outcome.js"; + +const USER_ID = "article-atlas-reader-001"; +const ITEM_ID = "article-atlas-story-101"; +const TIME_PREFERENCE = "2026-01-15T12:00:00.000Z"; +const TIME_ACTIVITY = "2026-01-15T12:00:01.000Z"; +const TIME_RECOMMENDATION = "2026-01-15T12:00:02.000Z"; + +function recommendationItem() { + return { + id: ITEM_ID, + title: "A field guide to urban trees", + description: "A synthetic Article Atlas recommendation.", + eyebrow: "Article Atlas", + }; +} + +function preferenceState(runId: string, preference: "on" | "off") { + const receipt = { + kind: "preference" as const, + receiptId: `${runId}-preference-receipt`, + sequence: 1, + receivedAt: TIME_PREFERENCE, + userId: USER_ID, + preference, + }; + + return { + request: { + targetUserId: USER_ID, + payload: { runId, preference }, + }, + response: { + userId: USER_ID, + preference, + updatedAt: TIME_PREFERENCE, + receipt, + }, + receipt, + }; +} + +function activityState(runId: string) { + const payload = { + runId, + userId: USER_ID, + eventType: "page_view" as const, + itemId: "article-atlas-origin-001", + clientSequence: 1, + occurredAt: TIME_ACTIVITY, + }; + + return { + payload, + receipt: { + kind: "activity" as const, + service: "recommendation" as const, + receiptId: `${runId}-activity-receipt`, + sequence: 2, + receivedAt: TIME_ACTIVITY, + payload, + }, + }; +} + +function offBundle(includeActivity: boolean): ExternalBundle { + const runId = includeActivity + ? "article-atlas-broken-off" + : "article-atlas-passing-off"; + const preference = preferenceState(runId, "off"); + const activity = activityState(runId); + + return { + schemaVersion: SUPPORTED_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + evidence: { + scenario: "off", + runId, + userId: USER_ID, + ui: { + preference: "off", + toggleChecked: false, + feedFunctional: true, + }, + storage: { preference: "off" }, + request: { + activityPayloads: includeActivity ? [activity.payload] : [], + preferenceUpdates: [preference.request], + }, + response: { + preferenceUpdates: [preference.response], + }, + backend: { + preference: "off", + activityReceipts: includeActivity ? [activity.receipt] : [], + recommendationReceipts: [ + { + kind: "recommendation", + receiptId: `${runId}-recommendation-receipt`, + sequence: 3, + receivedAt: TIME_RECOMMENDATION, + source: "contextual", + items: [recommendationItem()], + }, + ], + preferenceReceipts: [preference.receipt], + }, + recommendation: { + source: "contextual", + itemIds: [ITEM_ID], + }, + timestamps: { + clientTimeline: [ + { + sequence: 1, + event: "preference_restored", + timestamp: TIME_PREFERENCE, + }, + { + sequence: 2, + event: "recommendations_loaded", + timestamp: TIME_RECOMMENDATION, + }, + ], + activityReceivedAt: includeActivity ? [TIME_ACTIVITY] : [], + preferenceReceivedAt: [TIME_PREFERENCE], + recommendationReceivedAt: [TIME_RECOMMENDATION], + }, + journey: { reloadObserved: true }, + }, + }; +} + +function onBundle(): ExternalBundle { + const runId = "article-atlas-passing-on"; + const preference = preferenceState(runId, "on"); + const activity = activityState(runId); + + return { + schemaVersion: SUPPORTED_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + evidence: { + scenario: "on", + runId, + userId: USER_ID, + ui: { + preference: "on", + toggleChecked: true, + feedFunctional: true, + }, + storage: { preference: "on" }, + request: { + activityPayloads: [activity.payload], + preferenceUpdates: [preference.request], + }, + response: { + preferenceUpdates: [preference.response], + }, + backend: { + preference: "on", + activityReceipts: [activity.receipt], + recommendationReceipts: [ + { + kind: "recommendation", + receiptId: `${runId}-recommendation-receipt`, + sequence: 3, + receivedAt: TIME_RECOMMENDATION, + source: "behavioral", + userId: USER_ID, + items: [recommendationItem()], + }, + ], + preferenceReceipts: [preference.receipt], + }, + recommendation: { + source: "behavioral", + itemIds: [ITEM_ID], + }, + timestamps: { + clientTimeline: [ + { + sequence: 1, + event: "preference_restored", + timestamp: TIME_PREFERENCE, + }, + { + sequence: 2, + event: "activity_recorded", + timestamp: TIME_ACTIVITY, + }, + { + sequence: 3, + event: "recommendations_loaded", + timestamp: TIME_RECOMMENDATION, + }, + ], + activityReceivedAt: [TIME_ACTIVITY], + preferenceReceivedAt: [TIME_PREFERENCE], + recommendationReceivedAt: [TIME_RECOMMENDATION], + }, + journey: { reloadObserved: true }, + }, + }; +} + +export const brokenOffExample = offBundle(true); +export const passingOffExample = offBundle(false); +export const passingOnExample = onBundle(); + +export const producerTemplate = `import { readFile, writeFile } from "node:fs/promises"; + +const [evidenceFile = "promise-evidence.json", outputFile = "bundle.json"] = + process.argv.slice(2); +const evidence = JSON.parse(await readFile(evidenceFile, "utf8")); +const bundle = { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence, +}; + +await writeFile( + outputFile, + \`\${JSON.stringify(bundle, null, 2)}\\n\`, + { encoding: "utf8", flag: "wx" }, +); +`; + +export const scaffoldReadme = `# PromiseProof external evidence scaffold + +This scaffold supports exactly one contract family: +\`activity-personalization/v1\`. + +Use \`broken-off.example.json\`, \`passing-off.example.json\`, and +\`passing-on.example.json\` as fixed-shape examples for producing a complete +external evidence bundle. \`producer-template.mjs\` is a small wrapper for a +complete \`PromiseEvidence\` JSON object; it does not collect or attest evidence. +Then verify one bundle: + +\`\`\`text +npm run promiseproof -- verify --evidence --out +\`\`\` + +Or gate OFF and ON together: + +\`\`\`text +npm run promiseproof -- gate --off --on --out +\`\`\` + +The OFF contract checks zero identifiable recommendation activity, a functional +contextual feed, and witnessed OFF persistence across UI, toggle, storage, and +backend state. The ON control checks correlated identifiable activity and a +functional behavioral feed. + +Evidence is externally supplied. PromiseProof does not attest how it was +collected. The deterministic PromiseProof evaluator alone evaluates a bundle +that passes strict validation. + +These examples prove externally supplied evidence ingestion for one contract +family. They do not prove arbitrary evidence collection or arbitrary promise +support. Article Atlas is a neutral synthetic example product. +`; diff --git a/src/verify/outcome.ts b/src/verify/outcome.ts new file mode 100644 index 0000000..695d39f --- /dev/null +++ b/src/verify/outcome.ts @@ -0,0 +1,17 @@ +export type ExternalOutcome = + | "PASS" + | "BROKEN_PROMISE" + | "INVALID_EVIDENCE" + | "EXECUTION_ERROR"; + +export const EXIT_CODE = { + PASS: 0, + EXECUTION_ERROR: 1, + BROKEN_PROMISE: 2, + INVALID_EVIDENCE: 3, +} as const satisfies Record; + +export const SUPPORTED_SCHEMA_VERSION = "1" as const; +export const SUPPORTED_CONTRACT_FAMILY = + "activity-personalization/v1" as const; +export const REPORT_SCHEMA_VERSION = "1" as const; diff --git a/src/verify/report.ts b/src/verify/report.ts new file mode 100644 index 0000000..bb977c4 --- /dev/null +++ b/src/verify/report.ts @@ -0,0 +1,188 @@ +import type { + PromiseClauseResult, + PromiseEvaluation, + PromiseViolation, +} from "../shared/types.js"; +import { + REPORT_SCHEMA_VERSION, + SUPPORTED_CONTRACT_FAMILY, +} from "./outcome.js"; +import type { GateResult, VerifyResult } from "./verify.js"; + +const authority = { + evidenceSource: "externally-supplied", + collectionAttested: false, + evaluation: "deterministic-promiseproof-evaluator", +} as const; + +export interface EvaluationReport { + readonly scenario: "on" | "off"; + readonly canonicalVerdict: "pass" | "fail"; + readonly clauses: readonly PromiseClauseResult[]; + readonly violations: readonly PromiseViolation[]; +} + +export interface VerifyReport extends EvaluationReport { + readonly schemaVersion: typeof REPORT_SCHEMA_VERSION; + readonly contractFamily: typeof SUPPORTED_CONTRACT_FAMILY; + readonly outcome: "PASS" | "BROKEN_PROMISE"; + readonly authority: typeof authority; +} + +export interface GateReport { + readonly schemaVersion: typeof REPORT_SCHEMA_VERSION; + readonly contractFamily: typeof SUPPORTED_CONTRACT_FAMILY; + readonly outcome: "PASS" | "BROKEN_PROMISE"; + readonly evaluations: { + readonly off: EvaluationReport; + readonly on: EvaluationReport; + }; + readonly authority: typeof authority; +} + +function requireEvaluation(result: VerifyResult): PromiseEvaluation { + if (result.evaluation === null) { + throw new Error("Cannot render a canonical report for invalid evidence."); + } + return result.evaluation; +} + +function evaluationReport(result: VerifyResult): EvaluationReport { + const evaluation = requireEvaluation(result); + if (result.scenario === null) { + throw new Error("Evaluated evidence is missing its scenario."); + } + + return { + scenario: result.scenario, + canonicalVerdict: evaluation.verdict, + clauses: evaluation.clauses.map((clause) => ({ ...clause })), + violations: evaluation.violations.map((violation) => ({ ...violation })), + }; +} + +export function createVerifyReport(result: VerifyResult): VerifyReport { + if ( + result.outcome !== "PASS" && + result.outcome !== "BROKEN_PROMISE" + ) { + throw new Error("Invalid evidence cannot be rendered as a product verdict."); + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + outcome: result.outcome, + ...evaluationReport(result), + authority, + }; +} + +export function createGateReport(result: GateResult): GateReport { + if ( + result.outcome !== "PASS" && + result.outcome !== "BROKEN_PROMISE" + ) { + throw new Error("Invalid gate input cannot be rendered as a product verdict."); + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + outcome: result.outcome, + evaluations: { + off: evaluationReport(result.off), + on: evaluationReport(result.on), + }, + authority, + }; +} + +export function serializeReportJson( + report: VerifyReport | GateReport, +): string { + return `${JSON.stringify(report, null, 2)}\n`; +} + +function markdownInline(value: string): string { + return value + .replaceAll("\\", "\\\\") + .replaceAll("|", "\\|") + .replaceAll("\r\n", " ") + .replaceAll("\n", " ") + .replaceAll("\r", " "); +} + +function evaluationMarkdown( + heading: string, + report: EvaluationReport, +): string[] { + const lines = [ + `## ${heading}`, + "", + `Scenario: ${report.scenario.toUpperCase()}`, + `Canonical verdict: ${report.canonicalVerdict}`, + "", + "| Clause | Result | Expected | Observed |", + "| --- | --- | --- | --- |", + ...report.clauses.map( + (clause) => + `| ${clause.id} | ${clause.passed ? "PASS" : "FAIL"} | ${markdownInline(clause.expected)} | ${markdownInline(clause.observed)} |`, + ), + "", + "### Violations", + "", + ]; + + if (report.violations.length === 0) { + lines.push("None."); + } else { + lines.push( + ...report.violations.map( + (violation) => + `- ${violation.code} (${violation.clause}): ${markdownInline(violation.message)}`, + ), + ); + } + + return lines; +} + +function authorityMarkdown(): string[] { + return [ + "## Authority", + "", + "Evidence source: externally supplied", + "Collection integrity: not attested by PromiseProof", + "Evaluation authority: deterministic PromiseProof evaluator", + ]; +} + +export function serializeVerifyReportMarkdown(report: VerifyReport): string { + return [ + "# PromiseProof verification report", + "", + `Contract family: ${report.contractFamily}`, + `Outcome: ${report.outcome}`, + "", + ...evaluationMarkdown("Evaluation", report), + "", + ...authorityMarkdown(), + "", + ].join("\n"); +} +export function serializeGateReportMarkdown(report: GateReport): string { + return [ + "# PromiseProof OFF + ON gate report", + "", + `Contract family: ${report.contractFamily}`, + `Outcome: ${report.outcome}`, + "", + ...evaluationMarkdown("OFF evaluation", report.evaluations.off), + "", + ...evaluationMarkdown("ON evaluation", report.evaluations.on), + "", + ...authorityMarkdown(), + "", + ].join("\n"); +} diff --git a/src/verify/schema.ts b/src/verify/schema.ts new file mode 100644 index 0000000..95eaeec --- /dev/null +++ b/src/verify/schema.ts @@ -0,0 +1,198 @@ +import { z } from "zod"; + +import { + SUPPORTED_CONTRACT_FAMILY, + SUPPORTED_SCHEMA_VERSION, +} from "./outcome.js"; + +export const MAX_INPUT_BYTES = 256 * 1024; +export const MAX_COLLECTION_ITEMS = 100; + +const identifier = z + .string() + .min(1) + .max(256) + .regex(/\S/, "Identifier must contain a non-whitespace character"); +const shortText = z.string().max(512); +const longText = z.string().max(2_048); +const isoTimestamp = z.iso.datetime({ offset: true }); +const sequence = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const preference = z.enum(["on", "off"]); +const recommendationSource = z.enum(["contextual", "behavioral"]); + +function boundedArray(schema: T) { + return z.array(schema).max(MAX_COLLECTION_ITEMS); +} + +const activityPayloadSchema = z + .object({ + runId: identifier, + userId: identifier, + eventType: z.literal("page_view"), + itemId: identifier, + clientSequence: sequence, + occurredAt: isoTimestamp, + }) + .strict(); + +const activityReceiptSchema = z + .object({ + kind: z.literal("activity"), + service: z.literal("recommendation"), + receiptId: identifier, + sequence, + receivedAt: isoTimestamp, + payload: activityPayloadSchema, + }) + .strict(); + +const recommendationItemSchema = z + .object({ + id: identifier, + title: shortText, + description: longText, + eyebrow: shortText, + }) + .strict(); + +const recommendationReceiptSchema = z + .object({ + kind: z.literal("recommendation"), + receiptId: identifier, + sequence, + receivedAt: isoTimestamp, + source: recommendationSource, + userId: identifier.optional(), + items: boundedArray(recommendationItemSchema), + }) + .strict(); + +const preferenceReceiptSchema = z + .object({ + kind: z.literal("preference"), + receiptId: identifier, + sequence, + receivedAt: isoTimestamp, + userId: identifier, + preference, + }) + .strict(); + +const preferenceUpdatePayloadSchema = z + .object({ + runId: identifier, + preference, + }) + .strict(); + +const preferenceUpdateRequestSchema = z + .object({ + targetUserId: identifier, + payload: preferenceUpdatePayloadSchema, + }) + .strict(); + +const preferenceUpdateResponseSchema = z + .object({ + userId: identifier, + preference, + updatedAt: isoTimestamp, + receipt: preferenceReceiptSchema, + }) + .strict(); + +const timelineDetailValueSchema = z.union([ + z.string().max(2_048), + z.number().finite(), + z.boolean(), + z.null(), +]); + +const clientTimelineEntrySchema = z + .object({ + sequence, + event: identifier, + timestamp: isoTimestamp, + detail: z.record(z.string().max(256), timelineDetailValueSchema).optional(), + }) + .strict(); + +export const promiseEvidenceSchema = z + .object({ + scenario: preference, + runId: identifier, + userId: identifier, + ui: z + .object({ + preference, + toggleChecked: z.boolean(), + feedFunctional: z.boolean(), + }) + .strict(), + storage: z + .object({ + preference: preference.nullable(), + }) + .strict(), + request: z + .object({ + activityPayloads: boundedArray(activityPayloadSchema), + preferenceUpdates: boundedArray(preferenceUpdateRequestSchema), + }) + .strict(), + response: z + .object({ + preferenceUpdates: boundedArray(preferenceUpdateResponseSchema), + }) + .strict(), + backend: z + .object({ + preference, + activityReceipts: boundedArray(activityReceiptSchema), + recommendationReceipts: boundedArray(recommendationReceiptSchema), + preferenceReceipts: boundedArray(preferenceReceiptSchema), + }) + .strict(), + recommendation: z + .object({ + source: recommendationSource, + itemIds: boundedArray(identifier), + }) + .strict(), + timestamps: z + .object({ + clientTimeline: boundedArray(clientTimelineEntrySchema), + activityReceivedAt: boundedArray(isoTimestamp), + preferenceReceivedAt: boundedArray(isoTimestamp), + recommendationReceivedAt: boundedArray(isoTimestamp), + }) + .strict(), + journey: z + .object({ + reloadObserved: z.boolean(), + }) + .strict(), + }) + .strict(); + +export const externalBundleSchema = z + .object({ + schemaVersion: z.literal(SUPPORTED_SCHEMA_VERSION), + contractFamily: z.literal(SUPPORTED_CONTRACT_FAMILY), + evidence: promiseEvidenceSchema, + }) + .strict() + .meta({ + id: "https://promiseproof.local/schemas/activity-personalization.v1.schema.json", + title: "PromiseProof activity-personalization/v1 evidence bundle", + description: + "Externally supplied evidence for the single activity-personalization/v1 contract family.", + }); + +export type ExternalBundle = z.infer; + +export function externalBundleJsonSchema(): z.core.JSONSchema.JSONSchema { + return z.toJSONSchema(externalBundleSchema, { + target: "draft-2020-12", + }); +} diff --git a/src/verify/verify.ts b/src/verify/verify.ts new file mode 100644 index 0000000..bf40364 --- /dev/null +++ b/src/verify/verify.ts @@ -0,0 +1,101 @@ +import { evaluatePromise } from "../shared/evaluator.js"; +import type { + PromiseEvaluation, + PromiseEvidence, +} from "../shared/types.js"; +import { + SUPPORTED_CONTRACT_FAMILY, + type ExternalOutcome, +} from "./outcome.js"; +import { externalBundleSchema } from "./schema.js"; + +export interface VerifyResult { + readonly outcome: Exclude; + readonly contractFamily: string; + readonly scenario: "on" | "off" | null; + readonly evaluation: PromiseEvaluation | null; + readonly issues: readonly string[]; +} + +function issuePath(path: PropertyKey[]): string { + if (path.length === 0) { + return ""; + } + + return path.map((part) => String(part)).join("."); +} + +export function verifyBundle(raw: unknown): VerifyResult { + const parsed = externalBundleSchema.safeParse(raw); + if (!parsed.success) { + return { + outcome: "INVALID_EVIDENCE", + contractFamily: SUPPORTED_CONTRACT_FAMILY, + scenario: null, + evaluation: null, + issues: parsed.error.issues + .map((issue) => `${issuePath(issue.path)}: ${issue.message}`) + .sort(), + }; + } + + const evidence: PromiseEvidence = parsed.data.evidence; + const evaluation = evaluatePromise(evidence); + return { + outcome: evaluation.verdict === "pass" ? "PASS" : "BROKEN_PROMISE", + contractFamily: parsed.data.contractFamily, + scenario: evidence.scenario, + evaluation, + issues: [], + }; +} + +export interface GateResult { + readonly outcome: Exclude; + readonly contractFamily: string; + readonly off: VerifyResult; + readonly on: VerifyResult; + readonly issues: readonly string[]; +} + +export function runGate(rawOff: unknown, rawOn: unknown): GateResult { + const off = verifyBundle(rawOff); + const on = verifyBundle(rawOn); + const issues: string[] = []; + + if (off.outcome !== "INVALID_EVIDENCE" && off.scenario !== "off") { + issues.push("--off bundle must contain OFF-scenario evidence"); + } + if (on.outcome !== "INVALID_EVIDENCE" && on.scenario !== "on") { + issues.push("--on bundle must contain ON-scenario evidence"); + } + + let outcome: GateResult["outcome"]; + if ( + off.outcome === "INVALID_EVIDENCE" || + on.outcome === "INVALID_EVIDENCE" || + issues.length > 0 + ) { + outcome = "INVALID_EVIDENCE"; + } else if (off.outcome === "PASS" && on.outcome === "PASS") { + outcome = "PASS"; + } else { + outcome = "BROKEN_PROMISE"; + } + + return { + outcome, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + off, + on, + issues: issues.sort(), + }; +} + +export function gateValidationIssues(result: GateResult): string[] { + return [ + ...result.issues, + ...result.off.issues.map((issue) => `off.${issue}`), + ...result.on.issues.map((issue) => `on.${issue}`), + ].sort(); +} diff --git a/tests/verify/cli.unit.ts b/tests/verify/cli.unit.ts new file mode 100644 index 0000000..171eedd --- /dev/null +++ b/tests/verify/cli.unit.ts @@ -0,0 +1,368 @@ +import assert from "node:assert/strict"; +import { + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test, { after } from "node:test"; + +import { + brokenOffExample, + passingOffExample, + passingOnExample, +} from "../../src/verify/examples.js"; +import { MAX_INPUT_BYTES } from "../../src/verify/schema.js"; + +const projectRoot = process.cwd(); +const temporaryRoots: string[] = []; + +interface CliResult { + readonly status: number | null; + readonly stdout: string; + readonly stderr: string; +} + +function clone(value: T): T { + return structuredClone(value); +} + +async function temporaryRoot(): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), "promiseproof-cli-test-")); + temporaryRoots.push(root); + return root; +} + +async function writeJson(filename: string, value: unknown): Promise { + await writeFile(filename, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function runCli(args: readonly string[]): CliResult { + const env = { ...process.env }; + delete env.OPENAI_API_KEY; + env.OPENAI_BASE_URL = "http://127.0.0.1:9/no-network"; + env.CODEX_HOME = path.join(os.tmpdir(), "promiseproof-no-codex-home"); + + const result = spawnSync( + process.execPath, + [ + path.join(projectRoot, "node_modules", "tsx", "dist", "cli.mjs"), + path.join(projectRoot, "src", "verify", "cli.ts"), + ...args, + ], + { + cwd: projectRoot, + env, + encoding: "utf8", + timeout: 30_000, + windowsHide: true, + }, + ); + + if (result.error) { + throw result.error; + } + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + }; +} + +after(async () => { + await Promise.all( + temporaryRoots.map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +test("CLI verifies broken OFF with exit 2 and deterministic reports", async () => { + const root = await temporaryRoot(); + const evidence = path.join(root, "broken-off.json"); + const output = path.join(root, "report"); + await writeJson(evidence, brokenOffExample); + + const result = runCli([ + "verify", + "--evidence", + evidence, + "--out", + output, + ]); + + assert.equal(result.status, 2); + assert.match(result.stdout, /^BROKEN_PROMISE/); + const report = JSON.parse( + await readFile(path.join(output, "report.json"), "utf8"), + ) as { outcome: string; violations: Array<{ code: string }> }; + assert.equal(report.outcome, "BROKEN_PROMISE"); + assert.deepEqual(report.violations.map((item) => item.code), [ + "PP_IDENTIFIABLE_EVENT_LEAK", + ]); +}); + +test("CLI verifies passing OFF and ON with exit 0 without API key, network, browser, or Codex", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "passing-off.json"); + const on = path.join(root, "passing-on.json"); + await writeJson(off, passingOffExample); + await writeJson(on, passingOnExample); + + for (const [filename, directory] of [ + [off, "off-report"], + [on, "on-report"], + ] as const) { + const result = runCli([ + "verify", + "--evidence", + filename, + "--out", + path.join(root, directory), + ]); + assert.equal(result.status, 0); + assert.match(result.stdout, /^PASS/); + assert.equal(result.stderr, ""); + } +}); + +test("CLI verifies broken ON with exit 2", async () => { + const root = await temporaryRoot(); + const brokenOn = clone(passingOnExample); + brokenOn.evidence.backend.activityReceipts = []; + const evidence = path.join(root, "broken-on.json"); + await writeJson(evidence, brokenOn); + + const result = runCli([ + "verify", + "--evidence", + evidence, + "--out", + path.join(root, "report"), + ]); + + assert.equal(result.status, 2); + assert.match(result.stdout, /^BROKEN_PROMISE/); +}); + +test("CLI gate passes only passing OFF plus passing ON", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "passing-off.json"); + const brokenOff = path.join(root, "broken-off.json"); + const on = path.join(root, "passing-on.json"); + await writeJson(off, passingOffExample); + await writeJson(brokenOff, brokenOffExample); + await writeJson(on, passingOnExample); + + const passing = runCli([ + "gate", + "--off", + off, + "--on", + on, + "--out", + path.join(root, "passing-gate"), + ]); + const broken = runCli([ + "gate", + "--off", + brokenOff, + "--on", + on, + "--out", + path.join(root, "broken-gate"), + ]); + + assert.equal(passing.status, 0); + assert.equal(broken.status, 2); + const gateReport = JSON.parse( + await readFile(path.join(root, "passing-gate", "report.json"), "utf8"), + ) as { + evaluations: { + off: { canonicalVerdict: string }; + on: { canonicalVerdict: string }; + }; + }; + assert.equal(gateReport.evaluations.off.canonicalVerdict, "pass"); + assert.equal(gateReport.evaluations.on.canonicalVerdict, "pass"); +}); + +test("CLI returns INVALID_EVIDENCE for malformed JSON and missing fields", async () => { + const root = await temporaryRoot(); + const malformed = path.join(root, "malformed.json"); + const missing = path.join(root, "missing.json"); + await writeFile(malformed, "{not json", "utf8"); + await writeJson(missing, { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence: {}, + }); + + for (const [filename, reportName] of [ + [malformed, "malformed-report"], + [missing, "missing-report"], + ] as const) { + const result = runCli([ + "verify", + "--evidence", + filename, + "--out", + path.join(root, reportName), + ]); + assert.equal(result.status, 3); + assert.match(result.stderr, /^INVALID_EVIDENCE/m); + } +}); + +test("CLI rejects unknown keys, schema versions, and contract families with exit 3", async () => { + const root = await temporaryRoot(); + const candidates = [ + Object.assign(clone(passingOffExample), { unexpected: true }), + Object.assign(clone(passingOffExample), { schemaVersion: "2" }), + Object.assign(clone(passingOffExample), { + contractFamily: "arbitrary-promise/v1", + }), + ]; + + for (const [index, candidate] of candidates.entries()) { + const evidence = path.join(root, `invalid-${index}.json`); + await writeJson(evidence, candidate); + const result = runCli([ + "verify", + "--evidence", + evidence, + "--out", + path.join(root, `report-${index}`), + ]); + assert.equal(result.status, 3); + assert.match(result.stderr, /^INVALID_EVIDENCE/m); + } +}); + +test("CLI rejects a wrong gate scenario slot with exit 3", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "off.json"); + const on = path.join(root, "on.json"); + await writeJson(off, passingOnExample); + await writeJson(on, passingOffExample); + + const result = runCli([ + "gate", + "--off", + off, + "--on", + on, + "--out", + path.join(root, "gate"), + ]); + + assert.equal(result.status, 3); + assert.match(result.stderr, /--off bundle must contain OFF-scenario evidence/); + assert.match(result.stderr, /--on bundle must contain ON-scenario evidence/); +}); + +test("CLI rejects oversized input and bounded collection overflow with exit 3", async () => { + const root = await temporaryRoot(); + const oversized = path.join(root, "oversized.json"); + const collection = path.join(root, "collection.json"); + await writeFile(oversized, " ".repeat(MAX_INPUT_BYTES + 1), "utf8"); + const tooMany = clone(passingOffExample); + tooMany.evidence.recommendation.itemIds = Array.from( + { length: 101 }, + (_, index) => `article-${index}`, + ); + await writeJson(collection, tooMany); + + for (const [filename, output] of [ + [oversized, "oversized-report"], + [collection, "collection-report"], + ] as const) { + const result = runCli([ + "verify", + "--evidence", + filename, + "--out", + path.join(root, output), + ]); + assert.equal(result.status, 3); + assert.match(result.stderr, /^INVALID_EVIDENCE/m); + } +}); + +test("CLI scaffold is complete and refuses conflicting files", async () => { + const root = await temporaryRoot(); + const output = path.join(root, "scaffold"); + + const first = runCli(["init", "--out", output]); + const second = runCli(["init", "--out", output]); + + assert.equal(first.status, 0); + assert.equal(second.status, 1); + assert.match(second.stderr, /Refusing to overwrite existing scaffold file/); + for (const filename of [ + "activity-personalization.v1.schema.json", + "broken-off.example.json", + "passing-off.example.json", + "passing-on.example.json", + "producer-template.mjs", + "README.md", + ]) { + assert.ok((await readFile(path.join(output, filename))).byteLength > 0); + } +}); + +test("CLI unknown command, missing argument, and output traversal are usage errors", async () => { + const unknown = runCli(["unknown"]); + const missing = runCli(["verify", "--evidence", "bundle.json"]); + const traversal = runCli(["init", "--out", "../escaped"]); + + for (const result of [unknown, missing, traversal]) { + assert.equal(result.status, 1); + assert.match(result.stderr, /USAGE_ERROR/); + } +}); + +test("CLI creates byte-identical JSON and Markdown across repeated runs", async () => { + const root = await temporaryRoot(); + const evidence = path.join(root, "passing-off.json"); + const first = path.join(root, "first"); + const second = path.join(root, "second"); + await writeJson(evidence, passingOffExample); + + const firstResult = runCli([ + "verify", + "--evidence", + evidence, + "--out", + first, + ]); + const secondResult = runCli([ + "verify", + "--evidence", + evidence, + "--out", + second, + ]); + + assert.equal(firstResult.status, 0); + assert.equal(secondResult.status, 0); + assert.deepEqual( + await readFile(path.join(first, "report.json")), + await readFile(path.join(second, "report.json")), + ); + assert.deepEqual( + await readFile(path.join(first, "report.md")), + await readFile(path.join(second, "report.md")), + ); +}); + +test("CLI help succeeds and documents commands and exit codes", () => { + const result = runCli(["--help"]); + + assert.equal(result.status, 0); + assert.match(result.stdout, /init --out/); + assert.match(result.stdout, /verify --evidence/); + assert.match(result.stdout, /gate --off/); + assert.match(result.stdout, /3 INVALID_EVIDENCE/); +}); diff --git a/tests/verify/core.unit.ts b/tests/verify/core.unit.ts new file mode 100644 index 0000000..beee124 --- /dev/null +++ b/tests/verify/core.unit.ts @@ -0,0 +1,326 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; + +import { evaluatePromise } from "../../src/shared/evaluator.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, + producerTemplate, + scaffoldReadme, +} from "../../src/verify/examples.js"; +import { + createGateReport, + createVerifyReport, + serializeGateReportMarkdown, + serializeReportJson, + serializeVerifyReportMarkdown, +} from "../../src/verify/report.js"; +import { + externalBundleJsonSchema, + MAX_COLLECTION_ITEMS, +} from "../../src/verify/schema.js"; +import { + runGate, + verifyBundle, +} from "../../src/verify/verify.js"; + +const projectRoot = process.cwd(); + +function clone(value: T): T { + return structuredClone(value); +} + +test("broken OFF maps the unchanged canonical failure to BROKEN_PROMISE", () => { + const result = verifyBundle(brokenOffExample); + + assert.equal(result.outcome, "BROKEN_PROMISE"); + assert.equal(result.evaluation?.verdict, "fail"); + assert.deepEqual( + result.evaluation?.violations.map((violation) => violation.code), + ["PP_IDENTIFIABLE_EVENT_LEAK"], + ); +}); + +test("passing OFF and passing ON map canonical passes to PASS", () => { + const off = verifyBundle(passingOffExample); + const on = verifyBundle(passingOnExample); + + assert.equal(off.outcome, "PASS"); + assert.equal(off.evaluation?.verdict, "pass"); + assert.equal(on.outcome, "PASS"); + assert.equal(on.evaluation?.verdict, "pass"); +}); + +test("broken ON maps the unchanged canonical failure to BROKEN_PROMISE", () => { + const brokenOn = clone(passingOnExample); + brokenOn.evidence.backend.activityReceipts = []; + + const result = verifyBundle(brokenOn); + assert.equal(result.outcome, "BROKEN_PROMISE"); + assert.deepEqual( + result.evaluation?.violations.map((violation) => violation.code), + ["PP_EXPECTED_ACTIVITY_MISSING"], + ); +}); + +test("gate requires passing OFF and ON evidence", () => { + const passing = runGate(passingOffExample, passingOnExample); + const broken = runGate(brokenOffExample, passingOnExample); + + assert.equal(passing.outcome, "PASS"); + assert.equal(broken.outcome, "BROKEN_PROMISE"); + assert.equal(broken.off.outcome, "BROKEN_PROMISE"); + assert.equal(broken.on.outcome, "PASS"); +}); + +test("gate rejects evidence supplied in the wrong scenario slot", () => { + const result = runGate(passingOnExample, passingOffExample); + + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.deepEqual(result.issues, [ + "--off bundle must contain OFF-scenario evidence", + "--on bundle must contain ON-scenario evidence", + ]); + assert.equal(result.off.evaluation?.verdict, "pass"); + assert.equal(result.on.evaluation?.verdict, "pass"); +}); + +test("strict validation rejects missing fields, unknown fields, and unsupported envelope values", () => { + const missing = clone(passingOffExample) as Record; + delete (missing.evidence as Record).journey; + + const unknown = clone(passingOffExample) as Record; + (unknown.evidence as Record).unexpected = true; + const nestedUnknown = clone(passingOffExample); + Object.assign(nestedUnknown.evidence.ui, { unexpected: true }); + + const version = { + ...clone(passingOffExample), + schemaVersion: "2", + }; + const family = { + ...clone(passingOffExample), + contractFamily: "generic-consent/v1", + }; + + for (const candidate of [ + missing, + unknown, + nestedUnknown, + version, + family, + ]) { + const result = verifyBundle(candidate); + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.evaluation, null); + assert.ok(result.issues.length > 0); + } +}); + +test("validation rejects malformed identifiers and timestamps without coercion", () => { + const identifier = clone(passingOffExample); + identifier.evidence.runId = " "; + const timestamp = clone(passingOffExample); + timestamp.evidence.timestamps.preferenceReceivedAt = ["not-a-timestamp"]; + + assert.equal(verifyBundle(identifier).outcome, "INVALID_EVIDENCE"); + assert.equal(verifyBundle(timestamp).outcome, "INVALID_EVIDENCE"); + assert.equal(identifier.evidence.runId, " "); +}); + +test("bounded collections are rejected before canonical evaluation", () => { + const oversized = clone(passingOffExample); + oversized.evidence.timestamps.activityReceivedAt = Array.from( + { length: MAX_COLLECTION_ITEMS + 1 }, + () => "2026-01-15T12:00:01.000Z", + ); + + const result = verifyBundle(oversized); + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.evaluation, null); + assert.match(result.issues.join("\n"), /Too big/); +}); + +test("valid evidence reaches the unchanged evaluator only after validation", () => { + const invalid = { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence: {}, + }; + const rejected = verifyBundle(invalid); + const accepted = verifyBundle(passingOffExample); + + assert.equal(rejected.outcome, "INVALID_EVIDENCE"); + assert.equal(rejected.evaluation, null); + assert.deepEqual( + accepted.evaluation, + evaluatePromise(passingOffExample.evidence), + ); +}); + +test("single reports are deterministic with canonical clause and violation ordering", () => { + const broken = verifyBundle(brokenOffExample); + const first = createVerifyReport(broken); + const second = createVerifyReport(verifyBundle(clone(brokenOffExample))); + + assert.deepEqual( + first.clauses.map((clause) => clause.id), + [ + "no_identifiable_activity", + "contextual_feed_functional", + "preference_survives_reload", + ], + ); + assert.deepEqual( + first.violations.map((violation) => violation.code), + ["PP_IDENTIFIABLE_EVENT_LEAK"], + ); + assert.equal(serializeReportJson(first), serializeReportJson(second)); + assert.equal( + serializeVerifyReportMarkdown(first), + serializeVerifyReportMarkdown(second), + ); +}); + +test("gate reports preserve separate deterministic OFF and ON evaluations", () => { + const first = createGateReport( + runGate(passingOffExample, passingOnExample), + ); + const second = createGateReport( + runGate(clone(passingOffExample), clone(passingOnExample)), + ); + + assert.deepEqual( + first.evaluations.off.clauses.map((clause) => clause.id), + [ + "no_identifiable_activity", + "contextual_feed_functional", + "preference_survives_reload", + ], + ); + assert.deepEqual( + first.evaluations.on.clauses.map((clause) => clause.id), + ["expected_activity_received", "behavioral_feed_functional"], + ); + assert.equal(serializeReportJson(first), serializeReportJson(second)); + assert.equal( + serializeGateReportMarkdown(first), + serializeGateReportMarkdown(second), + ); +}); + +test("reports disclose external evidence authority and contain no machine metadata", () => { + const report = serializeReportJson( + createVerifyReport(verifyBundle(passingOffExample)), + ); + const markdown = serializeVerifyReportMarkdown( + createVerifyReport(verifyBundle(passingOffExample)), + ); + + assert.match(report, /"evidenceSource": "externally-supplied"/); + assert.match(report, /"collectionAttested": false/); + assert.match( + report, + /"evaluation": "deterministic-promiseproof-evaluator"/, + ); + assert.match(markdown, /Evidence source: externally supplied/); + assert.match( + markdown, + /Collection integrity: not attested by PromiseProof/, + ); + assert.match( + markdown, + /Evaluation authority: deterministic PromiseProof evaluator/, + ); + assert.doesNotMatch(report, /generatedAt|hostname|username|[A-Z]:\\/i); +}); + +test("invalid evidence cannot be rendered as a canonical report", () => { + const invalid = verifyBundle({ malformed: true }); + + assert.equal(invalid.outcome, "INVALID_EVIDENCE"); + assert.throws( + () => createVerifyReport(invalid), + /Invalid evidence cannot be rendered as a product verdict/, + ); +}); + +test("committed JSON Schema is generated byte-for-byte from the runtime Zod schema", async () => { + const committed = await readFile( + path.join( + projectRoot, + "artifacts", + "verify", + "activity-personalization.v1.schema.json", + ), + "utf8", + ); + const generated = `${JSON.stringify(externalBundleJsonSchema(), null, 2)}\n`; + + assert.equal(committed, generated); +}); + +test("committed scaffold text artifacts match the runtime scaffold exactly", async () => { + const artifactRoot = path.join(projectRoot, "artifacts", "verify"); + + assert.equal( + await readFile(path.join(artifactRoot, "producer-template.mjs"), "utf8"), + producerTemplate, + ); + assert.equal( + await readFile(path.join(artifactRoot, "README.md"), "utf8"), + scaffoldReadme, + ); +}); + +test("Article Atlas examples are independent and limited to the supported family", () => { + const serialized = JSON.stringify([ + brokenOffExample, + passingOffExample, + passingOnExample, + ]); + + assert.match(serialized, /article-atlas/i); + assert.doesNotMatch( + serialized, + /signal shelf|initialization-race|propagation-failure|fixture|src\//i, + ); + assert.equal(brokenOffExample.contractFamily, "activity-personalization/v1"); + assert.equal(passingOffExample.contractFamily, "activity-personalization/v1"); + assert.equal(passingOnExample.contractFamily, "activity-personalization/v1"); +}); + +test("external verifier modules have no forbidden implementation imports", async () => { + const verifyDirectory = path.join(projectRoot, "src", "verify"); + const filenames = [ + "cli.ts", + "examples.ts", + "outcome.ts", + "report.ts", + "schema.ts", + "verify.ts", + ]; + const forbidden = + /from\s+["'][^"']*(client|server|investigation|repair|judge|tests|docs\/evidence|artifacts|@openai|openai|codex|playwright)[^"']*["']/; + + for (const filename of filenames) { + const source = await readFile(path.join(verifyDirectory, filename), "utf8"); + assert.doesNotMatch(source, forbidden, filename); + } + + const verifierSource = await readFile( + path.join(verifyDirectory, "verify.ts"), + "utf8", + ); + assert.match( + verifierSource, + /import \{ evaluatePromise \} from "\.\.\/shared\/evaluator\.js"/, + ); + assert.ok( + verifierSource.indexOf("if (!parsed.success)") < + verifierSource.indexOf("evaluatePromise(evidence)"), + ); +}); From ceb9b03e4ed4abe3a2047d016472885f91a8efc0 Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Mon, 20 Jul 2026 21:26:26 +0100 Subject: [PATCH 2/7] fix: harden external verifier contract --- artifacts/verify/README.md | 15 +- .../activity-personalization.v1.schema.json | 606 ++++-------------- artifacts/verify/broken-off.example.json | 130 +--- artifacts/verify/passing-off.example.json | 109 +--- artifacts/verify/passing-on.example.json | 137 +--- artifacts/verify/producer-template.mjs | 2 +- src/verify/adapter.ts | 104 +++ src/verify/examples.ts | 265 +++----- src/verify/report.ts | 4 + src/verify/schema.ts | 170 ++--- src/verify/verify.ts | 147 ++++- tests/verify/canonical-projection.ts | 226 +++++++ tests/verify/cli.unit.ts | 4 +- tests/verify/core.unit.ts | 541 +++++++++++----- 14 files changed, 1171 insertions(+), 1289 deletions(-) create mode 100644 src/verify/adapter.ts create mode 100644 tests/verify/canonical-projection.ts diff --git a/artifacts/verify/README.md b/artifacts/verify/README.md index dbd6dd9..01b74cb 100644 --- a/artifacts/verify/README.md +++ b/artifacts/verify/README.md @@ -3,11 +3,18 @@ This scaffold supports exactly one contract family: `activity-personalization/v1`. +The public `ExternalEvidenceV1` contract contains only facts used by the +unchanged PromiseProof evaluator: + +- scenario and subject identifier; +- UI, toggle, storage, backend, and reload control state; +- captured activities and recommendation-service activity receipts; +- rendered feed state and recommendation-service recommendation receipts. + Use `broken-off.example.json`, `passing-off.example.json`, and -`passing-on.example.json` as fixed-shape examples for producing a complete -external evidence bundle. `producer-template.mjs` is a small wrapper for a -complete `PromiseEvidence` JSON object; it does not collect or attest evidence. -Then verify one bundle: +`passing-on.example.json` as fixed-shape examples. `producer-template.mjs` +wraps an `ExternalEvidenceV1` JSON object; it does not collect or attest +evidence. Then verify one bundle: ```text npm run promiseproof -- verify --evidence --out diff --git a/artifacts/verify/activity-personalization.v1.schema.json b/artifacts/verify/activity-personalization.v1.schema.json index 1228500..4df6cfa 100644 --- a/artifacts/verify/activity-personalization.v1.schema.json +++ b/artifacts/verify/activity-personalization.v1.schema.json @@ -20,22 +20,23 @@ "off" ] }, - "runId": { + "subjectId": { "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" - }, - "userId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] }, - "ui": { + "control": { "type": "object", "properties": { - "preference": { + "uiPreference": { "type": "string", "enum": [ "on", @@ -45,21 +46,7 @@ "toggleChecked": { "type": "boolean" }, - "feedFunctional": { - "type": "boolean" - } - }, - "required": [ - "preference", - "toggleChecked", - "feedFunctional" - ], - "additionalProperties": false - }, - "storage": { - "type": "object", - "properties": { - "preference": { + "storedPreference": { "anyOf": [ { "type": "string", @@ -72,17 +59,31 @@ "type": "null" } ] + }, + "backendPreference": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "reloadObserved": { + "type": "boolean" } }, "required": [ - "preference" + "uiPreference", + "toggleChecked", + "storedPreference", + "backendPreference", + "reloadObserved" ], "additionalProperties": false }, - "request": { + "activity": { "type": "object", "properties": { - "activityPayloads": { + "capturedActivities": { "maxItems": 100, "type": "array", "items": { @@ -92,13 +93,27 @@ "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] }, - "userId": { + "subjectId": { "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] }, "eventType": { "type": "string", @@ -108,7 +123,14 @@ "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] }, "clientSequence": { "type": "integer", @@ -123,7 +145,7 @@ }, "required": [ "runId", - "userId", + "subjectId", "eventType", "itemId", "clientSequence", @@ -132,535 +154,181 @@ "additionalProperties": false } }, - "preferenceUpdates": { + "recommendationServiceReceipts": { "maxItems": 100, "type": "array", "items": { "type": "object", "properties": { - "targetUserId": { + "runId": { "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" - }, - "payload": { - "type": "object", - "properties": { - "runId": { - "type": "string", - "minLength": 1, - "maxLength": 256, + "allOf": [ + { "pattern": "\\S" }, - "preference": { - "type": "string", - "enum": [ - "on", - "off" - ] + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" } - }, - "required": [ - "runId", - "preference" - ], - "additionalProperties": false - } - }, - "required": [ - "targetUserId", - "payload" - ], - "additionalProperties": false - } - } - }, - "required": [ - "activityPayloads", - "preferenceUpdates" - ], - "additionalProperties": false - }, - "response": { - "type": "object", - "properties": { - "preferenceUpdates": { - "maxItems": 100, - "type": "array", - "items": { - "type": "object", - "properties": { - "userId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "preference": { - "type": "string", - "enum": [ - "on", - "off" ] }, - "updatedAt": { + "subjectId": { "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "receipt": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "const": "preference" - }, - "receiptId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "sequence": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "receivedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "userId": { - "type": "string", - "minLength": 1, - "maxLength": 256, + "minLength": 1, + "maxLength": 256, + "allOf": [ + { "pattern": "\\S" }, - "preference": { - "type": "string", - "enum": [ - "on", - "off" - ] + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" } - }, - "required": [ - "kind", - "receiptId", - "sequence", - "receivedAt", - "userId", - "preference" - ], - "additionalProperties": false - } - }, - "required": [ - "userId", - "preference", - "updatedAt", - "receipt" - ], - "additionalProperties": false - } - } - }, - "required": [ - "preferenceUpdates" - ], - "additionalProperties": false - }, - "backend": { - "type": "object", - "properties": { - "preference": { - "type": "string", - "enum": [ - "on", - "off" - ] - }, - "activityReceipts": { - "maxItems": 100, - "type": "array", - "items": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "const": "activity" + ] }, - "service": { + "eventType": { "type": "string", - "const": "recommendation" + "const": "page_view" }, - "receiptId": { + "itemId": { "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" - }, - "sequence": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "receivedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "payload": { - "type": "object", - "properties": { - "runId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "userId": { - "type": "string", - "minLength": 1, - "maxLength": 256, + "allOf": [ + { "pattern": "\\S" }, - "eventType": { - "type": "string", - "const": "page_view" - }, - "itemId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "clientSequence": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "occurredAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" } - }, - "required": [ - "runId", - "userId", - "eventType", - "itemId", - "clientSequence", - "occurredAt" - ], - "additionalProperties": false - } - }, - "required": [ - "kind", - "service", - "receiptId", - "sequence", - "receivedAt", - "payload" - ], - "additionalProperties": false - } - }, - "recommendationReceipts": { - "maxItems": 100, - "type": "array", - "items": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "const": "recommendation" - }, - "receiptId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "sequence": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 - }, - "receivedAt": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "source": { - "type": "string", - "enum": [ - "contextual", - "behavioral" ] }, - "userId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "items": { - "maxItems": 100, - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "title": { - "type": "string", - "maxLength": 512 - }, - "description": { - "type": "string", - "maxLength": 2048 - }, - "eyebrow": { - "type": "string", - "maxLength": 512 - } - }, - "required": [ - "id", - "title", - "description", - "eyebrow" - ], - "additionalProperties": false - } - } - }, - "required": [ - "kind", - "receiptId", - "sequence", - "receivedAt", - "source", - "items" - ], - "additionalProperties": false - } - }, - "preferenceReceipts": { - "maxItems": 100, - "type": "array", - "items": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "const": "preference" - }, - "receiptId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "sequence": { + "clientSequence": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, - "receivedAt": { + "occurredAt": { "type": "string", "format": "date-time", "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - }, - "userId": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "\\S" - }, - "preference": { - "type": "string", - "enum": [ - "on", - "off" - ] } }, "required": [ - "kind", - "receiptId", - "sequence", - "receivedAt", - "userId", - "preference" + "runId", + "subjectId", + "eventType", + "itemId", + "clientSequence", + "occurredAt" ], "additionalProperties": false } } }, "required": [ - "preference", - "activityReceipts", - "recommendationReceipts", - "preferenceReceipts" + "capturedActivities", + "recommendationServiceReceipts" ], "additionalProperties": false }, - "recommendation": { + "recommendations": { "type": "object", "properties": { - "source": { + "feedFunctional": { + "type": "boolean" + }, + "renderedSource": { "type": "string", "enum": [ "contextual", "behavioral" ] }, - "itemIds": { + "renderedItemIds": { "maxItems": 100, "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] } - } - }, - "required": [ - "source", - "itemIds" - ], - "additionalProperties": false - }, - "timestamps": { - "type": "object", - "properties": { - "clientTimeline": { + }, + "recommendationServiceReceipts": { "maxItems": 100, "type": "array", "items": { "type": "object", "properties": { - "sequence": { - "type": "integer", - "minimum": 0, - "maximum": 9007199254740991 + "source": { + "type": "string", + "enum": [ + "contextual", + "behavioral" + ] }, - "event": { + "subjectId": { "type": "string", "minLength": 1, "maxLength": 256, - "pattern": "\\S" - }, - "timestamp": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] }, - "detail": { - "type": "object", - "propertyNames": { + "itemIds": { + "maxItems": 100, + "type": "array", + "items": { "type": "string", - "maxLength": 256 - }, - "additionalProperties": { - "anyOf": [ + "minLength": 1, + "maxLength": 256, + "allOf": [ { - "type": "string", - "maxLength": 2048 - }, - { - "type": "number" - }, - { - "type": "boolean" + "pattern": "\\S" }, { - "type": "null" + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" } ] } } }, "required": [ - "sequence", - "event", - "timestamp" + "source", + "itemIds" ], "additionalProperties": false } - }, - "activityReceivedAt": { - "maxItems": 100, - "type": "array", - "items": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - } - }, - "preferenceReceivedAt": { - "maxItems": 100, - "type": "array", - "items": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - } - }, - "recommendationReceivedAt": { - "maxItems": 100, - "type": "array", - "items": { - "type": "string", - "format": "date-time", - "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" - } } }, "required": [ - "clientTimeline", - "activityReceivedAt", - "preferenceReceivedAt", - "recommendationReceivedAt" - ], - "additionalProperties": false - }, - "journey": { - "type": "object", - "properties": { - "reloadObserved": { - "type": "boolean" - } - }, - "required": [ - "reloadObserved" + "feedFunctional", + "renderedSource", + "renderedItemIds", + "recommendationServiceReceipts" ], "additionalProperties": false } }, "required": [ "scenario", - "runId", - "userId", - "ui", - "storage", - "request", - "response", - "backend", - "recommendation", - "timestamps", - "journey" + "subjectId", + "control", + "activity", + "recommendations" ], "additionalProperties": false } @@ -672,5 +340,5 @@ ], "additionalProperties": false, "title": "PromiseProof activity-personalization/v1 evidence bundle", - "description": "Externally supplied evidence for the single activity-personalization/v1 contract family." + "description": "Externally supplied evaluator-relevant facts for the single activity-personalization/v1 contract family." } diff --git a/artifacts/verify/broken-off.example.json b/artifacts/verify/broken-off.example.json index 132a1cd..2cae644 100644 --- a/artifacts/verify/broken-off.example.json +++ b/artifacts/verify/broken-off.example.json @@ -3,132 +3,50 @@ "contractFamily": "activity-personalization/v1", "evidence": { "scenario": "off", - "runId": "article-atlas-broken-off", - "userId": "article-atlas-reader-001", - "ui": { - "preference": "off", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "off", "toggleChecked": false, - "feedFunctional": true - }, - "storage": { - "preference": "off" + "storedPreference": "off", + "backendPreference": "off", + "reloadObserved": true }, - "request": { - "activityPayloads": [ + "activity": { + "capturedActivities": [ { "runId": "article-atlas-broken-off", - "userId": "article-atlas-reader-001", + "subjectId": "article-atlas-reader-001", "eventType": "page_view", "itemId": "article-atlas-origin-001", "clientSequence": 1, "occurredAt": "2026-01-15T12:00:01.000Z" } ], - "preferenceUpdates": [ - { - "targetUserId": "article-atlas-reader-001", - "payload": { - "runId": "article-atlas-broken-off", - "preference": "off" - } - } - ] - }, - "response": { - "preferenceUpdates": [ + "recommendationServiceReceipts": [ { - "userId": "article-atlas-reader-001", - "preference": "off", - "updatedAt": "2026-01-15T12:00:00.000Z", - "receipt": { - "kind": "preference", - "receiptId": "article-atlas-broken-off-preference-receipt", - "sequence": 1, - "receivedAt": "2026-01-15T12:00:00.000Z", - "userId": "article-atlas-reader-001", - "preference": "off" - } + "runId": "article-atlas-broken-off", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" } ] }, - "backend": { - "preference": "off", - "activityReceipts": [ - { - "kind": "activity", - "service": "recommendation", - "receiptId": "article-atlas-broken-off-activity-receipt", - "sequence": 2, - "receivedAt": "2026-01-15T12:00:01.000Z", - "payload": { - "runId": "article-atlas-broken-off", - "userId": "article-atlas-reader-001", - "eventType": "page_view", - "itemId": "article-atlas-origin-001", - "clientSequence": 1, - "occurredAt": "2026-01-15T12:00:01.000Z" - } - } + "recommendations": { + "feedFunctional": true, + "renderedSource": "contextual", + "renderedItemIds": [ + "article-atlas-story-101" ], - "recommendationReceipts": [ + "recommendationServiceReceipts": [ { - "kind": "recommendation", - "receiptId": "article-atlas-broken-off-recommendation-receipt", - "sequence": 3, - "receivedAt": "2026-01-15T12:00:02.000Z", "source": "contextual", - "items": [ - { - "id": "article-atlas-story-101", - "title": "A field guide to urban trees", - "description": "A synthetic Article Atlas recommendation.", - "eyebrow": "Article Atlas" - } + "itemIds": [ + "article-atlas-story-101" ] } - ], - "preferenceReceipts": [ - { - "kind": "preference", - "receiptId": "article-atlas-broken-off-preference-receipt", - "sequence": 1, - "receivedAt": "2026-01-15T12:00:00.000Z", - "userId": "article-atlas-reader-001", - "preference": "off" - } - ] - }, - "recommendation": { - "source": "contextual", - "itemIds": [ - "article-atlas-story-101" ] - }, - "timestamps": { - "clientTimeline": [ - { - "sequence": 1, - "event": "preference_restored", - "timestamp": "2026-01-15T12:00:00.000Z" - }, - { - "sequence": 2, - "event": "recommendations_loaded", - "timestamp": "2026-01-15T12:00:02.000Z" - } - ], - "activityReceivedAt": [ - "2026-01-15T12:00:01.000Z" - ], - "preferenceReceivedAt": [ - "2026-01-15T12:00:00.000Z" - ], - "recommendationReceivedAt": [ - "2026-01-15T12:00:02.000Z" - ] - }, - "journey": { - "reloadObserved": true } } } diff --git a/artifacts/verify/passing-off.example.json b/artifacts/verify/passing-off.example.json index 6e92f86..7f9ef81 100644 --- a/artifacts/verify/passing-off.example.json +++ b/artifacts/verify/passing-off.example.json @@ -3,105 +3,32 @@ "contractFamily": "activity-personalization/v1", "evidence": { "scenario": "off", - "runId": "article-atlas-passing-off", - "userId": "article-atlas-reader-001", - "ui": { - "preference": "off", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "off", "toggleChecked": false, - "feedFunctional": true + "storedPreference": "off", + "backendPreference": "off", + "reloadObserved": true }, - "storage": { - "preference": "off" + "activity": { + "capturedActivities": [], + "recommendationServiceReceipts": [] }, - "request": { - "activityPayloads": [], - "preferenceUpdates": [ - { - "targetUserId": "article-atlas-reader-001", - "payload": { - "runId": "article-atlas-passing-off", - "preference": "off" - } - } - ] - }, - "response": { - "preferenceUpdates": [ - { - "userId": "article-atlas-reader-001", - "preference": "off", - "updatedAt": "2026-01-15T12:00:00.000Z", - "receipt": { - "kind": "preference", - "receiptId": "article-atlas-passing-off-preference-receipt", - "sequence": 1, - "receivedAt": "2026-01-15T12:00:00.000Z", - "userId": "article-atlas-reader-001", - "preference": "off" - } - } - ] - }, - "backend": { - "preference": "off", - "activityReceipts": [], - "recommendationReceipts": [ + "recommendations": { + "feedFunctional": true, + "renderedSource": "contextual", + "renderedItemIds": [ + "article-atlas-story-101" + ], + "recommendationServiceReceipts": [ { - "kind": "recommendation", - "receiptId": "article-atlas-passing-off-recommendation-receipt", - "sequence": 3, - "receivedAt": "2026-01-15T12:00:02.000Z", "source": "contextual", - "items": [ - { - "id": "article-atlas-story-101", - "title": "A field guide to urban trees", - "description": "A synthetic Article Atlas recommendation.", - "eyebrow": "Article Atlas" - } + "itemIds": [ + "article-atlas-story-101" ] } - ], - "preferenceReceipts": [ - { - "kind": "preference", - "receiptId": "article-atlas-passing-off-preference-receipt", - "sequence": 1, - "receivedAt": "2026-01-15T12:00:00.000Z", - "userId": "article-atlas-reader-001", - "preference": "off" - } - ] - }, - "recommendation": { - "source": "contextual", - "itemIds": [ - "article-atlas-story-101" ] - }, - "timestamps": { - "clientTimeline": [ - { - "sequence": 1, - "event": "preference_restored", - "timestamp": "2026-01-15T12:00:00.000Z" - }, - { - "sequence": 2, - "event": "recommendations_loaded", - "timestamp": "2026-01-15T12:00:02.000Z" - } - ], - "activityReceivedAt": [], - "preferenceReceivedAt": [ - "2026-01-15T12:00:00.000Z" - ], - "recommendationReceivedAt": [ - "2026-01-15T12:00:02.000Z" - ] - }, - "journey": { - "reloadObserved": true } } } diff --git a/artifacts/verify/passing-on.example.json b/artifacts/verify/passing-on.example.json index b9b6909..ce54a9e 100644 --- a/artifacts/verify/passing-on.example.json +++ b/artifacts/verify/passing-on.example.json @@ -3,138 +3,51 @@ "contractFamily": "activity-personalization/v1", "evidence": { "scenario": "on", - "runId": "article-atlas-passing-on", - "userId": "article-atlas-reader-001", - "ui": { - "preference": "on", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "on", "toggleChecked": true, - "feedFunctional": true - }, - "storage": { - "preference": "on" + "storedPreference": "on", + "backendPreference": "on", + "reloadObserved": true }, - "request": { - "activityPayloads": [ + "activity": { + "capturedActivities": [ { "runId": "article-atlas-passing-on", - "userId": "article-atlas-reader-001", + "subjectId": "article-atlas-reader-001", "eventType": "page_view", "itemId": "article-atlas-origin-001", "clientSequence": 1, "occurredAt": "2026-01-15T12:00:01.000Z" } ], - "preferenceUpdates": [ - { - "targetUserId": "article-atlas-reader-001", - "payload": { - "runId": "article-atlas-passing-on", - "preference": "on" - } - } - ] - }, - "response": { - "preferenceUpdates": [ + "recommendationServiceReceipts": [ { - "userId": "article-atlas-reader-001", - "preference": "on", - "updatedAt": "2026-01-15T12:00:00.000Z", - "receipt": { - "kind": "preference", - "receiptId": "article-atlas-passing-on-preference-receipt", - "sequence": 1, - "receivedAt": "2026-01-15T12:00:00.000Z", - "userId": "article-atlas-reader-001", - "preference": "on" - } + "runId": "article-atlas-passing-on", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" } ] }, - "backend": { - "preference": "on", - "activityReceipts": [ - { - "kind": "activity", - "service": "recommendation", - "receiptId": "article-atlas-passing-on-activity-receipt", - "sequence": 2, - "receivedAt": "2026-01-15T12:00:01.000Z", - "payload": { - "runId": "article-atlas-passing-on", - "userId": "article-atlas-reader-001", - "eventType": "page_view", - "itemId": "article-atlas-origin-001", - "clientSequence": 1, - "occurredAt": "2026-01-15T12:00:01.000Z" - } - } + "recommendations": { + "feedFunctional": true, + "renderedSource": "behavioral", + "renderedItemIds": [ + "article-atlas-story-101" ], - "recommendationReceipts": [ + "recommendationServiceReceipts": [ { - "kind": "recommendation", - "receiptId": "article-atlas-passing-on-recommendation-receipt", - "sequence": 3, - "receivedAt": "2026-01-15T12:00:02.000Z", "source": "behavioral", - "userId": "article-atlas-reader-001", - "items": [ - { - "id": "article-atlas-story-101", - "title": "A field guide to urban trees", - "description": "A synthetic Article Atlas recommendation.", - "eyebrow": "Article Atlas" - } + "subjectId": "article-atlas-reader-001", + "itemIds": [ + "article-atlas-story-101" ] } - ], - "preferenceReceipts": [ - { - "kind": "preference", - "receiptId": "article-atlas-passing-on-preference-receipt", - "sequence": 1, - "receivedAt": "2026-01-15T12:00:00.000Z", - "userId": "article-atlas-reader-001", - "preference": "on" - } - ] - }, - "recommendation": { - "source": "behavioral", - "itemIds": [ - "article-atlas-story-101" ] - }, - "timestamps": { - "clientTimeline": [ - { - "sequence": 1, - "event": "preference_restored", - "timestamp": "2026-01-15T12:00:00.000Z" - }, - { - "sequence": 2, - "event": "activity_recorded", - "timestamp": "2026-01-15T12:00:01.000Z" - }, - { - "sequence": 3, - "event": "recommendations_loaded", - "timestamp": "2026-01-15T12:00:02.000Z" - } - ], - "activityReceivedAt": [ - "2026-01-15T12:00:01.000Z" - ], - "preferenceReceivedAt": [ - "2026-01-15T12:00:00.000Z" - ], - "recommendationReceivedAt": [ - "2026-01-15T12:00:02.000Z" - ] - }, - "journey": { - "reloadObserved": true } } } diff --git a/artifacts/verify/producer-template.mjs b/artifacts/verify/producer-template.mjs index c7d48da..ead64c3 100644 --- a/artifacts/verify/producer-template.mjs +++ b/artifacts/verify/producer-template.mjs @@ -1,6 +1,6 @@ import { readFile, writeFile } from "node:fs/promises"; -const [evidenceFile = "promise-evidence.json", outputFile = "bundle.json"] = +const [evidenceFile = "external-evidence.json", outputFile = "bundle.json"] = process.argv.slice(2); const evidence = JSON.parse(await readFile(evidenceFile, "utf8")); const bundle = { diff --git a/src/verify/adapter.ts b/src/verify/adapter.ts new file mode 100644 index 0000000..297ac5d --- /dev/null +++ b/src/verify/adapter.ts @@ -0,0 +1,104 @@ +import type { + ActivityPayload, + PromiseEvidence, + RecommendationReceipt, +} from "../shared/types.js"; +import type { + ExternalActivity, + ExternalEvidenceV1, + ExternalRecommendationReceipt, +} from "./schema.js"; + +const PLACEHOLDER_RUN_ID = "promiseproof-external-adapter"; +const PLACEHOLDER_TIMESTAMP = "2000-01-01T00:00:00.000Z"; +const PLACEHOLDER_TEXT = "Not part of externally supplied evidence"; + +function adaptActivity(activity: ExternalActivity): ActivityPayload { + return { + runId: activity.runId, + userId: activity.subjectId, + eventType: activity.eventType, + itemId: activity.itemId, + clientSequence: activity.clientSequence, + occurredAt: activity.occurredAt, + }; +} + +function adaptRecommendationReceipt( + receipt: ExternalRecommendationReceipt, + index: number, +): RecommendationReceipt { + return { + kind: "recommendation", + receiptId: `external-recommendation-receipt-${index + 1}`, + sequence: index + 1, + receivedAt: PLACEHOLDER_TIMESTAMP, + source: receipt.source, + ...(receipt.subjectId === undefined + ? {} + : { userId: receipt.subjectId }), + items: receipt.itemIds.map((id) => ({ + id, + title: PLACEHOLDER_TEXT, + description: PLACEHOLDER_TEXT, + eyebrow: PLACEHOLDER_TEXT, + })), + }; +} + +export function adaptExternalEvidence( + external: ExternalEvidenceV1, +): PromiseEvidence { + return { + scenario: external.scenario, + runId: PLACEHOLDER_RUN_ID, + userId: external.subjectId, + ui: { + preference: external.control.uiPreference, + toggleChecked: external.control.toggleChecked, + feedFunctional: external.recommendations.feedFunctional, + }, + storage: { + preference: external.control.storedPreference, + }, + request: { + activityPayloads: external.activity.capturedActivities.map(adaptActivity), + preferenceUpdates: [], + }, + response: { + preferenceUpdates: [], + }, + backend: { + preference: external.control.backendPreference, + activityReceipts: + external.activity.recommendationServiceReceipts.map( + (activity, index) => ({ + kind: "activity", + service: "recommendation", + receiptId: `external-activity-receipt-${index + 1}`, + sequence: index + 1, + receivedAt: PLACEHOLDER_TIMESTAMP, + payload: adaptActivity(activity), + }), + ), + recommendationReceipts: + external.recommendations.recommendationServiceReceipts.map( + adaptRecommendationReceipt, + ), + preferenceReceipts: [], + }, + recommendation: { + source: external.recommendations.renderedSource, + itemIds: [...external.recommendations.renderedItemIds], + }, + timestamps: { + clientTimeline: [], + activityReceivedAt: [], + preferenceReceivedAt: [], + recommendationReceivedAt: [], + }, + journey: { + reloadObserved: external.control.reloadObserved, + }, + }; +} diff --git a/src/verify/examples.ts b/src/verify/examples.ts index 499014d..70839cc 100644 --- a/src/verify/examples.ts +++ b/src/verify/examples.ts @@ -1,219 +1,111 @@ -import type { ExternalBundle } from "./schema.js"; +import type { + ExternalActivity, + ExternalBundle, + ExternalEvidenceV1, +} from "./schema.js"; import { SUPPORTED_CONTRACT_FAMILY, SUPPORTED_SCHEMA_VERSION, } from "./outcome.js"; -const USER_ID = "article-atlas-reader-001"; +const SUBJECT_ID = "article-atlas-reader-001"; const ITEM_ID = "article-atlas-story-101"; -const TIME_PREFERENCE = "2026-01-15T12:00:00.000Z"; const TIME_ACTIVITY = "2026-01-15T12:00:01.000Z"; -const TIME_RECOMMENDATION = "2026-01-15T12:00:02.000Z"; -function recommendationItem() { +function activity(runId: string): ExternalActivity { return { - id: ITEM_ID, - title: "A field guide to urban trees", - description: "A synthetic Article Atlas recommendation.", - eyebrow: "Article Atlas", - }; -} - -function preferenceState(runId: string, preference: "on" | "off") { - const receipt = { - kind: "preference" as const, - receiptId: `${runId}-preference-receipt`, - sequence: 1, - receivedAt: TIME_PREFERENCE, - userId: USER_ID, - preference, - }; - - return { - request: { - targetUserId: USER_ID, - payload: { runId, preference }, - }, - response: { - userId: USER_ID, - preference, - updatedAt: TIME_PREFERENCE, - receipt, - }, - receipt, - }; -} - -function activityState(runId: string) { - const payload = { runId, - userId: USER_ID, - eventType: "page_view" as const, + subjectId: SUBJECT_ID, + eventType: "page_view", itemId: "article-atlas-origin-001", clientSequence: 1, occurredAt: TIME_ACTIVITY, }; +} +function bundle(evidence: ExternalEvidenceV1): ExternalBundle { return { - payload, - receipt: { - kind: "activity" as const, - service: "recommendation" as const, - receiptId: `${runId}-activity-receipt`, - sequence: 2, - receivedAt: TIME_ACTIVITY, - payload, - }, + schemaVersion: SUPPORTED_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + evidence, }; } -function offBundle(includeActivity: boolean): ExternalBundle { +function offEvidence(includeActivity: boolean): ExternalEvidenceV1 { const runId = includeActivity ? "article-atlas-broken-off" : "article-atlas-passing-off"; - const preference = preferenceState(runId, "off"); - const activity = activityState(runId); + const observedActivity = activity(runId); return { - schemaVersion: SUPPORTED_SCHEMA_VERSION, - contractFamily: SUPPORTED_CONTRACT_FAMILY, - evidence: { - scenario: "off", - runId, - userId: USER_ID, - ui: { - preference: "off", - toggleChecked: false, - feedFunctional: true, - }, - storage: { preference: "off" }, - request: { - activityPayloads: includeActivity ? [activity.payload] : [], - preferenceUpdates: [preference.request], - }, - response: { - preferenceUpdates: [preference.response], - }, - backend: { - preference: "off", - activityReceipts: includeActivity ? [activity.receipt] : [], - recommendationReceipts: [ - { - kind: "recommendation", - receiptId: `${runId}-recommendation-receipt`, - sequence: 3, - receivedAt: TIME_RECOMMENDATION, - source: "contextual", - items: [recommendationItem()], - }, - ], - preferenceReceipts: [preference.receipt], - }, - recommendation: { - source: "contextual", - itemIds: [ITEM_ID], - }, - timestamps: { - clientTimeline: [ - { - sequence: 1, - event: "preference_restored", - timestamp: TIME_PREFERENCE, - }, - { - sequence: 2, - event: "recommendations_loaded", - timestamp: TIME_RECOMMENDATION, - }, - ], - activityReceivedAt: includeActivity ? [TIME_ACTIVITY] : [], - preferenceReceivedAt: [TIME_PREFERENCE], - recommendationReceivedAt: [TIME_RECOMMENDATION], - }, - journey: { reloadObserved: true }, + scenario: "off", + subjectId: SUBJECT_ID, + control: { + uiPreference: "off", + toggleChecked: false, + storedPreference: "off", + backendPreference: "off", + reloadObserved: true, + }, + activity: { + capturedActivities: includeActivity ? [{ ...observedActivity }] : [], + recommendationServiceReceipts: includeActivity + ? [{ ...observedActivity }] + : [], + }, + recommendations: { + feedFunctional: true, + renderedSource: "contextual", + renderedItemIds: [ITEM_ID], + recommendationServiceReceipts: [ + { + source: "contextual", + itemIds: [ITEM_ID], + }, + ], }, }; } -function onBundle(): ExternalBundle { - const runId = "article-atlas-passing-on"; - const preference = preferenceState(runId, "on"); - const activity = activityState(runId); +function onEvidence(): ExternalEvidenceV1 { + const observedActivity = activity("article-atlas-passing-on"); return { - schemaVersion: SUPPORTED_SCHEMA_VERSION, - contractFamily: SUPPORTED_CONTRACT_FAMILY, - evidence: { - scenario: "on", - runId, - userId: USER_ID, - ui: { - preference: "on", - toggleChecked: true, - feedFunctional: true, - }, - storage: { preference: "on" }, - request: { - activityPayloads: [activity.payload], - preferenceUpdates: [preference.request], - }, - response: { - preferenceUpdates: [preference.response], - }, - backend: { - preference: "on", - activityReceipts: [activity.receipt], - recommendationReceipts: [ - { - kind: "recommendation", - receiptId: `${runId}-recommendation-receipt`, - sequence: 3, - receivedAt: TIME_RECOMMENDATION, - source: "behavioral", - userId: USER_ID, - items: [recommendationItem()], - }, - ], - preferenceReceipts: [preference.receipt], - }, - recommendation: { - source: "behavioral", - itemIds: [ITEM_ID], - }, - timestamps: { - clientTimeline: [ - { - sequence: 1, - event: "preference_restored", - timestamp: TIME_PREFERENCE, - }, - { - sequence: 2, - event: "activity_recorded", - timestamp: TIME_ACTIVITY, - }, - { - sequence: 3, - event: "recommendations_loaded", - timestamp: TIME_RECOMMENDATION, - }, - ], - activityReceivedAt: [TIME_ACTIVITY], - preferenceReceivedAt: [TIME_PREFERENCE], - recommendationReceivedAt: [TIME_RECOMMENDATION], - }, - journey: { reloadObserved: true }, + scenario: "on", + subjectId: SUBJECT_ID, + control: { + uiPreference: "on", + toggleChecked: true, + storedPreference: "on", + backendPreference: "on", + reloadObserved: true, + }, + activity: { + capturedActivities: [{ ...observedActivity }], + recommendationServiceReceipts: [{ ...observedActivity }], + }, + recommendations: { + feedFunctional: true, + renderedSource: "behavioral", + renderedItemIds: [ITEM_ID], + recommendationServiceReceipts: [ + { + source: "behavioral", + subjectId: SUBJECT_ID, + itemIds: [ITEM_ID], + }, + ], }, }; } -export const brokenOffExample = offBundle(true); -export const passingOffExample = offBundle(false); -export const passingOnExample = onBundle(); +export const brokenOffExample = bundle(offEvidence(true)); +export const passingOffExample = bundle(offEvidence(false)); +export const passingOnExample = bundle(onEvidence()); export const producerTemplate = `import { readFile, writeFile } from "node:fs/promises"; -const [evidenceFile = "promise-evidence.json", outputFile = "bundle.json"] = +const [evidenceFile = "external-evidence.json", outputFile = "bundle.json"] = process.argv.slice(2); const evidence = JSON.parse(await readFile(evidenceFile, "utf8")); const bundle = { @@ -234,11 +126,18 @@ export const scaffoldReadme = `# PromiseProof external evidence scaffold This scaffold supports exactly one contract family: \`activity-personalization/v1\`. +The public \`ExternalEvidenceV1\` contract contains only facts used by the +unchanged PromiseProof evaluator: + +- scenario and subject identifier; +- UI, toggle, storage, backend, and reload control state; +- captured activities and recommendation-service activity receipts; +- rendered feed state and recommendation-service recommendation receipts. + Use \`broken-off.example.json\`, \`passing-off.example.json\`, and -\`passing-on.example.json\` as fixed-shape examples for producing a complete -external evidence bundle. \`producer-template.mjs\` is a small wrapper for a -complete \`PromiseEvidence\` JSON object; it does not collect or attest evidence. -Then verify one bundle: +\`passing-on.example.json\` as fixed-shape examples. \`producer-template.mjs\` +wraps an \`ExternalEvidenceV1\` JSON object; it does not collect or attest +evidence. Then verify one bundle: \`\`\`text npm run promiseproof -- verify --evidence --out diff --git a/src/verify/report.ts b/src/verify/report.ts index bb977c4..e2798a8 100644 --- a/src/verify/report.ts +++ b/src/verify/report.ts @@ -106,6 +106,10 @@ export function serializeReportJson( function markdownInline(value: string): string { return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("`", "`") .replaceAll("\\", "\\\\") .replaceAll("|", "\\|") .replaceAll("\r\n", " ") diff --git a/src/verify/schema.ts b/src/verify/schema.ts index 95eaeec..7feb5a6 100644 --- a/src/verify/schema.ts +++ b/src/verify/schema.ts @@ -8,13 +8,18 @@ import { export const MAX_INPUT_BYTES = 256 * 1024; export const MAX_COLLECTION_ITEMS = 100; -const identifier = z +const SAFE_VISIBLE_CHARACTERS = + /^[^\u0000-\u001f\u007f\u202a-\u202e\u2066-\u2069]*$/u; + +const visibleIdentifier = z .string() .min(1) .max(256) - .regex(/\S/, "Identifier must contain a non-whitespace character"); -const shortText = z.string().max(512); -const longText = z.string().max(2_048); + .regex(/\S/, "Identifier must contain a non-whitespace character") + .regex( + SAFE_VISIBLE_CHARACTERS, + "Developer-visible text contains a forbidden control or bidi character", + ); const isoTimestamp = z.iso.datetime({ offset: true }); const sequence = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); const preference = z.enum(["on", "off"]); @@ -24,152 +29,52 @@ function boundedArray(schema: T) { return z.array(schema).max(MAX_COLLECTION_ITEMS); } -const activityPayloadSchema = z +export const externalActivitySchema = z .object({ - runId: identifier, - userId: identifier, + runId: visibleIdentifier, + subjectId: visibleIdentifier, eventType: z.literal("page_view"), - itemId: identifier, + itemId: visibleIdentifier, clientSequence: sequence, occurredAt: isoTimestamp, }) .strict(); -const activityReceiptSchema = z +export const externalRecommendationReceiptSchema = z .object({ - kind: z.literal("activity"), - service: z.literal("recommendation"), - receiptId: identifier, - sequence, - receivedAt: isoTimestamp, - payload: activityPayloadSchema, - }) - .strict(); - -const recommendationItemSchema = z - .object({ - id: identifier, - title: shortText, - description: longText, - eyebrow: shortText, - }) - .strict(); - -const recommendationReceiptSchema = z - .object({ - kind: z.literal("recommendation"), - receiptId: identifier, - sequence, - receivedAt: isoTimestamp, source: recommendationSource, - userId: identifier.optional(), - items: boundedArray(recommendationItemSchema), - }) - .strict(); - -const preferenceReceiptSchema = z - .object({ - kind: z.literal("preference"), - receiptId: identifier, - sequence, - receivedAt: isoTimestamp, - userId: identifier, - preference, + subjectId: visibleIdentifier.optional(), + itemIds: boundedArray(visibleIdentifier), }) .strict(); -const preferenceUpdatePayloadSchema = z - .object({ - runId: identifier, - preference, - }) - .strict(); - -const preferenceUpdateRequestSchema = z - .object({ - targetUserId: identifier, - payload: preferenceUpdatePayloadSchema, - }) - .strict(); - -const preferenceUpdateResponseSchema = z - .object({ - userId: identifier, - preference, - updatedAt: isoTimestamp, - receipt: preferenceReceiptSchema, - }) - .strict(); - -const timelineDetailValueSchema = z.union([ - z.string().max(2_048), - z.number().finite(), - z.boolean(), - z.null(), -]); - -const clientTimelineEntrySchema = z - .object({ - sequence, - event: identifier, - timestamp: isoTimestamp, - detail: z.record(z.string().max(256), timelineDetailValueSchema).optional(), - }) - .strict(); - -export const promiseEvidenceSchema = z +export const externalEvidenceV1Schema = z .object({ scenario: preference, - runId: identifier, - userId: identifier, - ui: z + subjectId: visibleIdentifier, + control: z .object({ - preference, + uiPreference: preference, toggleChecked: z.boolean(), - feedFunctional: z.boolean(), - }) - .strict(), - storage: z - .object({ - preference: preference.nullable(), - }) - .strict(), - request: z - .object({ - activityPayloads: boundedArray(activityPayloadSchema), - preferenceUpdates: boundedArray(preferenceUpdateRequestSchema), - }) - .strict(), - response: z - .object({ - preferenceUpdates: boundedArray(preferenceUpdateResponseSchema), - }) - .strict(), - backend: z - .object({ - preference, - activityReceipts: boundedArray(activityReceiptSchema), - recommendationReceipts: boundedArray(recommendationReceiptSchema), - preferenceReceipts: boundedArray(preferenceReceiptSchema), - }) - .strict(), - recommendation: z - .object({ - source: recommendationSource, - itemIds: boundedArray(identifier), + storedPreference: preference.nullable(), + backendPreference: preference, + reloadObserved: z.boolean(), }) .strict(), - timestamps: z + activity: z .object({ - clientTimeline: boundedArray(clientTimelineEntrySchema), - activityReceivedAt: boundedArray(isoTimestamp), - preferenceReceivedAt: boundedArray(isoTimestamp), - recommendationReceivedAt: boundedArray(isoTimestamp), + capturedActivities: boundedArray(externalActivitySchema), + recommendationServiceReceipts: boundedArray(externalActivitySchema), }) .strict(), - journey: z + recommendations: z .object({ - reloadObserved: z.boolean(), + feedFunctional: z.boolean(), + renderedSource: recommendationSource, + renderedItemIds: boundedArray(visibleIdentifier), + recommendationServiceReceipts: boundedArray( + externalRecommendationReceiptSchema, + ), }) .strict(), }) @@ -179,16 +84,21 @@ export const externalBundleSchema = z .object({ schemaVersion: z.literal(SUPPORTED_SCHEMA_VERSION), contractFamily: z.literal(SUPPORTED_CONTRACT_FAMILY), - evidence: promiseEvidenceSchema, + evidence: externalEvidenceV1Schema, }) .strict() .meta({ id: "https://promiseproof.local/schemas/activity-personalization.v1.schema.json", title: "PromiseProof activity-personalization/v1 evidence bundle", description: - "Externally supplied evidence for the single activity-personalization/v1 contract family.", + "Externally supplied evaluator-relevant facts for the single activity-personalization/v1 contract family.", }); +export type ExternalActivity = z.infer; +export type ExternalRecommendationReceipt = z.infer< + typeof externalRecommendationReceiptSchema +>; +export type ExternalEvidenceV1 = z.infer; export type ExternalBundle = z.infer; export function externalBundleJsonSchema(): z.core.JSONSchema.JSONSchema { diff --git a/src/verify/verify.ts b/src/verify/verify.ts index bf40364..b188bca 100644 --- a/src/verify/verify.ts +++ b/src/verify/verify.ts @@ -3,11 +3,17 @@ import type { PromiseEvaluation, PromiseEvidence, } from "../shared/types.js"; +import { adaptExternalEvidence } from "./adapter.js"; import { SUPPORTED_CONTRACT_FAMILY, type ExternalOutcome, } from "./outcome.js"; -import { externalBundleSchema } from "./schema.js"; +import { + externalBundleSchema, + type ExternalBundle, +} from "./schema.js"; + +type CanonicalEvaluator = (evidence: PromiseEvidence) => PromiseEvaluation; export interface VerifyResult { readonly outcome: Exclude; @@ -17,6 +23,18 @@ export interface VerifyResult { readonly issues: readonly string[]; } +interface ValidatedBundle { + readonly success: true; + readonly bundle: ExternalBundle; +} + +interface RejectedBundle { + readonly success: false; + readonly issues: readonly string[]; +} + +type BundleValidation = ValidatedBundle | RejectedBundle; + function issuePath(path: PropertyKey[]): string { if (path.length === 0) { return ""; @@ -25,31 +43,63 @@ function issuePath(path: PropertyKey[]): string { return path.map((part) => String(part)).join("."); } -export function verifyBundle(raw: unknown): VerifyResult { +export function validateExternalBundle(raw: unknown): BundleValidation { const parsed = externalBundleSchema.safeParse(raw); if (!parsed.success) { return { - outcome: "INVALID_EVIDENCE", - contractFamily: SUPPORTED_CONTRACT_FAMILY, - scenario: null, - evaluation: null, + success: false, issues: parsed.error.issues .map((issue) => `${issuePath(issue.path)}: ${issue.message}`) .sort(), }; } - const evidence: PromiseEvidence = parsed.data.evidence; - const evaluation = evaluatePromise(evidence); + return { + success: true, + bundle: parsed.data, + }; +} + +function rejectedResult(validation: BundleValidation): VerifyResult { + return { + outcome: "INVALID_EVIDENCE", + contractFamily: SUPPORTED_CONTRACT_FAMILY, + scenario: validation.success ? validation.bundle.evidence.scenario : null, + evaluation: null, + issues: [...(validation.success ? [] : validation.issues)].sort(), + }; +} + +function evaluateValidatedBundle( + validated: ValidatedBundle, + evaluator: CanonicalEvaluator, +): VerifyResult { + const evidence = adaptExternalEvidence(validated.bundle.evidence); + const evaluation = evaluator(evidence); return { outcome: evaluation.verdict === "pass" ? "PASS" : "BROKEN_PROMISE", - contractFamily: parsed.data.contractFamily, - scenario: evidence.scenario, + contractFamily: validated.bundle.contractFamily, + scenario: validated.bundle.evidence.scenario, evaluation, issues: [], }; } +function verifyBundleWithEvaluator( + raw: unknown, + evaluator: CanonicalEvaluator, +): VerifyResult { + const validated = validateExternalBundle(raw); + if (!validated.success) { + return rejectedResult(validated); + } + return evaluateValidatedBundle(validated, evaluator); +} + +export function verifyBundle(raw: unknown): VerifyResult { + return verifyBundleWithEvaluator(raw, evaluatePromise); +} + export interface GateResult { readonly outcome: Exclude; readonly contractFamily: string; @@ -58,44 +108,73 @@ export interface GateResult { readonly issues: readonly string[]; } -export function runGate(rawOff: unknown, rawOn: unknown): GateResult { - const off = verifyBundle(rawOff); - const on = verifyBundle(rawOn); - const issues: string[] = []; +function runGateWithEvaluator( + rawOff: unknown, + rawOn: unknown, + evaluator: CanonicalEvaluator, +): GateResult { + const validatedOff = validateExternalBundle(rawOff); + const validatedOn = validateExternalBundle(rawOn); + const gateIssues: string[] = []; + const offSlotIssues: string[] = []; + const onSlotIssues: string[] = []; - if (off.outcome !== "INVALID_EVIDENCE" && off.scenario !== "off") { - issues.push("--off bundle must contain OFF-scenario evidence"); + if ( + validatedOff.success && + validatedOff.bundle.evidence.scenario !== "off" + ) { + offSlotIssues.push("--off bundle must contain OFF-scenario evidence"); } - if (on.outcome !== "INVALID_EVIDENCE" && on.scenario !== "on") { - issues.push("--on bundle must contain ON-scenario evidence"); + if ( + validatedOn.success && + validatedOn.bundle.evidence.scenario !== "on" + ) { + onSlotIssues.push("--on bundle must contain ON-scenario evidence"); } - let outcome: GateResult["outcome"]; + gateIssues.push(...offSlotIssues, ...onSlotIssues); if ( - off.outcome === "INVALID_EVIDENCE" || - on.outcome === "INVALID_EVIDENCE" || - issues.length > 0 + !validatedOff.success || + !validatedOn.success || + gateIssues.length > 0 ) { - outcome = "INVALID_EVIDENCE"; - } else if (off.outcome === "PASS" && on.outcome === "PASS") { - outcome = "PASS"; - } else { - outcome = "BROKEN_PROMISE"; + const off = rejectedResult(validatedOff); + const on = rejectedResult(validatedOn); + return { + outcome: "INVALID_EVIDENCE", + contractFamily: SUPPORTED_CONTRACT_FAMILY, + off, + on, + issues: [ + ...gateIssues, + ...off.issues.map((issue) => `off.${issue}`), + ...on.issues.map((issue) => `on.${issue}`), + ].sort(), + }; } + const off = evaluateValidatedBundle(validatedOff, evaluator); + const on = evaluateValidatedBundle(validatedOn, evaluator); return { - outcome, + outcome: + off.outcome === "PASS" && on.outcome === "PASS" + ? "PASS" + : "BROKEN_PROMISE", contractFamily: SUPPORTED_CONTRACT_FAMILY, off, on, - issues: issues.sort(), + issues: [], }; } +export function runGate(rawOff: unknown, rawOn: unknown): GateResult { + return runGateWithEvaluator(rawOff, rawOn, evaluatePromise); +} + +export const verifierTestHooks = Object.freeze({ + runGateWithEvaluator, +}); + export function gateValidationIssues(result: GateResult): string[] { - return [ - ...result.issues, - ...result.off.issues.map((issue) => `off.${issue}`), - ...result.on.issues.map((issue) => `on.${issue}`), - ].sort(); + return [...result.issues]; } diff --git a/tests/verify/canonical-projection.ts b/tests/verify/canonical-projection.ts new file mode 100644 index 0000000..d36624d --- /dev/null +++ b/tests/verify/canonical-projection.ts @@ -0,0 +1,226 @@ +import type { + ActivityPayload, + PromiseEvidence, + RecommendationReceipt, +} from "../../src/shared/types.js"; +import type { ExternalEvidenceV1 } from "../../src/verify/schema.js"; + +const USER_ID = "canonical-reader-001"; +const ITEM_ID = "canonical-item-001"; +const ACTIVITY_TIME = "2026-01-15T12:00:01.000Z"; +const RECEIPT_TIME = "2026-01-15T12:00:02.000Z"; + +function activity(runId: string): ActivityPayload { + return { + runId, + userId: USER_ID, + eventType: "page_view", + itemId: "canonical-origin-001", + clientSequence: 1, + occurredAt: ACTIVITY_TIME, + }; +} + +function recommendationReceipt( + source: "contextual" | "behavioral", + userId: string | undefined, + itemIds: string[], +): RecommendationReceipt { + return { + kind: "recommendation", + receiptId: `canonical-${source}-receipt`, + sequence: 2, + receivedAt: RECEIPT_TIME, + source, + ...(userId === undefined ? {} : { userId }), + items: itemIds.map((id) => ({ + id, + title: "Canonical fixture item", + description: "Canonical fixture description", + eyebrow: "Canonical fixture", + })), + }; +} + +function canonicalOff(runId: string): PromiseEvidence { + return { + scenario: "off", + runId, + userId: USER_ID, + ui: { + preference: "off", + toggleChecked: false, + feedFunctional: true, + }, + storage: { preference: "off" }, + request: { + activityPayloads: [], + preferenceUpdates: [], + }, + response: { preferenceUpdates: [] }, + backend: { + preference: "off", + activityReceipts: [], + recommendationReceipts: [ + recommendationReceipt("contextual", undefined, [ITEM_ID]), + ], + preferenceReceipts: [], + }, + recommendation: { + source: "contextual", + itemIds: [ITEM_ID], + }, + timestamps: { + clientTimeline: [], + activityReceivedAt: [], + preferenceReceivedAt: [], + recommendationReceivedAt: [], + }, + journey: { reloadObserved: true }, + }; +} + +function canonicalOn(runId: string): PromiseEvidence { + const observedActivity = activity(runId); + return { + scenario: "on", + runId, + userId: USER_ID, + ui: { + preference: "on", + toggleChecked: true, + feedFunctional: true, + }, + storage: { preference: "on" }, + request: { + activityPayloads: [observedActivity], + preferenceUpdates: [], + }, + response: { preferenceUpdates: [] }, + backend: { + preference: "on", + activityReceipts: [ + { + kind: "activity", + service: "recommendation", + receiptId: "canonical-activity-receipt", + sequence: 1, + receivedAt: RECEIPT_TIME, + payload: observedActivity, + }, + ], + recommendationReceipts: [ + recommendationReceipt("behavioral", USER_ID, [ITEM_ID]), + ], + preferenceReceipts: [], + }, + recommendation: { + source: "behavioral", + itemIds: [ITEM_ID], + }, + timestamps: { + clientTimeline: [], + activityReceivedAt: [RECEIPT_TIME], + preferenceReceivedAt: [], + recommendationReceivedAt: [RECEIPT_TIME], + }, + journey: { reloadObserved: true }, + }; +} + +const passingOff = canonicalOff("canonical-passing-off"); +const initializationRace = structuredClone(passingOff); +initializationRace.runId = "canonical-initialization-race"; +const leakedActivity = activity(initializationRace.runId); +initializationRace.request.activityPayloads = [leakedActivity]; +initializationRace.backend.activityReceipts = [ + { + kind: "activity", + service: "recommendation", + receiptId: "canonical-leak-receipt", + sequence: 1, + receivedAt: RECEIPT_TIME, + payload: leakedActivity, + }, +]; + +const propagationFailure = structuredClone(passingOff); +propagationFailure.runId = "canonical-propagation-failure"; +propagationFailure.backend.preference = "on"; + +const passingOn = canonicalOn("canonical-passing-on"); +const brokenOnMissingActivity = structuredClone(passingOn); +brokenOnMissingActivity.runId = "canonical-broken-on"; +brokenOnMissingActivity.backend.activityReceipts = []; + +const contextualFeedFailure = structuredClone(passingOff); +contextualFeedFailure.runId = "canonical-contextual-feed-failure"; +contextualFeedFailure.backend.recommendationReceipts = []; + +const behavioralFeedFailure = structuredClone(passingOn); +behavioralFeedFailure.runId = "canonical-behavioral-feed-failure"; +behavioralFeedFailure.backend.recommendationReceipts = []; + +const reloadPersistenceFailure = structuredClone(passingOff); +reloadPersistenceFailure.runId = "canonical-reload-failure"; +reloadPersistenceFailure.journey.reloadObserved = false; + +export const canonicalCases = { + initializationRace, + passingOff, + propagationFailure, + passingOn, + brokenOnMissingActivity, + contextualFeedFailure, + behavioralFeedFailure, + reloadPersistenceFailure, +} as const; + +export function projectCanonicalEvidence( + evidence: PromiseEvidence, +): ExternalEvidenceV1 { + return { + scenario: evidence.scenario, + subjectId: evidence.userId, + control: { + uiPreference: evidence.ui.preference, + toggleChecked: evidence.ui.toggleChecked, + storedPreference: evidence.storage.preference, + backendPreference: evidence.backend.preference, + reloadObserved: evidence.journey.reloadObserved, + }, + activity: { + capturedActivities: evidence.request.activityPayloads.map((payload) => ({ + runId: payload.runId, + subjectId: payload.userId, + eventType: payload.eventType, + itemId: payload.itemId, + clientSequence: payload.clientSequence, + occurredAt: payload.occurredAt, + })), + recommendationServiceReceipts: evidence.backend.activityReceipts.map( + (receipt) => ({ + runId: receipt.payload.runId, + subjectId: receipt.payload.userId, + eventType: receipt.payload.eventType, + itemId: receipt.payload.itemId, + clientSequence: receipt.payload.clientSequence, + occurredAt: receipt.payload.occurredAt, + }), + ), + }, + recommendations: { + feedFunctional: evidence.ui.feedFunctional, + renderedSource: evidence.recommendation.source, + renderedItemIds: [...evidence.recommendation.itemIds], + recommendationServiceReceipts: + evidence.backend.recommendationReceipts.map((receipt) => ({ + source: receipt.source, + ...(receipt.userId === undefined + ? {} + : { subjectId: receipt.userId }), + itemIds: receipt.items.map((item) => item.id), + })), + }, + }; +} diff --git a/tests/verify/cli.unit.ts b/tests/verify/cli.unit.ts index 171eedd..7d3fded 100644 --- a/tests/verify/cli.unit.ts +++ b/tests/verify/cli.unit.ts @@ -130,7 +130,7 @@ test("CLI verifies passing OFF and ON with exit 0 without API key, network, brow test("CLI verifies broken ON with exit 2", async () => { const root = await temporaryRoot(); const brokenOn = clone(passingOnExample); - brokenOn.evidence.backend.activityReceipts = []; + brokenOn.evidence.activity.recommendationServiceReceipts = []; const evidence = path.join(root, "broken-on.json"); await writeJson(evidence, brokenOn); @@ -268,7 +268,7 @@ test("CLI rejects oversized input and bounded collection overflow with exit 3", const collection = path.join(root, "collection.json"); await writeFile(oversized, " ".repeat(MAX_INPUT_BYTES + 1), "utf8"); const tooMany = clone(passingOffExample); - tooMany.evidence.recommendation.itemIds = Array.from( + tooMany.evidence.recommendations.renderedItemIds = Array.from( { length: 101 }, (_, index) => `article-${index}`, ); diff --git a/tests/verify/core.unit.ts b/tests/verify/core.unit.ts index beee124..b71dcc6 100644 --- a/tests/verify/core.unit.ts +++ b/tests/verify/core.unit.ts @@ -4,6 +4,7 @@ import path from "node:path"; import test from "node:test"; import { evaluatePromise } from "../../src/shared/evaluator.js"; +import { adaptExternalEvidence } from "../../src/verify/adapter.js"; import { brokenOffExample, passingOffExample, @@ -17,15 +18,22 @@ import { serializeGateReportMarkdown, serializeReportJson, serializeVerifyReportMarkdown, + type VerifyReport, } from "../../src/verify/report.js"; import { externalBundleJsonSchema, MAX_COLLECTION_ITEMS, + type ExternalBundle, } from "../../src/verify/schema.js"; import { runGate, + verifierTestHooks, verifyBundle, } from "../../src/verify/verify.js"; +import { + canonicalCases, + projectCanonicalEvidence, +} from "./canonical-projection.js"; const projectRoot = process.cwd(); @@ -33,78 +41,127 @@ function clone(value: T): T { return structuredClone(value); } -test("broken OFF maps the unchanged canonical failure to BROKEN_PROMISE", () => { - const result = verifyBundle(brokenOffExample); +function stable(value: unknown): string { + return JSON.stringify(value); +} + +test("broken OFF, passing OFF, passing ON, and broken ON preserve canonical outcomes", () => { + const brokenOff = verifyBundle(brokenOffExample); + const passingOff = verifyBundle(passingOffExample); + const passingOn = verifyBundle(passingOnExample); + const brokenOnBundle = clone(passingOnExample); + brokenOnBundle.evidence.activity.recommendationServiceReceipts = []; + const brokenOn = verifyBundle(brokenOnBundle); - assert.equal(result.outcome, "BROKEN_PROMISE"); - assert.equal(result.evaluation?.verdict, "fail"); + assert.equal(brokenOff.outcome, "BROKEN_PROMISE"); assert.deepEqual( - result.evaluation?.violations.map((violation) => violation.code), + brokenOff.evaluation?.violations.map((item) => item.code), ["PP_IDENTIFIABLE_EVENT_LEAK"], ); -}); - -test("passing OFF and passing ON map canonical passes to PASS", () => { - const off = verifyBundle(passingOffExample); - const on = verifyBundle(passingOnExample); - - assert.equal(off.outcome, "PASS"); - assert.equal(off.evaluation?.verdict, "pass"); - assert.equal(on.outcome, "PASS"); - assert.equal(on.evaluation?.verdict, "pass"); -}); - -test("broken ON maps the unchanged canonical failure to BROKEN_PROMISE", () => { - const brokenOn = clone(passingOnExample); - brokenOn.evidence.backend.activityReceipts = []; - - const result = verifyBundle(brokenOn); - assert.equal(result.outcome, "BROKEN_PROMISE"); + assert.equal(passingOff.outcome, "PASS"); + assert.equal(passingOn.outcome, "PASS"); + assert.equal(brokenOn.outcome, "BROKEN_PROMISE"); assert.deepEqual( - result.evaluation?.violations.map((violation) => violation.code), + brokenOn.evaluation?.violations.map((item) => item.code), ["PP_EXPECTED_ACTIVITY_MISSING"], ); }); -test("gate requires passing OFF and ON evidence", () => { - const passing = runGate(passingOffExample, passingOnExample); - const broken = runGate(brokenOffExample, passingOnExample); +test("gate evaluates exactly twice only after both inputs and slots are valid", () => { + let calls = 0; + const result = verifierTestHooks.runGateWithEvaluator( + passingOffExample, + passingOnExample, + (evidence) => { + calls += 1; + return evaluatePromise(evidence); + }, + ); + + assert.equal(result.outcome, "PASS"); + assert.equal(calls, 2); +}); + +test("invalid gates atomically invoke no canonical evaluation", () => { + const invalidCases: Array<[string, unknown, unknown]> = [ + ["OFF bundle in ON slot", passingOnExample, passingOnExample], + ["ON bundle in OFF slot", passingOffExample, passingOffExample], + ["malformed OFF", { malformed: true }, passingOnExample], + ["malformed ON", passingOffExample, { malformed: true }], + [ + "unsupported version", + { ...clone(passingOffExample), schemaVersion: "2" }, + passingOnExample, + ], + [ + "unsupported family", + passingOffExample, + { + ...clone(passingOnExample), + contractFamily: "generic-consent/v1", + }, + ], + ]; - assert.equal(passing.outcome, "PASS"); - assert.equal(broken.outcome, "BROKEN_PROMISE"); - assert.equal(broken.off.outcome, "BROKEN_PROMISE"); - assert.equal(broken.on.outcome, "PASS"); + for (const [name, off, on] of invalidCases) { + let calls = 0; + const result = verifierTestHooks.runGateWithEvaluator( + off, + on, + (evidence) => { + calls += 1; + return evaluatePromise(evidence); + }, + ); + + assert.equal(result.outcome, "INVALID_EVIDENCE", name); + assert.equal(result.off.evaluation, null, name); + assert.equal(result.on.evaluation, null, name); + assert.equal(calls, 0, name); + assert.ok(result.issues.length > 0, name); + assert.deepEqual([...result.issues], [...result.issues].sort(), name); + assert.deepEqual( + [...result.off.issues], + [...result.off.issues].sort(), + name, + ); + assert.deepEqual( + [...result.on.issues], + [...result.on.issues].sort(), + name, + ); + } }); -test("gate rejects evidence supplied in the wrong scenario slot", () => { +test("swapped gate slots remain unevaluated", () => { const result = runGate(passingOnExample, passingOffExample); assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.off.evaluation, null); + assert.equal(result.on.evaluation, null); assert.deepEqual(result.issues, [ "--off bundle must contain OFF-scenario evidence", "--on bundle must contain ON-scenario evidence", ]); - assert.equal(result.off.evaluation?.verdict, "pass"); - assert.equal(result.on.evaluation?.verdict, "pass"); }); -test("strict validation rejects missing fields, unknown fields, and unsupported envelope values", () => { +test("strict minimal validation rejects missing, unknown, unsupported, and malformed fields", () => { const missing = clone(passingOffExample) as Record; - delete (missing.evidence as Record).journey; - + delete (missing.evidence as Record).control; const unknown = clone(passingOffExample) as Record; (unknown.evidence as Record).unexpected = true; const nestedUnknown = clone(passingOffExample); - Object.assign(nestedUnknown.evidence.ui, { unexpected: true }); - - const version = { - ...clone(passingOffExample), - schemaVersion: "2", - }; + Object.assign(nestedUnknown.evidence.control, { unexpected: true }); + const version = { ...clone(passingOffExample), schemaVersion: "2" }; const family = { ...clone(passingOffExample), contractFamily: "generic-consent/v1", }; + const whitespace = clone(passingOffExample); + whitespace.evidence.subjectId = " "; + const timestamp = clone(passingOnExample); + timestamp.evidence.activity.capturedActivities[0]!.occurredAt = + "not-a-timestamp"; for (const candidate of [ missing, @@ -112,6 +169,8 @@ test("strict validation rejects missing fields, unknown fields, and unsupported nestedUnknown, version, family, + whitespace, + timestamp, ]) { const result = verifyBundle(candidate); assert.equal(result.outcome, "INVALID_EVIDENCE"); @@ -120,22 +179,41 @@ test("strict validation rejects missing fields, unknown fields, and unsupported } }); -test("validation rejects malformed identifiers and timestamps without coercion", () => { - const identifier = clone(passingOffExample); - identifier.evidence.runId = " "; - const timestamp = clone(passingOffExample); - timestamp.evidence.timestamps.preferenceReceivedAt = ["not-a-timestamp"]; +test("ASCII controls and Unicode bidi override/isolate characters are rejected", () => { + const forbidden = [ + "\u0000", + "\u0008", + "\t", + "\n", + "\r", + "\u007f", + "\u202a", + "\u202b", + "\u202c", + "\u202d", + "\u202e", + "\u2066", + "\u2067", + "\u2068", + "\u2069", + ]; - assert.equal(verifyBundle(identifier).outcome, "INVALID_EVIDENCE"); - assert.equal(verifyBundle(timestamp).outcome, "INVALID_EVIDENCE"); - assert.equal(identifier.evidence.runId, " "); + for (const character of forbidden) { + const candidate = clone(passingOnExample); + candidate.evidence.subjectId = `reader${character}name`; + assert.equal( + verifyBundle(candidate).outcome, + "INVALID_EVIDENCE", + `U+${character.codePointAt(0)!.toString(16).padStart(4, "0")}`, + ); + } }); test("bounded collections are rejected before canonical evaluation", () => { const oversized = clone(passingOffExample); - oversized.evidence.timestamps.activityReceivedAt = Array.from( + oversized.evidence.recommendations.renderedItemIds = Array.from( { length: MAX_COLLECTION_ITEMS + 1 }, - () => "2026-01-15T12:00:01.000Z", + (_, index) => `article-${index}`, ); const result = verifyBundle(oversized); @@ -144,57 +222,48 @@ test("bounded collections are rejected before canonical evaluation", () => { assert.match(result.issues.join("\n"), /Too big/); }); -test("valid evidence reaches the unchanged evaluator only after validation", () => { - const invalid = { - schemaVersion: "1", - contractFamily: "activity-personalization/v1", - evidence: {}, - }; - const rejected = verifyBundle(invalid); - const accepted = verifyBundle(passingOffExample); - - assert.equal(rejected.outcome, "INVALID_EVIDENCE"); - assert.equal(rejected.evaluation, null); - assert.deepEqual( - accepted.evaluation, - evaluatePromise(passingOffExample.evidence), - ); +test("Markdown encodes HTML, ampersands, backticks, pipes, and backslashes deterministically", () => { + const adversarial = clone(passingOnExample); + const subject = "&`pipe|slash\\"; + adversarial.evidence.subjectId = subject; + adversarial.evidence.activity.capturedActivities[0]!.subjectId = subject; + adversarial.evidence.activity.recommendationServiceReceipts[0]!.subjectId = + subject; + adversarial.evidence.recommendations.recommendationServiceReceipts[0]!.subjectId = + subject; + + const first = createVerifyReport(verifyBundle(adversarial)); + const second = createVerifyReport(verifyBundle(clone(adversarial))); + const markdown = serializeVerifyReportMarkdown(first); + + assert.equal(first.outcome, "PASS"); + assert.equal(markdown, serializeVerifyReportMarkdown(second)); + assert.match(markdown, /<reader>&`pipe\\\|slash\\\\/); + assert.doesNotMatch(markdown, //); }); -test("single reports are deterministic with canonical clause and violation ordering", () => { - const broken = verifyBundle(brokenOffExample); - const first = createVerifyReport(broken); - const second = createVerifyReport(verifyBundle(clone(brokenOffExample))); - - assert.deepEqual( - first.clauses.map((clause) => clause.id), - [ - "no_identifiable_activity", - "contextual_feed_functional", - "preference_survives_reload", - ], - ); - assert.deepEqual( - first.violations.map((violation) => violation.code), - ["PP_IDENTIFIABLE_EVENT_LEAK"], - ); - assert.equal(serializeReportJson(first), serializeReportJson(second)); - assert.equal( - serializeVerifyReportMarkdown(first), - serializeVerifyReportMarkdown(second), - ); +test("Markdown converts CR, LF, and CRLF to inert spaces after encoding", () => { + const report = clone( + createVerifyReport(verifyBundle(passingOffExample)), + ) as VerifyReport; + const mutableClause = report.clauses[0]!; + Object.assign(mutableClause, { + expected: "one\r\ntwo\nthree\rfour", + observed: "&`|\\", + }); + + const markdown = serializeVerifyReportMarkdown(report); + assert.match(markdown, /one two three four/); + assert.match(markdown, /<tag>&`\\\|\\\\/); + assert.doesNotMatch(markdown, /\r/); }); -test("gate reports preserve separate deterministic OFF and ON evaluations", () => { - const first = createGateReport( - runGate(passingOffExample, passingOnExample), - ); - const second = createGateReport( - runGate(clone(passingOffExample), clone(passingOnExample)), - ); +test("single and gate reports preserve deterministic canonical ordering", () => { + const single = createVerifyReport(verifyBundle(brokenOffExample)); + const gate = createGateReport(runGate(passingOffExample, passingOnExample)); assert.deepEqual( - first.evaluations.off.clauses.map((clause) => clause.id), + single.clauses.map((clause) => clause.id), [ "no_identifiable_activity", "contextual_feed_functional", @@ -202,70 +271,241 @@ test("gate reports preserve separate deterministic OFF and ON evaluations", () = ], ); assert.deepEqual( - first.evaluations.on.clauses.map((clause) => clause.id), + gate.evaluations.on.clauses.map((clause) => clause.id), ["expected_activity_received", "behavioral_feed_functional"], ); - assert.equal(serializeReportJson(first), serializeReportJson(second)); assert.equal( - serializeGateReportMarkdown(first), - serializeGateReportMarkdown(second), + serializeReportJson(single), + serializeReportJson( + createVerifyReport(verifyBundle(clone(brokenOffExample))), + ), + ); + assert.equal( + serializeGateReportMarkdown(gate), + serializeGateReportMarkdown( + createGateReport( + runGate(clone(passingOffExample), clone(passingOnExample)), + ), + ), ); }); -test("reports disclose external evidence authority and contain no machine metadata", () => { - const report = serializeReportJson( +test("adapter placeholders never enter public reports", () => { + const json = serializeReportJson( createVerifyReport(verifyBundle(passingOffExample)), ); const markdown = serializeVerifyReportMarkdown( createVerifyReport(verifyBundle(passingOffExample)), ); - assert.match(report, /"evidenceSource": "externally-supplied"/); - assert.match(report, /"collectionAttested": false/); - assert.match( - report, - /"evaluation": "deterministic-promiseproof-evaluator"/, - ); - assert.match(markdown, /Evidence source: externally supplied/); - assert.match( - markdown, - /Collection integrity: not attested by PromiseProof/, - ); - assert.match( - markdown, - /Evaluation authority: deterministic PromiseProof evaluator/, - ); - assert.doesNotMatch(report, /generatedAt|hostname|username|[A-Z]:\\/i); + for (const forbidden of [ + "promiseproof-external-adapter", + "2000-01-01T00:00:00.000Z", + "Not part of externally supplied evidence", + ]) { + assert.doesNotMatch(json, new RegExp(forbidden)); + assert.doesNotMatch(markdown, new RegExp(forbidden)); + } }); -test("invalid evidence cannot be rendered as a canonical report", () => { - const invalid = verifyBundle({ malformed: true }); +test("canonical projection and deterministic adapter preserve every evaluation byte-for-byte", () => { + for (const [name, canonical] of Object.entries(canonicalCases)) { + const original = evaluatePromise(canonical); + const roundTripped = evaluatePromise( + adaptExternalEvidence(projectCanonicalEvidence(canonical)), + ); - assert.equal(invalid.outcome, "INVALID_EVIDENCE"); - assert.throws( - () => createVerifyReport(invalid), - /Invalid evidence cannot be rendered as a product verdict/, - ); + assert.equal(stable(roundTripped), stable(original), name); + } }); -test("committed JSON Schema is generated byte-for-byte from the runtime Zod schema", async () => { - const committed = await readFile( - path.join( - projectRoot, - "artifacts", - "verify", - "activity-personalization.v1.schema.json", - ), - "utf8", - ); - const generated = `${JSON.stringify(externalBundleJsonSchema(), null, 2)}\n`; +test("every evaluator-relevant public field has mutation coverage", () => { + const mutationCases: Array<{ + name: string; + bundle: ExternalBundle; + mutate: (bundle: ExternalBundle) => void; + invalid?: boolean; + }> = [ + { + name: "scenario", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.scenario = "on"; + }, + }, + { + name: "subjectId", + bundle: passingOnExample, + mutate: (item) => { + item.evidence.subjectId = "different-subject"; + }, + }, + ...([ + ["uiPreference", "on"], + ["toggleChecked", true], + ["storedPreference", "on"], + ["backendPreference", "on"], + ["reloadObserved", false], + ] as const).map(([field, value]) => ({ + name: `control.${field}`, + bundle: passingOffExample, + mutate: (item: ExternalBundle) => { + Object.assign(item.evidence.control, { [field]: value }); + }, + })), + { + name: "capturedActivities", + bundle: passingOnExample, + mutate: (item) => { + item.evidence.activity.capturedActivities = []; + }, + }, + { + name: "recommendationServiceActivityReceipts", + bundle: passingOnExample, + mutate: (item) => { + item.evidence.activity.recommendationServiceReceipts = []; + }, + }, + ...(["runId", "subjectId", "itemId", "clientSequence", "occurredAt"] as const) + .flatMap((field) => [ + { + name: `capturedActivity.${field}`, + bundle: passingOnExample, + mutate: (item: ExternalBundle) => { + Object.assign(item.evidence.activity.capturedActivities[0]!, { + [field]: + field === "clientSequence" + ? 2 + : field === "occurredAt" + ? "2026-01-15T12:00:02.000Z" + : `different-${field}`, + }); + }, + }, + { + name: `activityReceipt.${field}`, + bundle: passingOnExample, + mutate: (item: ExternalBundle) => { + Object.assign( + item.evidence.activity.recommendationServiceReceipts[0]!, + { + [field]: + field === "clientSequence" + ? 2 + : field === "occurredAt" + ? "2026-01-15T12:00:02.000Z" + : `different-${field}`, + }, + ); + }, + }, + ]), + { + name: "capturedActivity.eventType", + bundle: passingOnExample, + invalid: true, + mutate: (item) => { + Object.assign(item.evidence.activity.capturedActivities[0]!, { + eventType: "click", + }); + }, + }, + { + name: "activityReceipt.eventType", + bundle: passingOnExample, + invalid: true, + mutate: (item) => { + Object.assign( + item.evidence.activity.recommendationServiceReceipts[0]!, + { eventType: "click" }, + ); + }, + }, + { + name: "feedFunctional", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.feedFunctional = false; + }, + }, + { + name: "renderedSource", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.renderedSource = "behavioral"; + }, + }, + { + name: "renderedItemIds", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.renderedItemIds = ["different-item"]; + }, + }, + { + name: "recommendationServiceReceipts", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts = []; + }, + }, + { + name: "recommendationReceipt.source", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts[0]!.source = + "behavioral"; + }, + }, + { + name: "recommendationReceipt.subjectId", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts[0]!.subjectId = + item.evidence.subjectId; + }, + }, + { + name: "recommendationReceipt.itemIds", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts[0]!.itemIds = + ["different-item"]; + }, + }, + ]; - assert.equal(committed, generated); + for (const mutation of mutationCases) { + const candidate = clone(mutation.bundle); + const baseline = verifyBundle(mutation.bundle); + mutation.mutate(candidate); + const changed = verifyBundle(candidate); + + if (mutation.invalid) { + assert.equal(changed.outcome, "INVALID_EVIDENCE", mutation.name); + } else { + assert.notEqual( + stable(changed.evaluation), + stable(baseline.evaluation), + mutation.name, + ); + assert.equal(changed.outcome, "BROKEN_PROMISE", mutation.name); + } + } }); -test("committed scaffold text artifacts match the runtime scaffold exactly", async () => { +test("committed schema and scaffold text are generated from runtime definitions", async () => { const artifactRoot = path.join(projectRoot, "artifacts", "verify"); + const committedSchema = await readFile( + path.join(artifactRoot, "activity-personalization.v1.schema.json"), + "utf8", + ); + assert.equal( + committedSchema, + `${JSON.stringify(externalBundleJsonSchema(), null, 2)}\n`, + ); assert.equal( await readFile(path.join(artifactRoot, "producer-template.mjs"), "utf8"), producerTemplate, @@ -274,9 +514,11 @@ test("committed scaffold text artifacts match the runtime scaffold exactly", asy await readFile(path.join(artifactRoot, "README.md"), "utf8"), scaffoldReadme, ); + assert.ok(committedSchema.split("\n").length < 400); + assert.ok(Buffer.byteLength(committedSchema) < 16_000); }); -test("Article Atlas examples are independent and limited to the supported family", () => { +test("Article Atlas examples remain independent and one-family only", () => { const serialized = JSON.stringify([ brokenOffExample, passingOffExample, @@ -288,14 +530,12 @@ test("Article Atlas examples are independent and limited to the supported family serialized, /signal shelf|initialization-race|propagation-failure|fixture|src\//i, ); - assert.equal(brokenOffExample.contractFamily, "activity-personalization/v1"); - assert.equal(passingOffExample.contractFamily, "activity-personalization/v1"); - assert.equal(passingOnExample.contractFamily, "activity-personalization/v1"); }); test("external verifier modules have no forbidden implementation imports", async () => { const verifyDirectory = path.join(projectRoot, "src", "verify"); const filenames = [ + "adapter.ts", "cli.ts", "examples.ts", "outcome.ts", @@ -310,17 +550,4 @@ test("external verifier modules have no forbidden implementation imports", async const source = await readFile(path.join(verifyDirectory, filename), "utf8"); assert.doesNotMatch(source, forbidden, filename); } - - const verifierSource = await readFile( - path.join(verifyDirectory, "verify.ts"), - "utf8", - ); - assert.match( - verifierSource, - /import \{ evaluatePromise \} from "\.\.\/shared\/evaluator\.js"/, - ); - assert.ok( - verifierSource.indexOf("if (!parsed.success)") < - verifierSource.indexOf("evaluatePromise(evidence)"), - ); }); From 1ef7df3ec5bb810b988d532b9e8fd03c46cb9178 Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Mon, 20 Jul 2026 23:02:47 +0100 Subject: [PATCH 3/7] fix: bind verifier reports to evidence --- package.json | 1 + src/verify/binding.ts | 105 +++++++++++ src/verify/cli.ts | 4 +- src/verify/outcome.ts | 2 +- src/verify/report.ts | 47 ++++- src/verify/verify.ts | 5 + tests/verify/cli.unit.ts | 72 ++++++++ tests/verify/core.unit.ts | 202 +++++++++++++++++++-- tests/verify/pipeline-equivalence.spec.ts | 37 ++++ tests/verify/pipeline.playwright.config.ts | 46 +++++ 10 files changed, 503 insertions(+), 18 deletions(-) create mode 100644 src/verify/binding.ts create mode 100644 tests/verify/pipeline-equivalence.spec.ts create mode 100644 tests/verify/pipeline.playwright.config.ts diff --git a/package.json b/package.json index 5e8f586..0d1bf4b 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "typecheck": "tsc --noEmit", "promiseproof": "tsx src/verify/cli.ts", "test:external": "tsx --test tests/verify/core.unit.ts tests/verify/cli.unit.ts", + "test:external:pipeline": "cross-env PROMISEPROOF_EQUIVALENCE_MODE=initialization-race playwright test --config=tests/verify/pipeline.playwright.config.ts && cross-env PROMISEPROOF_EQUIVALENCE_MODE=propagation-failure playwright test --config=tests/verify/pipeline.playwright.config.ts", "evidence:verify": "tsx scripts/evidence-verify.ts", "demo:rehearse": "tsx scripts/demo-rehearse.ts", "test:judge:unit": "tsx --test tests/judge/rehearsal.unit.ts", diff --git a/src/verify/binding.ts b/src/verify/binding.ts new file mode 100644 index 0000000..4d0e3b3 --- /dev/null +++ b/src/verify/binding.ts @@ -0,0 +1,105 @@ +import type { ExternalBundle } from "./schema.js"; + +export const CANONICAL_JSON_ID = "promiseproof-canonical-json/v1" as const; +export const SHA256_ALGORITHM = "SHA-256" as const; + +/** SHA-256 of src/shared/evaluator.ts after UTF-8 decoding and LF normalization. */ +export const EVALUATOR_SOURCE_SHA256 = + "fca20925861d1de2772ad0297a2465029678c4f34faa7a4755592f37aef9f87f" as const; + +export interface InputBinding { + readonly canonicalization: typeof CANONICAL_JSON_ID; + readonly algorithm: typeof SHA256_ALGORITHM; + readonly sha256: string; +} + +type CanonicalJson = + | null + | boolean + | number + | string + | readonly CanonicalJson[] + | { readonly [key: string]: CanonicalJson }; + +function compareCodeUnits(left: string, right: string): number { + if (left < right) { + return -1; + } + if (left > right) { + return 1; + } + return 0; +} + +function normalizeCanonicalJson(value: unknown): CanonicalJson { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new TypeError("Canonical JSON accepts only finite numbers."); + } + return value; + } + if (Array.isArray(value)) { + return value.map(normalizeCanonicalJson); + } + if (typeof value === "object") { + const normalized: Record = {}; + for (const key of Object.keys(value).sort(compareCodeUnits)) { + normalized[key] = normalizeCanonicalJson( + (value as Record)[key], + ); + } + return normalized; + } + throw new TypeError(`Canonical JSON does not accept ${typeof value}.`); +} + +export function canonicalizeJson(value: unknown): string { + return JSON.stringify(normalizeCanonicalJson(value)); +} + +function lowercaseHex(bytes: ArrayBuffer): string { + return Array.from(new Uint8Array(bytes), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); +} + +export async function sha256Utf8(value: string): Promise { + const bytes = new TextEncoder().encode(value); + return lowercaseHex(await globalThis.crypto.subtle.digest(SHA256_ALGORITHM, bytes)); +} + +export async function bindExternalBundle( + bundle: ExternalBundle, +): Promise { + return bindCanonicalJson(canonicalizeJson(bundle)); +} + +export async function bindCanonicalJson( + canonicalJson: string, +): Promise { + return { + canonicalization: CANONICAL_JSON_ID, + algorithm: SHA256_ALGORITHM, + sha256: await sha256Utf8(canonicalJson), + }; +} + +export function canonicalizeSourceLineEndings(source: string): string { + return source.replaceAll("\r\n", "\n").replaceAll("\r", "\n"); +} + +export async function evaluatorSourceMatchesFingerprint( + source: string, + expected: string = EVALUATOR_SOURCE_SHA256, +): Promise { + return ( + (await sha256Utf8(canonicalizeSourceLineEndings(source))) === expected + ); +} diff --git a/src/verify/cli.ts b/src/verify/cli.ts index 426316c..ffda5aa 100644 --- a/src/verify/cli.ts +++ b/src/verify/cli.ts @@ -311,7 +311,7 @@ async function runVerify(args: readonly string[], io: CliIo): Promise { return printInvalid(io, result.issues); } - const report = createVerifyReport(result); + const report = await createVerifyReport(result); await writeReports( safeOutputDirectory(options.out!), serializeReportJson(report), @@ -333,7 +333,7 @@ async function runGateCommand( return printInvalid(io, gateValidationIssues(result)); } - const report = createGateReport(result); + const report = await createGateReport(result); await writeReports( safeOutputDirectory(options.out!), serializeReportJson(report), diff --git a/src/verify/outcome.ts b/src/verify/outcome.ts index 695d39f..048ec95 100644 --- a/src/verify/outcome.ts +++ b/src/verify/outcome.ts @@ -14,4 +14,4 @@ export const EXIT_CODE = { export const SUPPORTED_SCHEMA_VERSION = "1" as const; export const SUPPORTED_CONTRACT_FAMILY = "activity-personalization/v1" as const; -export const REPORT_SCHEMA_VERSION = "1" as const; +export const REPORT_SCHEMA_VERSION = "2" as const; diff --git a/src/verify/report.ts b/src/verify/report.ts index e2798a8..fa1e5f3 100644 --- a/src/verify/report.ts +++ b/src/verify/report.ts @@ -3,6 +3,13 @@ import type { PromiseEvaluation, PromiseViolation, } from "../shared/types.js"; +import { + bindCanonicalJson, + CANONICAL_JSON_ID, + EVALUATOR_SOURCE_SHA256, + SHA256_ALGORITHM, + type InputBinding, +} from "./binding.js"; import { REPORT_SCHEMA_VERSION, SUPPORTED_CONTRACT_FAMILY, @@ -13,6 +20,7 @@ const authority = { evidenceSource: "externally-supplied", collectionAttested: false, evaluation: "deterministic-promiseproof-evaluator", + evaluatorSourceSha256: EVALUATOR_SOURCE_SHA256, } as const; export interface EvaluationReport { @@ -26,6 +34,7 @@ export interface VerifyReport extends EvaluationReport { readonly schemaVersion: typeof REPORT_SCHEMA_VERSION; readonly contractFamily: typeof SUPPORTED_CONTRACT_FAMILY; readonly outcome: "PASS" | "BROKEN_PROMISE"; + readonly inputBinding: InputBinding; readonly authority: typeof authority; } @@ -33,6 +42,10 @@ export interface GateReport { readonly schemaVersion: typeof REPORT_SCHEMA_VERSION; readonly contractFamily: typeof SUPPORTED_CONTRACT_FAMILY; readonly outcome: "PASS" | "BROKEN_PROMISE"; + readonly inputBindings: { + readonly off: InputBinding; + readonly on: InputBinding; + }; readonly evaluations: { readonly off: EvaluationReport; readonly on: EvaluationReport; @@ -47,6 +60,13 @@ function requireEvaluation(result: VerifyResult): PromiseEvaluation { return result.evaluation; } +function requireValidatedCanonicalJson(result: VerifyResult): string { + if (result.validatedCanonicalJson === null) { + throw new Error("Cannot bind a report without validated evidence."); + } + return result.validatedCanonicalJson; +} + function evaluationReport(result: VerifyResult): EvaluationReport { const evaluation = requireEvaluation(result); if (result.scenario === null) { @@ -61,7 +81,9 @@ function evaluationReport(result: VerifyResult): EvaluationReport { }; } -export function createVerifyReport(result: VerifyResult): VerifyReport { +export async function createVerifyReport( + result: VerifyResult, +): Promise { if ( result.outcome !== "PASS" && result.outcome !== "BROKEN_PROMISE" @@ -73,12 +95,15 @@ export function createVerifyReport(result: VerifyResult): VerifyReport { schemaVersion: REPORT_SCHEMA_VERSION, contractFamily: SUPPORTED_CONTRACT_FAMILY, outcome: result.outcome, + inputBinding: await bindCanonicalJson( + requireValidatedCanonicalJson(result), + ), ...evaluationReport(result), authority, }; } -export function createGateReport(result: GateResult): GateReport { +export async function createGateReport(result: GateResult): Promise { if ( result.outcome !== "PASS" && result.outcome !== "BROKEN_PROMISE" @@ -90,6 +115,10 @@ export function createGateReport(result: GateResult): GateReport { schemaVersion: REPORT_SCHEMA_VERSION, contractFamily: SUPPORTED_CONTRACT_FAMILY, outcome: result.outcome, + inputBindings: { + off: await bindCanonicalJson(requireValidatedCanonicalJson(result.off)), + on: await bindCanonicalJson(requireValidatedCanonicalJson(result.on)), + }, evaluations: { off: evaluationReport(result.off), on: evaluationReport(result.on), @@ -159,6 +188,15 @@ function authorityMarkdown(): string[] { "Evidence source: externally supplied", "Collection integrity: not attested by PromiseProof", "Evaluation authority: deterministic PromiseProof evaluator", + `Evaluator source SHA-256 (canonical UTF-8/LF): ${EVALUATOR_SOURCE_SHA256}`, + ]; +} + +function inputBindingMarkdown(label: string, binding: InputBinding): string[] { + return [ + `${label} canonicalization: ${CANONICAL_JSON_ID}`, + `${label} digest algorithm: ${SHA256_ALGORITHM}`, + `${label} evidence SHA-256: ${binding.sha256}`, ]; } @@ -169,6 +207,8 @@ export function serializeVerifyReportMarkdown(report: VerifyReport): string { `Contract family: ${report.contractFamily}`, `Outcome: ${report.outcome}`, "", + ...inputBindingMarkdown("Input", report.inputBinding), + "", ...evaluationMarkdown("Evaluation", report), "", ...authorityMarkdown(), @@ -182,6 +222,9 @@ export function serializeGateReportMarkdown(report: GateReport): string { `Contract family: ${report.contractFamily}`, `Outcome: ${report.outcome}`, "", + ...inputBindingMarkdown("OFF input", report.inputBindings.off), + ...inputBindingMarkdown("ON input", report.inputBindings.on), + "", ...evaluationMarkdown("OFF evaluation", report.evaluations.off), "", ...evaluationMarkdown("ON evaluation", report.evaluations.on), diff --git a/src/verify/verify.ts b/src/verify/verify.ts index b188bca..930bcc3 100644 --- a/src/verify/verify.ts +++ b/src/verify/verify.ts @@ -4,6 +4,7 @@ import type { PromiseEvidence, } from "../shared/types.js"; import { adaptExternalEvidence } from "./adapter.js"; +import { canonicalizeJson } from "./binding.js"; import { SUPPORTED_CONTRACT_FAMILY, type ExternalOutcome, @@ -20,6 +21,7 @@ export interface VerifyResult { readonly contractFamily: string; readonly scenario: "on" | "off" | null; readonly evaluation: PromiseEvaluation | null; + readonly validatedCanonicalJson: string | null; readonly issues: readonly string[]; } @@ -66,6 +68,7 @@ function rejectedResult(validation: BundleValidation): VerifyResult { contractFamily: SUPPORTED_CONTRACT_FAMILY, scenario: validation.success ? validation.bundle.evidence.scenario : null, evaluation: null, + validatedCanonicalJson: null, issues: [...(validation.success ? [] : validation.issues)].sort(), }; } @@ -74,6 +77,7 @@ function evaluateValidatedBundle( validated: ValidatedBundle, evaluator: CanonicalEvaluator, ): VerifyResult { + const validatedCanonicalJson = canonicalizeJson(validated.bundle); const evidence = adaptExternalEvidence(validated.bundle.evidence); const evaluation = evaluator(evidence); return { @@ -81,6 +85,7 @@ function evaluateValidatedBundle( contractFamily: validated.bundle.contractFamily, scenario: validated.bundle.evidence.scenario, evaluation, + validatedCanonicalJson, issues: [], }; } diff --git a/tests/verify/cli.unit.ts b/tests/verify/cli.unit.ts index 7d3fded..784516f 100644 --- a/tests/verify/cli.unit.ts +++ b/tests/verify/cli.unit.ts @@ -15,6 +15,7 @@ import { passingOffExample, passingOnExample, } from "../../src/verify/examples.js"; +import { bindExternalBundle } from "../../src/verify/binding.js"; import { MAX_INPUT_BYTES } from "../../src/verify/schema.js"; const projectRoot = process.cwd(); @@ -357,6 +358,77 @@ test("CLI creates byte-identical JSON and Markdown across repeated runs", async ); }); +test("CLI uses the shared canonical binding and changes reports for semantic input changes", async () => { + const root = await temporaryRoot(); + const reorderedPath = path.join(root, "reordered.json"); + const changedPath = path.join(root, "changed.json"); + const reorderedOutput = path.join(root, "reordered-report"); + const changedOutput = path.join(root, "changed-report"); + const reordered = { + evidence: { + recommendations: passingOffExample.evidence.recommendations, + activity: passingOffExample.evidence.activity, + control: passingOffExample.evidence.control, + subjectId: passingOffExample.evidence.subjectId, + scenario: passingOffExample.evidence.scenario, + }, + contractFamily: passingOffExample.contractFamily, + schemaVersion: passingOffExample.schemaVersion, + }; + const changed = clone(passingOffExample); + changed.evidence.subjectId = "article-atlas-reader-002"; + await writeFile(reorderedPath, JSON.stringify(reordered), "utf8"); + await writeJson(changedPath, changed); + + assert.equal( + runCli([ + "verify", + "--evidence", + reorderedPath, + "--out", + reorderedOutput, + ]).status, + 0, + ); + assert.equal( + runCli([ + "verify", + "--evidence", + changedPath, + "--out", + changedOutput, + ]).status, + 0, + ); + + const reorderedReport = JSON.parse( + await readFile(path.join(reorderedOutput, "report.json"), "utf8"), + ) as { inputBinding: { sha256: string } }; + const changedReport = JSON.parse( + await readFile(path.join(changedOutput, "report.json"), "utf8"), + ) as { inputBinding: { sha256: string } }; + assert.equal( + reorderedReport.inputBinding.sha256, + (await bindExternalBundle(passingOffExample)).sha256, + ); + assert.match( + await readFile(path.join(reorderedOutput, "report.md"), "utf8"), + new RegExp(reorderedReport.inputBinding.sha256, "u"), + ); + assert.notEqual( + reorderedReport.inputBinding.sha256, + changedReport.inputBinding.sha256, + ); + assert.notDeepEqual( + await readFile(path.join(reorderedOutput, "report.json")), + await readFile(path.join(changedOutput, "report.json")), + ); + assert.notDeepEqual( + await readFile(path.join(reorderedOutput, "report.md")), + await readFile(path.join(changedOutput, "report.md")), + ); +}); + test("CLI help succeeds and documents commands and exit codes", () => { const result = runCli(["--help"]); diff --git a/tests/verify/core.unit.ts b/tests/verify/core.unit.ts index b71dcc6..aeb1ef0 100644 --- a/tests/verify/core.unit.ts +++ b/tests/verify/core.unit.ts @@ -1,10 +1,17 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import { readFile } from "node:fs/promises"; import path from "node:path"; import test from "node:test"; import { evaluatePromise } from "../../src/shared/evaluator.js"; import { adaptExternalEvidence } from "../../src/verify/adapter.js"; +import { + bindExternalBundle, + canonicalizeJson, + EVALUATOR_SOURCE_SHA256, + evaluatorSourceMatchesFingerprint, +} from "../../src/verify/binding.js"; import { brokenOffExample, passingOffExample, @@ -222,7 +229,7 @@ test("bounded collections are rejected before canonical evaluation", () => { assert.match(result.issues.join("\n"), /Too big/); }); -test("Markdown encodes HTML, ampersands, backticks, pipes, and backslashes deterministically", () => { +test("Markdown encodes HTML, ampersands, backticks, pipes, and backslashes deterministically", async () => { const adversarial = clone(passingOnExample); const subject = "&`pipe|slash\\"; adversarial.evidence.subjectId = subject; @@ -232,8 +239,8 @@ test("Markdown encodes HTML, ampersands, backticks, pipes, and backslashes deter adversarial.evidence.recommendations.recommendationServiceReceipts[0]!.subjectId = subject; - const first = createVerifyReport(verifyBundle(adversarial)); - const second = createVerifyReport(verifyBundle(clone(adversarial))); + const first = await createVerifyReport(verifyBundle(adversarial)); + const second = await createVerifyReport(verifyBundle(clone(adversarial))); const markdown = serializeVerifyReportMarkdown(first); assert.equal(first.outcome, "PASS"); @@ -242,9 +249,9 @@ test("Markdown encodes HTML, ampersands, backticks, pipes, and backslashes deter assert.doesNotMatch(markdown, //); }); -test("Markdown converts CR, LF, and CRLF to inert spaces after encoding", () => { +test("Markdown converts CR, LF, and CRLF to inert spaces after encoding", async () => { const report = clone( - createVerifyReport(verifyBundle(passingOffExample)), + await createVerifyReport(verifyBundle(passingOffExample)), ) as VerifyReport; const mutableClause = report.clauses[0]!; Object.assign(mutableClause, { @@ -258,9 +265,11 @@ test("Markdown converts CR, LF, and CRLF to inert spaces after encoding", () => assert.doesNotMatch(markdown, /\r/); }); -test("single and gate reports preserve deterministic canonical ordering", () => { - const single = createVerifyReport(verifyBundle(brokenOffExample)); - const gate = createGateReport(runGate(passingOffExample, passingOnExample)); +test("single and gate reports preserve deterministic canonical ordering", async () => { + const single = await createVerifyReport(verifyBundle(brokenOffExample)); + const gate = await createGateReport( + runGate(passingOffExample, passingOnExample), + ); assert.deepEqual( single.clauses.map((clause) => clause.id), @@ -277,25 +286,25 @@ test("single and gate reports preserve deterministic canonical ordering", () => assert.equal( serializeReportJson(single), serializeReportJson( - createVerifyReport(verifyBundle(clone(brokenOffExample))), + await createVerifyReport(verifyBundle(clone(brokenOffExample))), ), ); assert.equal( serializeGateReportMarkdown(gate), serializeGateReportMarkdown( - createGateReport( + await createGateReport( runGate(clone(passingOffExample), clone(passingOnExample)), ), ), ); }); -test("adapter placeholders never enter public reports", () => { +test("adapter placeholders never enter public reports", async () => { const json = serializeReportJson( - createVerifyReport(verifyBundle(passingOffExample)), + await createVerifyReport(verifyBundle(passingOffExample)), ); const markdown = serializeVerifyReportMarkdown( - createVerifyReport(verifyBundle(passingOffExample)), + await createVerifyReport(verifyBundle(passingOffExample)), ); for (const forbidden of [ @@ -308,6 +317,172 @@ test("adapter placeholders never enter public reports", () => { } }); +test("canonical bundle binding is stable across key order and JSON whitespace", async () => { + const reordered: ExternalBundle = { + evidence: { + recommendations: clone(passingOffExample.evidence.recommendations), + activity: clone(passingOffExample.evidence.activity), + control: clone(passingOffExample.evidence.control), + subjectId: passingOffExample.evidence.subjectId, + scenario: passingOffExample.evidence.scenario, + }, + contractFamily: passingOffExample.contractFamily, + schemaVersion: passingOffExample.schemaVersion, + }; + const compact = JSON.parse(JSON.stringify(passingOffExample)) as ExternalBundle; + const formatted = JSON.parse( + JSON.stringify(passingOffExample, null, 4), + ) as ExternalBundle; + const expected = await bindExternalBundle(passingOffExample); + + assert.deepEqual(await bindExternalBundle(reordered), expected); + assert.deepEqual(await bindExternalBundle(compact), expected); + assert.deepEqual(await bindExternalBundle(formatted), expected); + assert.equal(canonicalizeJson(reordered), canonicalizeJson(passingOffExample)); + assert.equal( + canonicalizeJson({ "ä": 1, a: 2, Z: 3, A: 4 }), + '{"A":4,"Z":3,"a":2,"ä":1}', + ); +}); + +test("array order and every evidence category are bound by the digest", async () => { + const baseline = (await bindExternalBundle(passingOnExample)).sha256; + const mutations: Array<[string, (bundle: ExternalBundle) => void]> = [ + [ + "subjectId", + (item) => { + item.evidence.subjectId = "different-subject"; + }, + ], + [ + "control.uiPreference", + (item) => { + item.evidence.control.uiPreference = "off"; + }, + ], + [ + "control.toggleChecked", + (item) => { + item.evidence.control.toggleChecked = false; + }, + ], + [ + "control.storedPreference", + (item) => { + item.evidence.control.storedPreference = "off"; + }, + ], + [ + "control.backendPreference", + (item) => { + item.evidence.control.backendPreference = "off"; + }, + ], + [ + "control.reloadObserved", + (item) => { + item.evidence.control.reloadObserved = false; + }, + ], + [ + "activity", + (item) => { + item.evidence.activity.capturedActivities[0]!.itemId = + "different-origin"; + }, + ], + [ + "recommendation", + (item) => { + item.evidence.recommendations.renderedItemIds = ["different-story"]; + }, + ], + ]; + + for (const [name, mutate] of mutations) { + const candidate = clone(passingOnExample); + mutate(candidate); + assert.notEqual((await bindExternalBundle(candidate)).sha256, baseline, name); + } + + const ordered = clone(passingOnExample); + ordered.evidence.recommendations.renderedItemIds = ["story-a", "story-b"]; + const reversed = clone(ordered); + reversed.evidence.recommendations.renderedItemIds.reverse(); + assert.notEqual( + (await bindExternalBundle(ordered)).sha256, + (await bindExternalBundle(reversed)).sha256, + ); +}); + +test("report schema v2 binds single and gate inputs and discloses evaluator fingerprint", async () => { + const single = await createVerifyReport(verifyBundle(passingOffExample)); + const gate = await createGateReport( + runGate(passingOffExample, passingOnExample), + ); + const expectedOff = await bindExternalBundle(passingOffExample); + const expectedOn = await bindExternalBundle(passingOnExample); + + assert.equal(single.schemaVersion, "2"); + assert.deepEqual(single.inputBinding, expectedOff); + assert.deepEqual(gate.inputBindings.off, expectedOff); + assert.deepEqual(gate.inputBindings.on, expectedOn); + assert.notEqual(gate.inputBindings.off.sha256, gate.inputBindings.on.sha256); + assert.equal(single.authority.evaluatorSourceSha256, EVALUATOR_SOURCE_SHA256); + assert.equal(gate.authority.evaluatorSourceSha256, EVALUATOR_SOURCE_SHA256); + + const singleJson = serializeReportJson(single); + const singleMarkdown = serializeVerifyReportMarkdown(single); + const gateJson = serializeReportJson(gate); + const gateMarkdown = serializeGateReportMarkdown(gate); + for (const digest of [expectedOff.sha256, expectedOn.sha256]) { + assert.match(digest, /^[0-9a-f]{64}$/u); + assert.ok( + digest === expectedOff.sha256 + ? singleJson.includes(digest) && singleMarkdown.includes(digest) + : true, + ); + assert.ok(gateJson.includes(digest)); + assert.ok(gateMarkdown.includes(digest)); + } + for (const output of [singleJson, singleMarkdown, gateJson, gateMarkdown]) { + assert.ok(output.includes(EVALUATOR_SOURCE_SHA256)); + } +}); + +test("unsupported envelope literals cannot reach report binding", async () => { + for (const candidate of [ + { ...clone(passingOffExample), schemaVersion: "2" }, + { ...clone(passingOffExample), contractFamily: "unsupported/v1" }, + ]) { + const result = verifyBundle(candidate); + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.validatedCanonicalJson, null); + await assert.rejects(createVerifyReport(result), /Invalid evidence/); + } +}); + +test("committed evaluator fingerprint matches canonical-LF source and known Git blob", async () => { + const evaluatorPath = path.join(projectRoot, "src", "shared", "evaluator.ts"); + const source = await readFile(evaluatorPath, "utf8"); + assert.equal(await evaluatorSourceMatchesFingerprint(source), true); + assert.equal( + await evaluatorSourceMatchesFingerprint(source, "0".repeat(64)), + false, + ); + + const git = spawnSync("git", ["hash-object", "src/shared/evaluator.ts"], { + cwd: projectRoot, + encoding: "utf8", + windowsHide: true, + }); + assert.equal(git.status, 0, git.stderr); + assert.equal( + git.stdout.trim(), + "9654cb43f4388c7f399bf0717ceb94bba2718962", + ); +}); + test("canonical projection and deterministic adapter preserve every evaluation byte-for-byte", () => { for (const [name, canonical] of Object.entries(canonicalCases)) { const original = evaluatePromise(canonical); @@ -536,6 +711,7 @@ test("external verifier modules have no forbidden implementation imports", async const verifyDirectory = path.join(projectRoot, "src", "verify"); const filenames = [ "adapter.ts", + "binding.ts", "cli.ts", "examples.ts", "outcome.ts", diff --git a/tests/verify/pipeline-equivalence.spec.ts b/tests/verify/pipeline-equivalence.spec.ts new file mode 100644 index 0000000..1bdc264 --- /dev/null +++ b/tests/verify/pipeline-equivalence.spec.ts @@ -0,0 +1,37 @@ +import { expect, test } from "@playwright/test"; + +import { evaluatePromise } from "../../src/shared/evaluator.js"; +import { adaptExternalEvidence } from "../../src/verify/adapter.js"; +import { runPromiseScenario } from "../support/scenario.js"; +import { projectCanonicalEvidence } from "./canonical-projection.js"; + +function stable(value: unknown): string { + return JSON.stringify(value); +} + +for (const scenario of ["off", "on"] as const) { + test(`real ${scenario.toUpperCase()} evidence survives projection and adaptation exactly`, async ({ + page, + request, + }, testInfo) => { + const result = await runPromiseScenario(page, request, testInfo, scenario, { + runId: `external-equivalence-${scenario}`, + userId: `external-equivalence-subject-${scenario}`, + }); + const roundTripped = evaluatePromise( + adaptExternalEvidence(projectCanonicalEvidence(result.evidence)), + ); + + expect(result.browserErrors).toEqual([]); + expect( + result.evaluation.violations.map((violation) => violation.code), + ).toEqual( + scenario === "on" + ? [] + : testInfo.project.name === "initialization-race" + ? ["PP_IDENTIFIABLE_EVENT_LEAK"] + : ["PP_PREFERENCE_NOT_PERSISTED"], + ); + expect(stable(roundTripped)).toBe(stable(result.evaluation)); + }); +} diff --git a/tests/verify/pipeline.playwright.config.ts b/tests/verify/pipeline.playwright.config.ts new file mode 100644 index 0000000..246274d --- /dev/null +++ b/tests/verify/pipeline.playwright.config.ts @@ -0,0 +1,46 @@ +import { fileURLToPath } from "node:url"; + +import { defineConfig } from "@playwright/test"; + +const mode = process.env.PROMISEPROOF_EQUIVALENCE_MODE; +if (mode !== "initialization-race" && mode !== "propagation-failure") { + throw new Error( + "PROMISEPROOF_EQUIVALENCE_MODE must select a registered demo mode.", + ); +} + +const baseURL = process.env.PROMISEPROOF_BASE_URL ?? "http://127.0.0.1:4173"; +const projectRoot = fileURLToPath(new URL("../..", import.meta.url)); + +export default defineConfig({ + testDir: ".", + testMatch: "pipeline-equivalence.spec.ts", + fullyParallel: false, + workers: 1, + retries: 0, + forbidOnly: Boolean(process.env.CI), + reporter: [["list"]], + projects: [{ name: mode }], + outputDir: fileURLToPath( + new URL(`../../test-results/external-equivalence-${mode}`, import.meta.url), + ), + expect: { timeout: 5_000 }, + use: { + baseURL, + trace: "retain-on-failure", + screenshot: "only-on-failure", + video: "retain-on-failure", + }, + webServer: { + command: + mode === "initialization-race" + ? "npm run dev:test:race" + : "npm run dev:test:propagation", + cwd: projectRoot, + url: `${baseURL}/api/health`, + reuseExistingServer: false, + timeout: 120_000, + stdout: "pipe", + stderr: "pipe", + }, +}); From d8c590b497d5bf5f8a13e82b6461f762c3c3e91d Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Tue, 21 Jul 2026 00:15:15 +0100 Subject: [PATCH 4/7] feat: add reproducible report check The check command re-runs the evaluator on the supplied evidence, rebuilds the full report, and compares the whole canonicalized document rather than trusting the embedded hashes. A report with the right digest and fingerprint but a faked verdict no longer passes. Works for single bundles and OFF/ON gates, with exit codes 0 reproduced, 4 mismatch, 3 invalid, 1 error. Adds 16 adversarial tests. --- package.json | 2 +- src/verify/check.ts | 88 ++++++++++++++++++++ src/verify/cli.ts | 51 +++++++++++- tests/verify/check.unit.ts | 164 +++++++++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 src/verify/check.ts create mode 100644 tests/verify/check.unit.ts diff --git a/package.json b/package.json index 0d1bf4b..b461411 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "start:propagation": "cross-env NODE_ENV=production DEMO_MODE=propagation-failure node dist/server.mjs", "typecheck": "tsc --noEmit", "promiseproof": "tsx src/verify/cli.ts", - "test:external": "tsx --test tests/verify/core.unit.ts tests/verify/cli.unit.ts", + "test:external": "tsx --test tests/verify/core.unit.ts tests/verify/cli.unit.ts tests/verify/check.unit.ts", "test:external:pipeline": "cross-env PROMISEPROOF_EQUIVALENCE_MODE=initialization-race playwright test --config=tests/verify/pipeline.playwright.config.ts && cross-env PROMISEPROOF_EQUIVALENCE_MODE=propagation-failure playwright test --config=tests/verify/pipeline.playwright.config.ts", "evidence:verify": "tsx scripts/evidence-verify.ts", "demo:rehearse": "tsx scripts/demo-rehearse.ts", diff --git a/src/verify/check.ts b/src/verify/check.ts new file mode 100644 index 0000000..6d53a64 --- /dev/null +++ b/src/verify/check.ts @@ -0,0 +1,88 @@ +// Reproducible report check. A bound report is only trustworthy if it can be +// REPRODUCED: re-run the unchanged evaluator on the supplied evidence, regenerate +// the complete report, and compare the whole canonical document. Comparing only +// the embedded hashes is insufficient, a fabricated report could carry the correct +// evidence digest and evaluator fingerprint while inventing the verdict, clauses, +// or violations. Full reproduction catches that. +import { z } from "zod"; +import { canonicalizeJson } from "./binding.js"; +import { REPORT_SCHEMA_VERSION } from "./outcome.js"; +import { createGateReport, createVerifyReport } from "./report.js"; +import { runGate, verifyBundle } from "./verify.js"; + +export const CHECK_EXIT = { + BOUND_AND_REPRODUCED: 0, + USAGE_OR_EXECUTION_ERROR: 1, + INVALID_REPORT_OR_EVIDENCE: 3, + STALE_OR_MISMATCH: 4, +} as const; + +export type CheckStatus = keyof typeof CHECK_EXIT; + +export interface CheckResult { + readonly status: CheckStatus; + readonly detail: string; +} + +// Minimal envelope: the supplied document must at least declare itself a report +// of the current schema version to be checkable. Anything else is INVALID (not a +// mere mismatch); a well-formed report that fails to reproduce is a mismatch. +const reportEnvelope = z + .object({ schemaVersion: z.literal(REPORT_SCHEMA_VERSION) }) + .passthrough(); + +function invalid(detail: string): CheckResult { + return { status: "INVALID_REPORT_OR_EVIDENCE", detail }; +} + +function compare(regenerated: unknown, supplied: unknown): CheckResult { + if (canonicalizeJson(regenerated) === canonicalizeJson(supplied)) { + return { + status: "BOUND_AND_REPRODUCED", + detail: + "the report is reproduced exactly by re-running the unchanged evaluator on the supplied evidence", + }; + } + return { + status: "STALE_OR_MISMATCH", + detail: + "the supplied report does not match the report reproduced from the supplied evidence", + }; +} + +export async function checkSingle( + rawReport: unknown, + rawEvidence: unknown, +): Promise { + if (!reportEnvelope.safeParse(rawReport).success) { + return invalid( + `report is not a schemaVersion ${REPORT_SCHEMA_VERSION} verification report`, + ); + } + const result = verifyBundle(rawEvidence); + if (result.outcome === "INVALID_EVIDENCE") { + return invalid( + "evidence is invalid; no canonical report can be reproduced from it", + ); + } + return compare(await createVerifyReport(result), rawReport); +} + +export async function checkGate( + rawReport: unknown, + rawOff: unknown, + rawOn: unknown, +): Promise { + if (!reportEnvelope.safeParse(rawReport).success) { + return invalid( + `report is not a schemaVersion ${REPORT_SCHEMA_VERSION} gate report`, + ); + } + const result = runGate(rawOff, rawOn); + if (result.outcome === "INVALID_EVIDENCE") { + return invalid( + "gate evidence is invalid; no canonical report can be reproduced from it", + ); + } + return compare(await createGateReport(result), rawReport); +} diff --git a/src/verify/cli.ts b/src/verify/cli.ts index ffda5aa..ceca74f 100644 --- a/src/verify/cli.ts +++ b/src/verify/cli.ts @@ -9,6 +9,11 @@ import path from "node:path"; import process from "node:process"; import { fileURLToPath } from "node:url"; +import { + CHECK_EXIT, + checkGate, + checkSingle, +} from "./check.js"; import { brokenOffExample, passingOffExample, @@ -43,14 +48,23 @@ Usage: npm run promiseproof -- init --out npm run promiseproof -- verify --evidence --out npm run promiseproof -- gate --off --on --out + npm run promiseproof -- check --report --evidence + npm run promiseproof -- check --report --off --on npm run promiseproof -- --help -Exit codes: +Exit codes (verify / gate): 0 PASS 1 usage, I/O, or unexpected execution error 2 BROKEN_PROMISE 3 INVALID_EVIDENCE +Exit codes (check): the report is reproduced by re-running the unchanged +evaluator on the supplied evidence, not merely hash-compared. + 0 BOUND_AND_REPRODUCED + 1 usage, I/O, or unexpected execution error + 3 INVALID_REPORT_OR_EVIDENCE + 4 STALE_OR_MISMATCH + Supported contract family: activity-personalization/v1 `; @@ -343,6 +357,38 @@ async function runGateCommand( return EXIT_CODE[result.outcome]; } +async function runCheck(args: readonly string[], io: CliIo): Promise { + const hasEvidence = args.includes("--evidence"); + const hasGate = args.includes("--off") || args.includes("--on"); + + if (hasEvidence && !hasGate) { + const options = parseOptions(args, ["report", "evidence"]); + const [report, evidence] = await Promise.all([ + readExternalJson(options.report!), + readExternalJson(options.evidence!), + ]); + const result = await checkSingle(report, evidence); + io.stdout(`${result.status}: ${result.detail}`); + return CHECK_EXIT[result.status]; + } + + if (hasGate && !hasEvidence) { + const options = parseOptions(args, ["report", "off", "on"]); + const [report, off, on] = await Promise.all([ + readExternalJson(options.report!), + readExternalJson(options.off!), + readExternalJson(options.on!), + ]); + const result = await checkGate(report, off, on); + io.stdout(`${result.status}: ${result.detail}`); + return CHECK_EXIT[result.status]; + } + + throw new UsageError( + "check requires --report with either --evidence, or both --off and --on.", + ); +} + export async function runCli( args: readonly string[], io: CliIo = { @@ -370,6 +416,9 @@ export async function runCli( if (command === "gate") { return await runGateCommand(commandArgs, io); } + if (command === "check") { + return await runCheck(commandArgs, io); + } throw new UsageError(`Unknown command: ${String(command)}`); } catch (error) { if (error instanceof InvalidEvidenceFileError) { diff --git a/tests/verify/check.unit.ts b/tests/verify/check.unit.ts new file mode 100644 index 0000000..32f6718 --- /dev/null +++ b/tests/verify/check.unit.ts @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { checkGate, checkSingle } from "../../src/verify/check.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, +} from "../../src/verify/examples.js"; +import { createGateReport, createVerifyReport } from "../../src/verify/report.js"; +import { runGate, verifyBundle } from "../../src/verify/verify.js"; + +// A genuine report is one produced by the real pipeline for the given evidence. +async function genuineSingle(bundle: unknown): Promise { + return createVerifyReport(verifyBundle(bundle)); +} +async function genuineGate(off: unknown, on: unknown): Promise { + return createGateReport(runGate(off, on)); +} +// Deep clone so a mutation never touches the shared example or the source report. +function clone(value: unknown): any { + return JSON.parse(JSON.stringify(value)); +} + +test("a genuine report reproduces from its own evidence", async () => { + const report = await genuineSingle(passingOffExample); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "BOUND_AND_REPRODUCED", + ); +}); + +test("a reformatted/round-tripped report still reproduces", async () => { + const report = clone(await genuineSingle(passingOffExample)); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "BOUND_AND_REPRODUCED", + ); +}); + +test("a report checked against different evidence does not reproduce", async () => { + const report = await genuineSingle(passingOffExample); + assert.equal( + (await checkSingle(report, passingOnExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a fabricated PASS over broken evidence is caught (hashes alone would miss it)", async () => { + // Same correct evidence digest and evaluator fingerprint, but an invented verdict. + const report = clone(await genuineSingle(brokenOffExample)); + report.outcome = "PASS"; + report.canonicalVerdict = "pass"; + report.violations = []; + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a tampered evidence digest is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.inputBinding.sha256 = "0".repeat(64); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a tampered evaluator fingerprint is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.authority.evaluatorSourceSha256 = "0".repeat(64); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("an altered non-attestation disclosure is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.authority.collectionAttested = true; + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a removed violation is caught", async () => { + const report = clone(await genuineSingle(brokenOffExample)); + report.violations = []; + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("an added violation is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.violations = [ + { + code: "PP_IDENTIFIABLE_EVENT_LEAK", + clause: "no_identifiable_activity", + message: "fabricated", + }, + ]; + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a changed clause observed value is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.clauses[0].observed = "tampered observation"; + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a reordered clauses array is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.clauses.reverse(); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a document that is not a v2 report is INVALID_REPORT_OR_EVIDENCE", async () => { + assert.equal( + (await checkSingle({ hello: "world" }, passingOffExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); +}); + +test("invalid evidence is INVALID_REPORT_OR_EVIDENCE", async () => { + const report = await genuineSingle(passingOffExample); + assert.equal( + (await checkSingle(report, { not: "a bundle" })).status, + "INVALID_REPORT_OR_EVIDENCE", + ); +}); + +test("a genuine gate report reproduces", async () => { + const report = await genuineGate(passingOffExample, passingOnExample); + assert.equal( + (await checkGate(report, passingOffExample, passingOnExample)).status, + "BOUND_AND_REPRODUCED", + ); +}); + +test("a gate report checked against a changed side does not reproduce", async () => { + const report = await genuineGate(passingOffExample, passingOnExample); + assert.equal( + (await checkGate(report, brokenOffExample, passingOnExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("gate OFF and ON evidence digests are distinct", async () => { + const report = clone(await genuineGate(passingOffExample, passingOnExample)); + assert.notEqual(report.inputBindings.off.sha256, report.inputBindings.on.sha256); +}); From c104edbaf7f455ad45ddecad33739dcc860da6e8 Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Tue, 21 Jul 2026 00:15:15 +0100 Subject: [PATCH 5/7] feat: add hosted verifier at /verify/ A no-network page that runs the same verifier modules as the CLI over the passing OFF and ON example, opening on PASS with a bound report. Tampering the OFF evidence adds one identifiable activity and receipt, so the evaluator returns a broken promise and the sealed report stops reproducing. Re-sealing binds the broken verdict, never a pass. It also verifies bring-your-own bundles and downloads the report. build:site now assembles /verify/ next to / and /walkthrough/. A Playwright suite covers the load state, the absence of network traffic, the tamper flip, and byte-identical browser and CLI reports. --- package.json | 1 + scripts/build-site.mjs | 51 +- scripts/serve-site.mjs | 47 ++ src/verify-ui/main.ts | 707 +++++++++++++++++++ src/verify-ui/styles.css | 696 ++++++++++++++++++ tests/verify/judge-mode.playwright.config.ts | 34 + tests/verify/judge-mode.spec.ts | 144 ++++ verify.html | 103 +++ vite.config.ts | 5 +- 9 files changed, 1772 insertions(+), 16 deletions(-) create mode 100644 scripts/serve-site.mjs create mode 100644 src/verify-ui/main.ts create mode 100644 src/verify-ui/styles.css create mode 100644 tests/verify/judge-mode.playwright.config.ts create mode 100644 tests/verify/judge-mode.spec.ts create mode 100644 verify.html diff --git a/package.json b/package.json index b461411..054604b 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "promiseproof": "tsx src/verify/cli.ts", "test:external": "tsx --test tests/verify/core.unit.ts tests/verify/cli.unit.ts tests/verify/check.unit.ts", "test:external:pipeline": "cross-env PROMISEPROOF_EQUIVALENCE_MODE=initialization-race playwright test --config=tests/verify/pipeline.playwright.config.ts && cross-env PROMISEPROOF_EQUIVALENCE_MODE=propagation-failure playwright test --config=tests/verify/pipeline.playwright.config.ts", + "test:verify:judge": "playwright test --config=tests/verify/judge-mode.playwright.config.ts", "evidence:verify": "tsx scripts/evidence-verify.ts", "demo:rehearse": "tsx scripts/demo-rehearse.ts", "test:judge:unit": "tsx --test tests/judge/rehearsal.unit.ts", diff --git a/scripts/build-site.mjs b/scripts/build-site.mjs index fdbd23d..168f753 100644 --- a/scripts/build-site.mjs +++ b/scripts/build-site.mjs @@ -1,6 +1,7 @@ // Assembles the combined public site into dist/site: // / -> landing/index.html (the front door) // /walkthrough/ -> the judge walkthrough, built with a /walkthrough/ base +// /verify/ -> the hosted deterministic verifier, built with a /verify/ base // // Deploy the result with `npx wrangler deploy` (see wrangler.toml). // @@ -17,39 +18,59 @@ import { const SITE = "dist/site"; const WT = "dist/_walkthrough"; +const VF = "dist/_verify"; -// 1) Build the walkthrough with the /walkthrough/ base so its asset URLs resolve -// under the sub-path. The env var is read by vite.config.ts. +// 1) Build each sub-path page with the base its sub-path needs, so asset URLs +// resolve under the sub-path. The walkthrough reads PP_WALKTHROUGH_BASE from +// vite.config.ts; the verifier passes --base, which overrides the default "/". console.log("• building walkthrough (base=/walkthrough/) ..."); execSync(`npx vite build --outDir ${WT} --emptyOutDir`, { stdio: "inherit", env: { ...process.env, PP_WALKTHROUGH_BASE: "/walkthrough/" }, }); +console.log("• building verifier (base=/verify/) ..."); +execSync(`npx vite build --base=/verify/ --outDir ${VF} --emptyOutDir`, { + stdio: "inherit", +}); + // 2) Fresh output tree. rmSync(SITE, { recursive: true, force: true }); mkdirSync(`${SITE}/walkthrough/assets`, { recursive: true }); +mkdirSync(`${SITE}/verify/assets`, { recursive: true }); -// 3) Landing at the root (already carries relative /walkthrough/ CTAs). +// 3) Landing at the root (already carries relative /walkthrough/ and /verify/ CTAs). copyFileSync("landing/index.html", `${SITE}/index.html`); -// 4) Shared static assets. Root serves the landing's; the walkthrough HTML -// references /walkthrough/favicon.* (Vite rewrote those under the base). +// 4) Shared static assets. Root serves the landing's; the sub-pages reference +// /walkthrough/favicon.* and /verify/favicon.* (Vite rewrote those under each base). const shared = ["favicon.svg", "favicon.ico", "apple-touch-icon.png", "og-card.png"]; for (const f of shared) copyFileSync(`public/${f}`, `${SITE}/${f}`); -for (const f of ["favicon.svg", "favicon.ico", "apple-touch-icon.png"]) { - copyFileSync(`public/${f}`, `${SITE}/walkthrough/${f}`); +for (const sub of ["walkthrough", "verify"]) { + for (const f of ["favicon.svg", "favicon.ico", "apple-touch-icon.png"]) { + copyFileSync(`public/${f}`, `${SITE}/${sub}/${f}`); + } } -// 5) Walkthrough page + only the assets it actually references. -copyFileSync(`${WT}/judge.html`, `${SITE}/walkthrough/index.html`); -const wtHtml = readFileSync(`${SITE}/walkthrough/index.html`, "utf8"); -const assets = [...wtHtml.matchAll(/\/walkthrough\/assets\/([^"?#]+)/g)].map((m) => m[1]); -for (const a of new Set(assets)) { - copyFileSync(`${WT}/assets/${a}`, `${SITE}/walkthrough/assets/${a}`); +// 5) Each sub-page + only the assets it actually references. +function assemble(intermediateDir, htmlName, sub) { + copyFileSync(`${intermediateDir}/${htmlName}`, `${SITE}/${sub}/index.html`); + const html = readFileSync(`${SITE}/${sub}/index.html`, "utf8"); + const pattern = new RegExp(`/${sub}/assets/([^"?#]+)`, "g"); + const assets = [...html.matchAll(pattern)].map((m) => m[1]); + for (const a of new Set(assets)) { + copyFileSync(`${intermediateDir}/assets/${a}`, `${SITE}/${sub}/assets/${a}`); + } + return new Set(assets).size; } -// 6) Drop the intermediate build. +const wtCount = assemble(WT, "judge.html", "walkthrough"); +const vfCount = assemble(VF, "verify.html", "verify"); + +// 6) Drop the intermediate builds. rmSync(WT, { recursive: true, force: true }); +rmSync(VF, { recursive: true, force: true }); -console.log(`✓ built ${SITE} (landing + /walkthrough/, ${assets.length} walkthrough assets)`); +console.log( + `✓ built ${SITE} (landing + /walkthrough/ [${wtCount} assets] + /verify/ [${vfCount} assets])`, +); diff --git a/scripts/serve-site.mjs b/scripts/serve-site.mjs new file mode 100644 index 0000000..951da7a --- /dev/null +++ b/scripts/serve-site.mjs @@ -0,0 +1,47 @@ +// Minimal static file server for the assembled site (dist/site). Used only by +// the Judge Mode Playwright harness; the real site is served by Cloudflare. +// +// node scripts/serve-site.mjs [rootDir] [port] + +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { extname, join, normalize, resolve, sep } from "node:path"; + +const ROOT = resolve(process.argv[2] ?? "dist/site"); +const PORT = Number(process.argv[3] ?? 4188); + +const TYPES = { + ".html": "text/html; charset=utf-8", + ".js": "text/javascript; charset=utf-8", + ".mjs": "text/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", + ".png": "image/png", + ".map": "application/json", +}; + +const server = createServer(async (req, res) => { + try { + let pathname = decodeURIComponent((req.url ?? "/").split("?")[0]); + if (pathname.endsWith("/")) pathname += "index.html"; + const full = normalize(join(ROOT, pathname)); + if (full !== ROOT && !full.startsWith(ROOT + sep)) { + res.writeHead(403).end("forbidden"); + return; + } + const body = await readFile(full); + res.writeHead(200, { + "content-type": TYPES[extname(full)] ?? "application/octet-stream", + "cache-control": "no-store", + }); + res.end(body); + } catch { + res.writeHead(404).end("not found"); + } +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`serving ${ROOT} on http://127.0.0.1:${PORT}`); +}); diff --git a/src/verify-ui/main.ts b/src/verify-ui/main.ts new file mode 100644 index 0000000..cfa52df --- /dev/null +++ b/src/verify-ui/main.ts @@ -0,0 +1,707 @@ +import "./styles.css"; + +import { + CANONICAL_JSON_ID, + EVALUATOR_SOURCE_SHA256, + SHA256_ALGORITHM, +} from "../verify/binding.js"; +import { checkGate } from "../verify/check.js"; +import { passingOffExample, passingOnExample } from "../verify/examples.js"; +import { REPORT_SCHEMA_VERSION } from "../verify/outcome.js"; +import { + createGateReport, + createVerifyReport, + serializeGateReportMarkdown, + serializeReportJson, + type GateReport, +} from "../verify/report.js"; +import type { ExternalActivity } from "../verify/schema.js"; +import { runGate, verifyBundle, type VerifyResult } from "../verify/verify.js"; + +// --------------------------------------------------------------------------- +// Working evidence. runGate/verifyBundle/checkGate take `unknown`, so the page +// keeps mutable working copies of the shipped examples and never re-implements +// any evaluator rule. Tampering the OFF bundle just adds the one identifiable +// activity + receipt that the shipped broken-off example carries, so the +// tampered digest matches what the CLI would compute for the same evidence. +// --------------------------------------------------------------------------- +interface WorkEvidence { + scenario: "off" | "on"; + subjectId: string; + control: { + uiPreference: string; + toggleChecked: boolean; + storedPreference: string; + backendPreference: string; + reloadObserved: boolean; + }; + activity: { + capturedActivities: ExternalActivity[]; + recommendationServiceReceipts: ExternalActivity[]; + }; + recommendations: { + feedFunctional: boolean; + renderedSource: string; + renderedItemIds: string[]; + recommendationServiceReceipts: unknown[]; + }; +} +interface WorkBundle { + schemaVersion: string; + contractFamily: string; + evidence: WorkEvidence; +} + +const LEAK_ACTIVITY: ExternalActivity = { + runId: "article-atlas-broken-off", + subjectId: "article-atlas-reader-001", + eventType: "page_view", + itemId: "article-atlas-origin-001", + clientSequence: 1, + occurredAt: "2026-01-15T12:00:01.000Z", +}; + +function freshOff(): WorkBundle { + return structuredClone(passingOffExample as unknown as WorkBundle); +} +function freshOn(): WorkBundle { + return structuredClone(passingOnExample as unknown as WorkBundle); +} +function withLeak(base: WorkBundle): WorkBundle { + const next = structuredClone(base); + next.evidence.activity.capturedActivities = [{ ...LEAK_ACTIVITY }]; + next.evidence.activity.recommendationServiceReceipts = [{ ...LEAK_ACTIVITY }]; + return next; +} + +interface State { + off: WorkBundle; + on: WorkBundle; + sealed: GateReport; + tampered: boolean; +} + +// --------------------------------------------------------------------------- +// Small DOM helpers. Every value derived from evidence is written with +// textContent, never innerHTML, so a hostile BYO file cannot inject markup. +// --------------------------------------------------------------------------- +function byId(id: string): T { + const node = document.getElementById(id); + if (node === null) { + throw new Error(`missing element #${id}`); + } + return node as T; +} +function tag( + name: K, + className?: string, + text?: string, +): HTMLElementTagNameMap[K] { + const node = document.createElement(name); + if (className !== undefined) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} +function download(name: string, text: string, mime: string): void { + const url = URL.createObjectURL(new Blob([text], { type: mime })); + const anchor = tag("a"); + anchor.href = url; + anchor.download = name; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + setTimeout(() => URL.revokeObjectURL(url), 0); +} + +interface Fact { + k: string; + v: string; + leak?: boolean; +} +function offFacts(ev: WorkEvidence): Fact[] { + const captured = ev.activity.capturedActivities.length; + const receipts = ev.activity.recommendationServiceReceipts.length; + return [ + { k: "Scenario", v: "OFF — personalization off" }, + { k: "Subject", v: ev.subjectId }, + { + k: "Preference (ui / stored / backend)", + v: `${ev.control.uiPreference} / ${ev.control.storedPreference} / ${ev.control.backendPreference}`, + }, + { k: "Reload witnessed", v: ev.control.reloadObserved ? "yes" : "no" }, + { k: "Identifiable captured activity", v: String(captured), leak: captured > 0 }, + { k: "Recommendation-service receipts", v: String(receipts), leak: receipts > 0 }, + { + k: "Feed", + v: `${ev.recommendations.renderedSource}, ${ev.recommendations.renderedItemIds.length} item(s)`, + }, + ]; +} +function onFacts(ev: WorkEvidence): Fact[] { + return [ + { k: "Scenario", v: "ON — personalization on" }, + { k: "Subject", v: ev.subjectId }, + { + k: "Preference (ui / stored / backend)", + v: `${ev.control.uiPreference} / ${ev.control.storedPreference} / ${ev.control.backendPreference}`, + }, + { + k: "Correlated activity captured", + v: String(ev.activity.capturedActivities.length), + }, + { + k: "Feed", + v: `${ev.recommendations.renderedSource}, ${ev.recommendations.renderedItemIds.length} item(s)`, + }, + ]; +} +function renderFacts(container: HTMLElement, facts: Fact[]): void { + container.replaceChildren(); + for (const fact of facts) { + const row = tag("div", `vf-fact${fact.leak ? " leak" : ""}`); + row.append(tag("span", "vf-fact-k", fact.k), tag("span", "vf-fact-v", fact.v)); + container.append(row); + } +} + +interface ClauseLike { + id: string; + passed: boolean; + expected: string; + observed: string; +} +function renderClauses( + tbody: HTMLElement, + off: readonly ClauseLike[], + on: readonly ClauseLike[], +): void { + tbody.replaceChildren(); + const scoped: Array<{ scope: string; clause: ClauseLike }> = [ + ...off.map((clause) => ({ scope: "OFF", clause })), + ...on.map((clause) => ({ scope: "ON", clause })), + ]; + for (const { scope, clause } of scoped) { + const row = tag("tr"); + + const idCell = tag("td"); + idCell.append( + tag("span", "vf-clause-scope", scope), + tag("span", "vf-clause-id", clause.id), + ); + + const resultCell = tag("td"); + resultCell.append( + tag( + "span", + `vf-tag ${clause.passed ? "pass" : "fail"}`, + clause.passed ? "PASS" : "FAIL", + ), + ); + + const detailCell = tag("td", "vf-clause-detail"); + detailCell.append( + tag("b", undefined, "Expected "), + document.createTextNode(clause.expected), + tag("br"), + tag("b", undefined, "Observed "), + document.createTextNode(clause.observed), + ); + + row.append(idCell, resultCell, detailCell); + tbody.append(row); + } +} + +interface ViolationLike { + code: string; + clause: string; + message: string; +} +function renderViolations(list: HTMLElement, violations: readonly ViolationLike[]): void { + list.replaceChildren(); + if (violations.length === 0) { + list.className = "vf-violations empty"; + list.append(tag("li", undefined, "Promise verified — no violations.")); + return; + } + list.className = "vf-violations"; + for (const violation of violations) { + const item = tag("li"); + item.append( + tag("code", undefined, violation.code), + document.createTextNode(` (${violation.clause}) — ${violation.message}`), + ); + list.append(item); + } +} + +function setPill(pill: HTMLElement, text: string, kind: "good" | "bad" | "warn" | "idle"): void { + const label = pill.querySelector(".vf-pill-label"); + pill.className = `vf-pill${kind === "idle" ? "" : ` ${kind}`}`; + pill.replaceChildren(); + if (label !== null) pill.append(label); + pill.append(document.createTextNode(text)); +} + +function outcomeKind(outcome: string): "good" | "bad" | "warn" { + if (outcome === "PASS") return "good"; + if (outcome === "BROKEN_PROMISE") return "bad"; + return "warn"; +} +function reproKind(status: string): "good" | "warn" { + return status === "BOUND_AND_REPRODUCED" ? "good" : "warn"; +} +function noteFor(outcome: string, repro: string, tampered: boolean): string { + if (outcome === "INVALID_EVIDENCE") { + return "This evidence does not pass strict validation, so the deterministic evaluator does not run on it."; + } + if (outcome === "PASS" && repro === "BOUND_AND_REPRODUCED") { + return "PASS. The deterministic evaluator verified the promise over the recorded evidence, and the sealed report reproduces exactly from it. No model graded this — an unchanged evaluator did."; + } + if (outcome === "BROKEN_PROMISE" && repro !== "BOUND_AND_REPRODUCED") { + return "Broken promise. Identifiable activity is present in the OFF evidence, so the same evaluator now fails no_identifiable_activity and raises PP_IDENTIFIABLE_EVENT_LEAK. The report sealed a moment ago no longer reproduces from this evidence — a PASS cannot be silently carried onto changed evidence."; + } + if (outcome === "BROKEN_PROMISE" && repro === "BOUND_AND_REPRODUCED") { + return "Broken promise, sealed honestly. A new report was sealed for the current evidence and it reproduces — as BROKEN_PROMISE. Sealing binds whatever is true; it cannot manufacture a PASS."; + } + if (tampered) { + return "The evidence changed. Re-evaluate it, and verify the sealed report against it."; + } + return "PASS on this evidence, but the sealed report was bound to different evidence, so it does not reproduce here."; +} + +// --------------------------------------------------------------------------- +// Page assembly. +// --------------------------------------------------------------------------- +const APP_HTML = ` +
+
+ Deterministic verifier · running in this page + contract activity-personalization/v1 +
+
+
+

Evaluator source SHA-256

+

+

Pinned to this build and verified against evaluator.ts by repository tests.

+
+
+

Canonicalization / digest

+

+

Evidence digests below are recomputed here from the evidence shown.

+
+
+

Report schema

+

+

The same verifier modules the CLI ships. No evaluator rule runs in this UI.

+
+
+
+
+
+
OFF evidencethe promise under test
+
+
+
+
ON controlthe counter-example
+
+
+
+ +
+ + + + + + + +
+ +
+
+
+ Current evidence— + Sealed report— +
+

+
+ + + +
ClauseResultExpected vs observed
+
    +
    +

    +
    +
    +
    + +
    +
    Reproduce it from a terminalbyte-identical report
    +
    +

    The browser and the CLI run the same modules, so the report you download here is byte-identical to the one the CLI writes for the same evidence.

    +
    +
    passing example
    +
    +
    $ npm run promiseproof -- gate --off passing-off.json --on passing-on.json
    +
    PASS # exit 0
    +
    $ npm run promiseproof -- check --report report.json --off passing-off.json --on passing-on.json
    +
    BOUND_AND_REPRODUCED # exit 0
    +
    +
    +
    +
    after tampering the OFF evidence
    +
    +
    $ npm run promiseproof -- gate --off tampered-off.json --on passing-on.json
    +
    BROKEN_PROMISE # exit 2 · PP_IDENTIFIABLE_EVENT_LEAK
    +
    $ npm run promiseproof -- check --report report.json --off tampered-off.json --on passing-on.json
    +
    STALE_OR_MISMATCH # exit 4
    +
    +
    +
    +
    + +
    +
    Bring your own evidenceparsed locally, never uploaded
    +
    +
    +
    + + +

    +
    +
    + + +

    +
    +
    +
    + + +
    + +

    Provide one bundle for a single verification, or both an OFF and ON bundle to run the gate. Files are read with the browser's local file API; nothing is sent anywhere.

    +
    +
    + +
    +
    What this proves, and what it doesn't
    +
    +

    What it proves. The verdict comes from one deterministic evaluator whose source is fingerprinted above. The AI that diagnoses and repairs the promise never writes PASS; this unchanged evaluator does, and you can re-derive it here or from the CLI and get the same answer.

    +

    Tamper-evidence. A sealed report is bound to the exact evidence it was computed from. Change the evidence and the report no longer reproduces (STALE_OR_MISMATCH); you can only ever seal what is actually true.

    +

    What it does not claim. Evidence is externally supplied and, here, synthetic. PromiseProof does not attest how evidence was collected — only that a bundle which passes strict validation is evaluated deterministically. Full loop and source: github.com/AlexPaiva/PromiseProof.

    +
    +
    +`; + +const state: State = { + off: freshOff(), + on: freshOn(), + sealed: undefined as unknown as GateReport, + tampered: false, +}; + +// Cached refs, populated after the shell renders. +let offFactsEl: HTMLElement; +let onFactsEl: HTMLElement; +let evOffEl: HTMLElement; +let outcomePill: HTMLElement; +let reproPill: HTMLElement; +let noteEl: HTMLElement; +let clausesEl: HTMLElement; +let violationsEl: HTMLElement; +let digestsEl: HTMLElement; +let toastEl: HTMLElement; +let tamperBtn: HTMLButtonElement; + +function toast(message: string): void { + toastEl.textContent = message; +} + +function renderDigestPairs(target: HTMLElement, entries: Array<[string, string]>): void { + target.replaceChildren(); + for (const [k, v] of entries) { + const cell = tag("div"); + cell.append(tag("p", "vf-digest-k", k), tag("p", "vf-digest-v", v)); + target.append(cell); + } +} +function gateDigestPairs(report: GateReport): Array<[string, string]> { + return [ + ["OFF evidence SHA-256", report.inputBindings.off.sha256], + ["ON evidence SHA-256", report.inputBindings.on.sha256], + ]; +} + +async function refresh(): Promise<{ outcome: string; repro: string }> { + const gate = runGate(state.off, state.on); + renderFacts(offFactsEl, offFacts(state.off.evidence)); + renderFacts(onFactsEl, onFacts(state.on.evidence)); + evOffEl.classList.toggle("tampered", state.tampered); + + setPill(outcomePill, gate.outcome, outcomeKind(gate.outcome)); + renderClauses( + clausesEl, + gate.off.evaluation?.clauses ?? [], + gate.on.evaluation?.clauses ?? [], + ); + renderViolations(violationsEl, [ + ...(gate.off.evaluation?.violations ?? []), + ...(gate.on.evaluation?.violations ?? []), + ]); + + const repro = await checkGate(state.sealed, state.off, state.on); + setPill(reproPill, repro.status, reproKind(repro.status)); + + if (gate.outcome !== "INVALID_EVIDENCE") { + renderDigestPairs(digestsEl, gateDigestPairs(await createGateReport(gate))); + } else { + digestsEl.replaceChildren(); + } + + noteEl.textContent = noteFor(gate.outcome, repro.status, state.tampered); + tamperBtn.disabled = state.tampered; + return { outcome: gate.outcome, repro: repro.status }; +} + +async function seal(): Promise { + const gate = runGate(state.off, state.on); + if (gate.outcome === "INVALID_EVIDENCE") { + toast("Cannot seal invalid evidence."); + return; + } + state.sealed = await createGateReport(gate); + const { outcome, repro } = await refresh(); + toast(`Sealed the current result: ${outcome} · ${repro}`); +} + +async function currentReport(): Promise { + const gate = runGate(state.off, state.on); + if (gate.outcome === "INVALID_EVIDENCE") return null; + return createGateReport(gate); +} + +// ---- BYO ---- +let byoOffText: string | null = null; +let byoOnText: string | null = null; + +function parseBundle(text: string): unknown { + return JSON.parse(text); +} + +function renderByo( + off: VerifyResult, + on: VerifyResult | null, + digests: Array<[string, string]>, +): void { + const result = byId("vf-byo-result"); + const outcome = byId("vf-byo-outcome"); + const issues = byId("vf-byo-issues"); + const table = byId("vf-byo-table"); + const violations = byId("vf-byo-violations"); + const digestBox = byId("vf-byo-digests"); + result.hidden = false; + + const combinedOutcome = + on === null + ? off.outcome + : off.outcome === "INVALID_EVIDENCE" || on.outcome === "INVALID_EVIDENCE" + ? "INVALID_EVIDENCE" + : off.outcome === "PASS" && on.outcome === "PASS" + ? "PASS" + : "BROKEN_PROMISE"; + setPill(outcome, combinedOutcome, outcomeKind(combinedOutcome)); + + const allIssues = [ + ...off.issues.map((issue) => (on === null ? issue : `off · ${issue}`)), + ...(on?.issues ?? []).map((issue) => `on · ${issue}`), + ]; + issues.replaceChildren(); + issues.hidden = allIssues.length === 0; + for (const issue of allIssues) issues.append(tag("li", undefined, issue)); + + const offClauses = off.evaluation?.clauses ?? []; + const onClauses = on?.evaluation?.clauses ?? []; + const hasClauses = offClauses.length + onClauses.length > 0; + table.hidden = !hasClauses; + if (hasClauses) { + renderClauses(byId("vf-byo-clauses"), offClauses, onClauses); + } + renderViolations(violations, [ + ...(off.evaluation?.violations ?? []), + ...(on?.evaluation?.violations ?? []), + ]); + renderDigestPairs(digestBox, digests); +} + +async function runByo(): Promise { + if (byoOffText === null && byoOnText === null) { + toast("Select at least one bundle file first."); + return; + } + try { + if (byoOffText !== null && byoOnText !== null) { + const off = verifyBundle(parseBundle(byoOffText)); + const on = verifyBundle(parseBundle(byoOnText)); + const gate = runGate(parseBundle(byoOffText), parseBundle(byoOnText)); + const digests = + gate.outcome === "INVALID_EVIDENCE" + ? [] + : gateDigestPairs(await createGateReport(gate)); + renderByo(off, on, digests); + } else { + const single = verifyBundle(parseBundle((byoOffText ?? byoOnText) as string)); + const digests = + single.outcome === "INVALID_EVIDENCE" + ? [] + : [ + [ + "Evidence SHA-256", + (await createVerifyReport(single)).inputBinding.sha256, + ] as [string, string], + ]; + renderByo(single, null, digests); + } + toast("Verified local evidence."); + } catch { + toast("That file is not valid JSON."); + } +} + +function wireFile( + inputId: string, + dropId: string, + nameId: string, + assign: (text: string | null, name: string) => void, +): void { + const input = byId(inputId); + const drop = byId(dropId); + const nameEl = byId(nameId); + const load = (file: File): void => { + const reader = new FileReader(); + reader.onload = () => { + assign(String(reader.result), file.name); + nameEl.textContent = file.name; + }; + reader.onerror = () => toast("Could not read that file."); + reader.readAsText(file); + }; + input.addEventListener("change", () => { + const file = input.files?.[0]; + if (file) load(file); + }); + drop.addEventListener("dragover", (event) => { + event.preventDefault(); + drop.classList.add("drag"); + }); + drop.addEventListener("dragleave", () => drop.classList.remove("drag")); + drop.addEventListener("drop", (event) => { + event.preventDefault(); + drop.classList.remove("drag"); + const file = event.dataTransfer?.files?.[0]; + if (file) load(file); + }); +} + +async function init(): Promise { + const appRoot = byId("vf-app"); + appRoot.innerHTML = APP_HTML; + + offFactsEl = byId("vf-off-facts"); + onFactsEl = byId("vf-on-facts"); + evOffEl = byId("vf-ev-off"); + outcomePill = byId("vf-outcome"); + reproPill = byId("vf-repro"); + noteEl = byId("vf-note"); + clausesEl = byId("vf-clauses"); + violationsEl = byId("vf-violations"); + digestsEl = byId("vf-digests"); + toastEl = byId("vf-toast"); + tamperBtn = byId("vf-tamper"); + + byId("vf-fp").textContent = EVALUATOR_SOURCE_SHA256; + byId("vf-canon").textContent = `${CANONICAL_JSON_ID} · ${SHA256_ALGORITHM}`; + byId("vf-schema").textContent = `report schema v${REPORT_SCHEMA_VERSION}`; + + // Seal the passing example on load so Judge Mode opens on PASS + BOUND. + state.sealed = await createGateReport(runGate(state.off, state.on)); + + tamperBtn.addEventListener("click", async () => { + state.off = withLeak(state.off); + state.tampered = true; + const { outcome, repro } = await refresh(); + toast(`Added one identifiable page_view + receipt to OFF evidence → ${outcome} · sealed report ${repro}`); + }); + byId("vf-evaluate").addEventListener("click", async () => { + const { outcome } = await refresh(); + toast(`Evaluated current evidence: ${outcome}`); + }); + byId("vf-check").addEventListener("click", async () => { + const check = await checkGate(state.sealed, state.off, state.on); + await refresh(); + toast(`Loaded report vs current evidence: ${check.status}`); + }); + byId("vf-seal").addEventListener("click", () => { + void seal(); + }); + byId("vf-reset").addEventListener("click", async () => { + state.off = freshOff(); + state.on = freshOn(); + state.tampered = false; + state.sealed = await createGateReport(runGate(state.off, state.on)); + await refresh(); + toast("Reset to the passing example."); + }); + byId("vf-dl-json").addEventListener("click", async () => { + const report = await currentReport(); + if (report === null) { + toast("No report for invalid evidence."); + return; + } + download("promiseproof-report.json", serializeReportJson(report), "application/json"); + }); + byId("vf-dl-md").addEventListener("click", async () => { + const report = await currentReport(); + if (report === null) { + toast("No report for invalid evidence."); + return; + } + download("promiseproof-report.md", serializeGateReportMarkdown(report), "text/markdown"); + }); + + wireFile("vf-file-off", "vf-drop-off", "vf-name-off", (text) => { + byoOffText = text; + }); + wireFile("vf-file-on", "vf-drop-on", "vf-name-on", (text) => { + byoOnText = text; + }); + byId("vf-byo-run").addEventListener("click", () => { + void runByo(); + }); + byId("vf-byo-clear").addEventListener("click", () => { + byoOffText = null; + byoOnText = null; + byId("vf-name-off").textContent = ""; + byId("vf-name-on").textContent = ""; + byId("vf-file-off").value = ""; + byId("vf-file-on").value = ""; + byId("vf-byo-result").hidden = true; + }); + + await refresh(); +} + +void init(); diff --git a/src/verify-ui/styles.css b/src/verify-ui/styles.css new file mode 100644 index 0000000..9ce33fb --- /dev/null +++ b/src/verify-ui/styles.css @@ -0,0 +1,696 @@ +/* PromiseProof Verify — Judge Mode. Dark control-console styling on the same + palette and system type as the public landing, so /verify/ reads as one site. + No web fonts (CSP-safe, offline-safe); state is carried by semantic colour. */ + +:root { + --bg: #080b12; + --bg2: #0c111c; + --panel: #0e1524; + --panel2: #121a2b; + --line: rgba(255, 255, 255, 0.1); + --line-soft: rgba(255, 255, 255, 0.06); + --text: #e8edf6; + --muted: #8ea3c4; + --muted2: #7c8ca6; + --blue: #3f6bf0; + --blue-bright: #6f9bff; + --red: #e5484d; + --red-bright: #ff8f93; + --green: #2fa968; + --green-bright: #5ad39a; + --amber: #e0a63b; + --amber-bright: #f2c66b; + --mono: ui-monospace, "SF Mono", Menlo, Consolas, monospace; + --sans: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + --maxw: 1080px; + --radius: 14px; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + font-family: var(--sans); + background: + radial-gradient(1200px 600px at 50% -8%, rgba(63, 107, 240, 0.14), transparent 60%), + var(--bg); + color: var(--text); + line-height: 1.55; + -webkit-font-smoothing: antialiased; + overflow-x: hidden; +} + +.mono { + font-family: var(--mono); +} + +a { + color: var(--blue-bright); + text-decoration: none; +} +a:hover { + text-decoration: underline; +} + +:focus-visible { + outline: 2px solid var(--blue-bright); + outline-offset: 2px; + border-radius: 6px; +} + +.vf-shell { + max-width: var(--maxw); + margin: 0 auto; + padding: 0 22px 72px; +} + +/* ---- Topbar ---- */ +.vf-topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + flex-wrap: wrap; + padding: 20px 0 18px; + border-bottom: 1px solid var(--line-soft); +} +.vf-brand { + display: inline-flex; + align-items: center; + gap: 12px; + color: var(--text); +} +.vf-brand:hover { + text-decoration: none; +} +.vf-mark { + width: 30px; + height: 30px; + border-radius: 8px; + background: linear-gradient(160deg, #12203b, #0a1120); + display: grid; + place-items: center; + gap: 2px; + grid-auto-flow: column; + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.08); +} +.vf-mark span { + width: 3px; + height: 12px; + border-radius: 2px; + background: var(--green-bright); +} +.vf-mark span:nth-child(2) { + height: 8px; + background: var(--blue-bright); +} +.vf-mark span:nth-child(3) { + height: 15px; + background: var(--green); +} +.vf-brand-text strong { + display: block; + font-size: 0.98rem; + letter-spacing: 0.01em; +} +.vf-brand-text small { + display: block; + color: var(--muted); + font-size: 0.72rem; +} +.vf-topbar-actions { + display: flex; + align-items: center; + gap: 14px; + flex-wrap: wrap; +} +.vf-chip { + margin: 0; + display: inline-flex; + align-items: center; + gap: 8px; + font-family: var(--mono); + font-size: 0.66rem; + letter-spacing: 0.04em; + color: #93c2a8; + border: 1px solid rgba(90, 211, 154, 0.25); + padding: 6px 12px; + border-radius: 999px; +} +.vf-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--green-bright); + box-shadow: 0 0 0 3px rgba(90, 211, 154, 0.18); +} +.vf-toplink { + font-family: var(--mono); + font-size: 0.72rem; + letter-spacing: 0.03em; + color: var(--muted); +} + +/* ---- Hero ---- */ +.vf-hero { + padding: 44px 0 30px; +} +.vf-eyebrow { + display: inline-flex; + align-items: center; + gap: 9px; + font-family: var(--mono); + font-size: 0.72rem; + letter-spacing: 0.1em; + text-transform: uppercase; + color: #93c2a8; + border: 1px solid rgba(90, 211, 154, 0.25); + padding: 6px 13px; + border-radius: 999px; + margin: 0 0 22px; +} +.vf-h1 { + font-size: clamp(2.1rem, 5.4vw, 3.2rem); + line-height: 1.04; + letter-spacing: -0.02em; + margin: 0 0 18px; + text-wrap: balance; +} +.vf-h1 .g { + background: linear-gradient(100deg, #9db8ff, #6f9bff 60%, #5ad39a); + -webkit-background-clip: text; + background-clip: text; + color: transparent; +} +.vf-lede { + max-width: 68ch; + font-size: 1.06rem; + color: #cfd8ea; + margin: 0 0 16px; +} +.vf-thesis { + max-width: 62ch; + font-family: var(--mono); + font-size: 0.82rem; + color: var(--green-bright); + border-left: 2px solid rgba(90, 211, 154, 0.4); + padding-left: 14px; + margin: 0; +} + +/* ---- App container ---- */ +.vf-app { + display: grid; + gap: 22px; +} +.vf-fallback { + border: 1px solid var(--line); + border-radius: var(--radius); + padding: 22px; + color: var(--muted); + background: var(--panel); +} + +.vf-panel { + border: 1px solid var(--line); + border-radius: 16px; + background: linear-gradient(180deg, rgba(18, 26, 42, 0.72), rgba(10, 15, 25, 0.72)); + box-shadow: 0 30px 70px -46px rgba(0, 0, 0, 0.8); + overflow: hidden; +} +.vf-panel-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + padding: 13px 20px; + border-bottom: 1px solid var(--line-soft); + font-family: var(--mono); + font-size: 0.62rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted); +} +.vf-panel-body { + padding: 20px; +} +.vf-section-title { + font-size: 1.02rem; + margin: 0; + letter-spacing: -0.01em; +} + +/* ---- Authority strip ---- */ +.vf-authority { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); + gap: 1px; + background: var(--line-soft); +} +.vf-auth-cell { + background: var(--panel); + padding: 14px 18px; +} +.vf-auth-k { + font-family: var(--mono); + font-size: 0.58rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted2); + margin: 0 0 5px; +} +.vf-auth-v { + font-family: var(--mono); + font-size: 0.76rem; + color: var(--text); + word-break: break-all; + line-height: 1.5; +} +.vf-auth-note { + color: var(--muted); + font-size: 0.68rem; + margin-top: 4px; + font-family: var(--sans); + letter-spacing: 0; +} + +/* ---- Evidence panels ---- */ +.vf-evidence { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 16px; +} +.vf-ev { + border: 1px solid var(--line); + border-radius: 13px; + background: var(--panel); + overflow: hidden; +} +.vf-ev-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + padding: 11px 16px; + border-bottom: 1px solid var(--line-soft); + font-family: var(--mono); + font-size: 0.64rem; + letter-spacing: 0.09em; + text-transform: uppercase; + color: var(--muted); +} +.vf-ev-badge { + font-weight: 700; + color: var(--blue-bright); +} +.vf-ev-body { + padding: 6px 16px 14px; +} +.vf-fact { + display: flex; + justify-content: space-between; + gap: 12px; + padding: 7px 0; + border-bottom: 1px dashed var(--line-soft); + font-size: 0.82rem; +} +.vf-fact:last-child { + border-bottom: none; +} +.vf-fact-k { + color: var(--muted); +} +.vf-fact-v { + font-family: var(--mono); + font-size: 0.78rem; + text-align: right; + word-break: break-word; +} +.vf-fact.leak { + color: var(--red-bright); +} +.vf-fact.leak .vf-fact-k { + color: var(--red-bright); +} +.vf-fact.leak .vf-fact-v { + color: var(--red-bright); + font-weight: 700; +} +.vf-ev.tampered { + border-color: rgba(229, 72, 77, 0.55); + box-shadow: 0 0 0 1px rgba(229, 72, 77, 0.25); +} + +/* ---- Actions ---- */ +.vf-actions { + display: flex; + flex-wrap: wrap; + gap: 10px; +} +.vf-btn { + appearance: none; + font: inherit; + font-size: 0.84rem; + font-weight: 600; + cursor: pointer; + border-radius: 10px; + padding: 10px 16px; + border: 1px solid var(--line); + background: rgba(255, 255, 255, 0.04); + color: var(--text); + transition: transform 0.14s ease, border-color 0.14s ease, background 0.14s ease; +} +.vf-btn:hover { + border-color: rgba(255, 255, 255, 0.24); + transform: translateY(-1px); +} +.vf-btn:active { + transform: translateY(0); +} +.vf-btn:disabled { + opacity: 0.42; + cursor: not-allowed; + transform: none; +} +.vf-btn-tamper { + border-color: rgba(229, 72, 77, 0.5); + background: linear-gradient(180deg, rgba(229, 72, 77, 0.18), rgba(229, 72, 77, 0.06)); + color: #ffd9da; +} +.vf-btn-tamper:hover { + border-color: rgba(229, 72, 77, 0.8); +} +.vf-btn-primary { + border-color: rgba(111, 155, 255, 0.5); + background: linear-gradient(180deg, rgba(63, 107, 240, 0.28), rgba(63, 107, 240, 0.1)); + color: #dfe8ff; +} +.vf-btn-quiet { + color: var(--muted); + background: transparent; +} + +/* ---- Result ---- */ +.vf-verdict { + display: flex; + align-items: center; + gap: 12px; + flex-wrap: wrap; + margin-bottom: 6px; +} +.vf-pill { + display: inline-flex; + align-items: center; + gap: 8px; + font-family: var(--mono); + font-size: 0.74rem; + font-weight: 700; + letter-spacing: 0.06em; + padding: 7px 13px; + border-radius: 999px; + border: 1px solid var(--line); + color: var(--muted); + background: rgba(255, 255, 255, 0.03); +} +.vf-pill-label { + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--muted2); + margin-right: 2px; +} +.vf-pill.good { + color: var(--green-bright); + border-color: rgba(90, 211, 154, 0.45); + background: rgba(47, 169, 104, 0.12); +} +.vf-pill.bad { + color: var(--red-bright); + border-color: rgba(229, 72, 77, 0.5); + background: rgba(229, 72, 77, 0.12); +} +.vf-pill.warn { + color: var(--amber-bright); + border-color: rgba(224, 166, 59, 0.5); + background: rgba(224, 166, 59, 0.12); +} +.vf-result-note { + color: #cfd8ea; + font-size: 0.9rem; + margin: 10px 0 16px; +} +.vf-result-note strong { + color: var(--text); +} + +.vf-clauses { + width: 100%; + border-collapse: collapse; + font-size: 0.82rem; + margin: 4px 0 6px; +} +.vf-clauses th { + text-align: left; + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted2); + font-weight: 600; + padding: 6px 10px; + border-bottom: 1px solid var(--line-soft); +} +.vf-clauses td { + padding: 9px 10px; + border-bottom: 1px solid var(--line-soft); + vertical-align: top; +} +.vf-clauses tr:last-child td { + border-bottom: none; +} +.vf-clause-id { + font-family: var(--mono); + font-size: 0.76rem; + color: var(--text); +} +.vf-clause-scope { + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--muted2); + display: block; +} +.vf-clause-detail { + color: var(--muted); + font-size: 0.74rem; + font-family: var(--mono); + line-height: 1.5; +} +.vf-clause-detail b { + color: #c7d2e6; + font-weight: 600; +} +.vf-tag { + font-family: var(--mono); + font-size: 0.6rem; + font-weight: 700; + letter-spacing: 0.06em; + padding: 3px 8px; + border-radius: 6px; + white-space: nowrap; +} +.vf-tag.pass { + color: var(--green-bright); + background: rgba(47, 169, 104, 0.14); +} +.vf-tag.fail { + color: var(--red-bright); + background: rgba(229, 72, 77, 0.16); +} + +.vf-violations { + list-style: none; + margin: 12px 0 0; + padding: 0; + display: grid; + gap: 8px; +} +.vf-violations li { + border: 1px solid rgba(229, 72, 77, 0.32); + background: rgba(229, 72, 77, 0.08); + border-radius: 10px; + padding: 10px 13px; + font-size: 0.82rem; +} +.vf-violations code { + font-family: var(--mono); + font-size: 0.76rem; + color: var(--red-bright); + font-weight: 700; +} +.vf-violations.empty li { + border-color: rgba(90, 211, 154, 0.3); + background: rgba(47, 169, 104, 0.08); + color: var(--green-bright); +} + +.vf-digests { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 10px; + margin-top: 16px; + padding-top: 14px; + border-top: 1px solid var(--line-soft); +} +.vf-digest-k { + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted2); +} +.vf-digest-v { + font-family: var(--mono); + font-size: 0.72rem; + color: #c7d2e6; + word-break: break-all; +} + +/* ---- CLI mirror ---- */ +.vf-cli { + background: #0b111c; + border: 1px solid var(--line); + border-radius: 13px; + overflow: hidden; + font-family: var(--mono); + font-size: 0.78rem; +} +.vf-cli-head { + padding: 9px 15px; + border-bottom: 1px solid var(--line-soft); + color: var(--muted2); + font-size: 0.6rem; + letter-spacing: 0.11em; + text-transform: uppercase; +} +.vf-cli-body { + padding: 14px 16px; + display: grid; + gap: 6px; + overflow-x: auto; +} +.vf-cli-line { + white-space: pre; + color: #c7d2e6; +} +.vf-cli-line .p { + color: var(--green-bright); +} +.vf-cli-line .c { + color: var(--muted2); +} +.vf-cli-line .ok { + color: var(--green-bright); +} +.vf-cli-line .no { + color: var(--red-bright); +} + +/* ---- BYO ---- */ +.vf-byo-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 14px; + margin-bottom: 14px; +} +.vf-drop { + border: 1px dashed rgba(255, 255, 255, 0.22); + border-radius: 12px; + padding: 16px; + background: rgba(255, 255, 255, 0.02); + transition: border-color 0.14s ease, background 0.14s ease; +} +.vf-drop.drag { + border-color: var(--blue-bright); + background: rgba(63, 107, 240, 0.1); +} +.vf-drop label { + display: block; + font-family: var(--mono); + font-size: 0.62rem; + letter-spacing: 0.09em; + text-transform: uppercase; + color: var(--muted); + margin-bottom: 8px; +} +.vf-drop input[type="file"] { + font-family: var(--mono); + font-size: 0.74rem; + color: var(--muted); + max-width: 100%; +} +.vf-drop-name { + margin-top: 8px; + font-family: var(--mono); + font-size: 0.72rem; + color: var(--green-bright); + word-break: break-all; +} +.vf-issues { + list-style: none; + margin: 10px 0 0; + padding: 0; + display: grid; + gap: 6px; +} +.vf-issues li { + font-family: var(--mono); + font-size: 0.72rem; + color: var(--amber-bright); + border-left: 2px solid rgba(224, 166, 59, 0.5); + padding-left: 10px; +} + +.vf-muted { + color: var(--muted); + font-size: 0.82rem; +} +.vf-disclose p { + color: #cfd8ea; + font-size: 0.88rem; +} +.vf-disclose strong { + color: var(--text); +} + +.vf-footer { + margin-top: 20px; + padding-top: 18px; + border-top: 1px solid var(--line-soft); + color: var(--muted2); + font-size: 0.78rem; + max-width: 76ch; +} + +/* ---- Responsive ---- */ +@media (max-width: 720px) { + .vf-evidence, + .vf-byo-grid { + grid-template-columns: 1fr; + } + .vf-shell { + padding: 0 16px 56px; + } +} + +@media (prefers-reduced-motion: reduce) { + * { + transition: none !important; + animation: none !important; + } +} diff --git a/tests/verify/judge-mode.playwright.config.ts b/tests/verify/judge-mode.playwright.config.ts new file mode 100644 index 0000000..e2ed3f0 --- /dev/null +++ b/tests/verify/judge-mode.playwright.config.ts @@ -0,0 +1,34 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { defineConfig } from "@playwright/test"; + +// Serves the assembled combined site (dist/site) statically and drives the +// hosted verifier at /verify/. The verifier is a static, self-contained page, +// so it needs no application server — only the built assets. +const PORT = 4188; +const BASE = `http://127.0.0.1:${PORT}`; +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +export default defineConfig({ + testDir: ".", + testMatch: /judge-mode\.spec\.ts/, + fullyParallel: false, + workers: 1, + retries: 0, + reporter: [["list"]], + timeout: 30_000, + use: { + baseURL: BASE, + trace: "retain-on-failure", + }, + webServer: { + command: `npm run build:site && node scripts/serve-site.mjs dist/site ${PORT}`, + cwd: REPO_ROOT, + url: `${BASE}/verify/`, + reuseExistingServer: false, + timeout: 180_000, + stdout: "pipe", + stderr: "pipe", + }, +}); diff --git a/tests/verify/judge-mode.spec.ts b/tests/verify/judge-mode.spec.ts new file mode 100644 index 0000000..4e06a78 --- /dev/null +++ b/tests/verify/judge-mode.spec.ts @@ -0,0 +1,144 @@ +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { expect, test, type Page, type Request } from "@playwright/test"; + +import { EVALUATOR_SOURCE_SHA256 } from "../../src/verify/binding.js"; +import { passingOffExample, passingOnExample } from "../../src/verify/examples.js"; + +const ROUTE = "/verify/"; +const REPO = process.cwd(); + +async function open(page: Page): Promise { + await page.goto(ROUTE); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); +} + +test("Judge Mode opens on PASS + BOUND with the pinned evaluator fingerprint", async ({ + page, +}) => { + await open(page); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + + // Five clause rows, all PASS, in the healthy state. + const rows = page.getByTestId("clauses").locator("tr"); + await expect(rows).toHaveCount(5); + await expect(page.getByTestId("clauses").locator(".vf-tag.fail")).toHaveCount(0); + + // The fingerprint is the pinned build value, verbatim, not recomputed copy. + await expect(page.locator("#vf-fp")).toHaveText(EVALUATOR_SOURCE_SHA256); + await expect(page.getByTestId("violations")).toContainText("no violations"); +}); + +test("nothing leaves the page: no cross-origin, fetch, socket, or POST traffic", async ({ + page, + baseURL, +}) => { + const offenders: string[] = []; + const record = (request: Request): void => { + const url = request.url(); + const sameOrigin = baseURL !== undefined && url.startsWith(baseURL); + const local = url.startsWith("blob:") || url.startsWith("data:") || url.startsWith("about:"); + const chatty = ["fetch", "xhr", "websocket", "eventsource"].includes( + request.resourceType(), + ); + if ((!sameOrigin && !local) || chatty || request.method() === "POST") { + offenders.push(`${request.method()} ${request.resourceType()} ${url}`); + } + }; + page.on("request", record); + + await open(page); + // Exercise every interactive path that could plausibly phone home. + await page.getByTestId("tamper").click(); + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + await page.getByTestId("evaluate").click(); + await page.getByTestId("check").click(); + const [download] = await Promise.all([ + page.waitForEvent("download"), + page.locator("#vf-dl-json").click(), + ]); + await download.path(); + await page.waitForTimeout(200); + + expect(offenders, `unexpected egress:\n${offenders.join("\n")}`).toEqual([]); +}); + +test("tampering the OFF evidence flips the same evaluator to a broken promise", async ({ + page, +}) => { + await open(page); + await page.getByTestId("tamper").click(); + + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + await expect(page.getByTestId("repro-pill")).toContainText("STALE_OR_MISMATCH"); + await expect(page.getByTestId("violations")).toContainText("PP_IDENTIFIABLE_EVENT_LEAK"); + + // The failing clause is the identifiable-activity clause, and the OFF panel is flagged. + const failing = page.getByTestId("clauses").locator("tr", { + has: page.locator(".vf-tag.fail"), + }); + await expect(failing).toContainText("no_identifiable_activity"); + await expect(page.locator("#vf-ev-off")).toHaveClass(/tampered/); +}); + +test("you can only seal the truth: re-sealing tampered evidence binds BROKEN, not PASS", async ({ + page, +}) => { + await open(page); + await page.getByTestId("tamper").click(); + await expect(page.getByTestId("repro-pill")).toContainText("STALE_OR_MISMATCH"); + + await page.getByTestId("seal").click(); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + + // Reset returns to the passing, reproduced baseline. + await page.getByTestId("reset").click(); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); +}); + +test("bring-your-own invalid evidence surfaces validation issues, not a verdict", async ({ + page, +}) => { + await open(page); + const bogus = JSON.stringify({ not: "a bundle" }); + await page + .locator("#vf-file-off") + .setInputFiles({ name: "bad.json", mimeType: "application/json", buffer: Buffer.from(bogus) }); + // Wait for the local FileReader to finish before verifying. + await expect(page.locator("#vf-name-off")).toHaveText("bad.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toBeVisible(); +}); + +test("parity: the report the browser downloads is byte-identical to the CLI's", async ({ + page, +}) => { + await open(page); + const [download] = await Promise.all([ + page.waitForEvent("download"), + page.locator("#vf-dl-json").click(), + ]); + const browserReport = readFileSync(await download.path(), "utf8"); + + // Produce the CLI report for the same OFF/ON examples the page ships. + const kit = mkdtempSync(path.join(tmpdir(), "pp-parity-")); + const offPath = path.join(kit, "off.json"); + const onPath = path.join(kit, "on.json"); + writeFileSync(offPath, `${JSON.stringify(passingOffExample, null, 2)}\n`); + writeFileSync(onPath, `${JSON.stringify(passingOnExample, null, 2)}\n`); + execFileSync( + "npx", + ["tsx", "src/verify/cli.ts", "gate", "--off", offPath, "--on", onPath, "--out", kit], + { cwd: REPO, stdio: "pipe", shell: process.platform === "win32" }, + ); + const cliReport = readFileSync(path.join(kit, "report.json"), "utf8"); + + expect(browserReport).toBe(cliReport); +}); diff --git a/verify.html b/verify.html new file mode 100644 index 0000000..92ec0b3 --- /dev/null +++ b/verify.html @@ -0,0 +1,103 @@ + + + + + + + + + + PromiseProof Verify: re-derive the verdict yourself + + + + + + + + + + + + + + + + + + + + +
    +
    + + + + PromiseProof + Verify — re-derive the verdict yourself. + + +
    +

    + + Runs in your browser. No network, no upload. +

    + Walkthrough → + ← Overview +
    +
    + +
    +
    +

    PromiseProof Verify · Judge Mode

    +

    Re-derive the verdict yourself.

    +

    + A user turned activity-based personalization off. This page runs the + same frozen, deterministic evaluator the repository ships — right here in your + browser, with no network call — over the recorded evidence. It says + PASS. Now tamper the evidence and watch the same evaluator + return a broken promise, while the report you sealed a moment ago refuses to reproduce. +

    +

    + The AI that repairs the promise never owns PASS. An unchanged evaluator does — + and you are about to run it. +

    +
    + +
    +

    + This verifier is interactive and needs JavaScript. The same deterministic evaluator, + evidence, and reports are in the + GitHub repository, and a + plain-language summary is on the overview page. +

    +
    +
    + +
    +

    + Evidence here is externally supplied and synthetic; PromiseProof does not attest how it + was collected. Only the deterministic evaluator decides a bundle that passes strict + validation. Nothing you load or tamper leaves this page. +

    +
    +
    + + + + diff --git a/vite.config.ts b/vite.config.ts index 20c4c11..86d1602 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -12,9 +12,12 @@ export default defineConfig({ emptyOutDir: true, rollupOptions: { input: { - // The seeded Signal Shelf app and the separate judge walkthrough. + // The seeded Signal Shelf app, the judge walkthrough, and the hosted + // deterministic verifier (Judge Mode). build:site builds each page with + // the base its sub-path needs. main: fileURLToPath(new URL("./index.html", import.meta.url)), judge: fileURLToPath(new URL("./judge.html", import.meta.url)), + verify: fileURLToPath(new URL("./verify.html", import.meta.url)), }, }, }, From f305b582842d4ccfe2a0c789d440c922776a4de2 Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Tue, 21 Jul 2026 00:56:15 +0100 Subject: [PATCH 6/7] test: harden and gate hosted verifier --- .github/workflows/submission-hardening.yml | 3 + src/verify-ui/main.ts | 42 ++- src/verify/check.ts | 106 +++++- src/verify/cli.ts | 49 +-- tests/verify/check.unit.ts | 67 ++++ tests/verify/cli.unit.ts | 49 +++ tests/verify/judge-mode.spec.ts | 379 +++++++++++++++++++-- 7 files changed, 628 insertions(+), 67 deletions(-) diff --git a/.github/workflows/submission-hardening.yml b/.github/workflows/submission-hardening.yml index 2623227..0541546 100644 --- a/.github/workflows/submission-hardening.yml +++ b/.github/workflows/submission-hardening.yml @@ -39,6 +39,7 @@ jobs: - run: npx playwright install chromium - run: npm run build - run: npm run evidence:verify + - run: npm run test:external - run: npm run test:evidence:unit - run: npm run test:investigation:unit - run: npm run test:investigation:offline @@ -72,6 +73,7 @@ jobs: - run: npm ci - run: npx playwright install chromium - run: npm test + - run: npm run test:external:pipeline - run: npm run test:expected-red repair-rehearsal: @@ -138,6 +140,7 @@ jobs: - run: npm ci - run: npx playwright install chromium - run: npm run build:site + - run: npm run test:verify:judge - run: npm run test:judge:unit - run: npm run test:judge:interface - run: npm run demo:rehearse diff --git a/src/verify-ui/main.ts b/src/verify-ui/main.ts index cfa52df..f55dbe7 100644 --- a/src/verify-ui/main.ts +++ b/src/verify-ui/main.ts @@ -15,7 +15,10 @@ import { serializeReportJson, type GateReport, } from "../verify/report.js"; -import type { ExternalActivity } from "../verify/schema.js"; +import { + MAX_INPUT_BYTES, + type ExternalActivity, +} from "../verify/schema.js"; import { runGate, verifyBundle, type VerifyResult } from "../verify/verify.js"; // --------------------------------------------------------------------------- @@ -506,6 +509,7 @@ function renderByo( off: VerifyResult, on: VerifyResult | null, digests: Array<[string, string]>, + gateIssues: readonly string[] = [], ): void { const result = byId("vf-byo-result"); const outcome = byId("vf-byo-outcome"); @@ -526,6 +530,7 @@ function renderByo( setPill(outcome, combinedOutcome, outcomeKind(combinedOutcome)); const allIssues = [ + ...gateIssues, ...off.issues.map((issue) => (on === null ? issue : `off · ${issue}`)), ...(on?.issues ?? []).map((issue) => `on · ${issue}`), ]; @@ -554,14 +559,12 @@ async function runByo(): Promise { } try { if (byoOffText !== null && byoOnText !== null) { - const off = verifyBundle(parseBundle(byoOffText)); - const on = verifyBundle(parseBundle(byoOnText)); const gate = runGate(parseBundle(byoOffText), parseBundle(byoOnText)); const digests = gate.outcome === "INVALID_EVIDENCE" ? [] : gateDigestPairs(await createGateReport(gate)); - renderByo(off, on, digests); + renderByo(gate.off, gate.on, digests, gate.issues); } else { const single = verifyBundle(parseBundle((byoOffText ?? byoOnText) as string)); const digests = @@ -577,10 +580,25 @@ async function runByo(): Promise { } toast("Verified local evidence."); } catch { + renderByoInvalid(": malformed JSON"); toast("That file is not valid JSON."); } } +function renderByoInvalid(issue: string): void { + const result = byId("vf-byo-result"); + const outcome = byId("vf-byo-outcome"); + const issues = byId("vf-byo-issues"); + result.hidden = false; + setPill(outcome, "INVALID_EVIDENCE", "warn"); + issues.replaceChildren(tag("li", undefined, issue)); + issues.hidden = false; + byId("vf-byo-table").hidden = true; + byId("vf-byo-clauses").replaceChildren(); + byId("vf-byo-violations").replaceChildren(); + byId("vf-byo-digests").replaceChildren(); +} + function wireFile( inputId: string, dropId: string, @@ -591,12 +609,26 @@ function wireFile( const drop = byId(dropId); const nameEl = byId(nameId); const load = (file: File): void => { + assign(null, file.name); + byId("vf-byo-result").hidden = true; + if (file.size > MAX_INPUT_BYTES) { + const issue = `: input exceeds ${MAX_INPUT_BYTES}-byte limit`; + nameEl.textContent = `${file.name} — rejected`; + input.value = ""; + renderByoInvalid(issue); + toast("That file exceeds the verifier input-size limit."); + return; + } const reader = new FileReader(); reader.onload = () => { assign(String(reader.result), file.name); nameEl.textContent = file.name; }; - reader.onerror = () => toast("Could not read that file."); + reader.onerror = () => { + assign(null, file.name); + renderByoInvalid(": file could not be read"); + toast("Could not read that file."); + }; reader.readAsText(file); }; input.addEventListener("change", () => { diff --git a/src/verify/check.ts b/src/verify/check.ts index 6d53a64..62aa516 100644 --- a/src/verify/check.ts +++ b/src/verify/check.ts @@ -5,8 +5,15 @@ // evidence digest and evaluator fingerprint while inventing the verdict, clauses, // or violations. Full reproduction catches that. import { z } from "zod"; -import { canonicalizeJson } from "./binding.js"; -import { REPORT_SCHEMA_VERSION } from "./outcome.js"; +import { + CANONICAL_JSON_ID, + canonicalizeJson, + SHA256_ALGORITHM, +} from "./binding.js"; +import { + REPORT_SCHEMA_VERSION, + SUPPORTED_CONTRACT_FAMILY, +} from "./outcome.js"; import { createGateReport, createVerifyReport } from "./report.js"; import { runGate, verifyBundle } from "./verify.js"; @@ -24,12 +31,91 @@ export interface CheckResult { readonly detail: string; } -// Minimal envelope: the supplied document must at least declare itself a report -// of the current schema version to be checkable. Anything else is INVALID (not a -// mere mismatch); a well-formed report that fails to reproduce is a mismatch. -const reportEnvelope = z - .object({ schemaVersion: z.literal(REPORT_SCHEMA_VERSION) }) - .passthrough(); +const sha256 = z.string().regex(/^[0-9a-f]{64}$/u); +const inputBindingSchema = z + .object({ + canonicalization: z.literal(CANONICAL_JSON_ID), + algorithm: z.literal(SHA256_ALGORITHM), + sha256, + }) + .strict(); +const clauseIdSchema = z.enum([ + "no_identifiable_activity", + "contextual_feed_functional", + "preference_survives_reload", + "expected_activity_received", + "behavioral_feed_functional", +]); +const violationCodeSchema = z.enum([ + "PP_IDENTIFIABLE_EVENT_LEAK", + "PP_CONTEXTUAL_FEED_MISSING", + "PP_PREFERENCE_NOT_PERSISTED", + "PP_EXPECTED_ACTIVITY_MISSING", + "PP_BEHAVIORAL_FEED_MISSING", +]); +const clauseSchema = z + .object({ + id: clauseIdSchema, + passed: z.boolean(), + expected: z.string(), + observed: z.string(), + }) + .strict(); +const violationSchema = z + .object({ + code: violationCodeSchema, + clause: clauseIdSchema, + message: z.string(), + }) + .strict(); +const evaluationShape = { + scenario: z.enum(["off", "on"]), + canonicalVerdict: z.enum(["pass", "fail"]), + clauses: z.array(clauseSchema), + violations: z.array(violationSchema), +} as const; +const authoritySchema = z + .object({ + evidenceSource: z.string(), + collectionAttested: z.boolean(), + evaluation: z.string(), + evaluatorSourceSha256: sha256, + }) + .strict(); +const reportHeaderShape = { + schemaVersion: z.literal(REPORT_SCHEMA_VERSION), + contractFamily: z.literal(SUPPORTED_CONTRACT_FAMILY), + outcome: z.enum(["PASS", "BROKEN_PROMISE"]), +} as const; + +// Report parsing validates only the versioned public report structure. Values +// that are structurally valid but forged (for example, a valid-shaped PASS over +// broken evidence or a different valid-shaped evaluator fingerprint) proceed to +// full reproduction and become STALE_OR_MISMATCH. They are not mistaken for a +// malformed report. +const verifyReportSchema = z + .object({ + ...reportHeaderShape, + inputBinding: inputBindingSchema, + ...evaluationShape, + authority: authoritySchema, + }) + .strict(); +const gateReportSchema = z + .object({ + ...reportHeaderShape, + inputBindings: z + .object({ off: inputBindingSchema, on: inputBindingSchema }) + .strict(), + evaluations: z + .object({ + off: z.object(evaluationShape).strict(), + on: z.object(evaluationShape).strict(), + }) + .strict(), + authority: authoritySchema, + }) + .strict(); function invalid(detail: string): CheckResult { return { status: "INVALID_REPORT_OR_EVIDENCE", detail }; @@ -54,7 +140,7 @@ export async function checkSingle( rawReport: unknown, rawEvidence: unknown, ): Promise { - if (!reportEnvelope.safeParse(rawReport).success) { + if (!verifyReportSchema.safeParse(rawReport).success) { return invalid( `report is not a schemaVersion ${REPORT_SCHEMA_VERSION} verification report`, ); @@ -73,7 +159,7 @@ export async function checkGate( rawOff: unknown, rawOn: unknown, ): Promise { - if (!reportEnvelope.safeParse(rawReport).success) { + if (!gateReportSchema.safeParse(rawReport).success) { return invalid( `report is not a schemaVersion ${REPORT_SCHEMA_VERSION} gate report`, ); diff --git a/src/verify/cli.ts b/src/verify/cli.ts index ceca74f..cc4ac46 100644 --- a/src/verify/cli.ts +++ b/src/verify/cli.ts @@ -361,27 +361,36 @@ async function runCheck(args: readonly string[], io: CliIo): Promise { const hasEvidence = args.includes("--evidence"); const hasGate = args.includes("--off") || args.includes("--on"); - if (hasEvidence && !hasGate) { - const options = parseOptions(args, ["report", "evidence"]); - const [report, evidence] = await Promise.all([ - readExternalJson(options.report!), - readExternalJson(options.evidence!), - ]); - const result = await checkSingle(report, evidence); - io.stdout(`${result.status}: ${result.detail}`); - return CHECK_EXIT[result.status]; - } + try { + if (hasEvidence && !hasGate) { + const options = parseOptions(args, ["report", "evidence"]); + const [report, evidence] = await Promise.all([ + readExternalJson(options.report!), + readExternalJson(options.evidence!), + ]); + const result = await checkSingle(report, evidence); + io.stdout(`${result.status}: ${result.detail}`); + return CHECK_EXIT[result.status]; + } - if (hasGate && !hasEvidence) { - const options = parseOptions(args, ["report", "off", "on"]); - const [report, off, on] = await Promise.all([ - readExternalJson(options.report!), - readExternalJson(options.off!), - readExternalJson(options.on!), - ]); - const result = await checkGate(report, off, on); - io.stdout(`${result.status}: ${result.detail}`); - return CHECK_EXIT[result.status]; + if (hasGate && !hasEvidence) { + const options = parseOptions(args, ["report", "off", "on"]); + const [report, off, on] = await Promise.all([ + readExternalJson(options.report!), + readExternalJson(options.off!), + readExternalJson(options.on!), + ]); + const result = await checkGate(report, off, on); + io.stdout(`${result.status}: ${result.detail}`); + return CHECK_EXIT[result.status]; + } + } catch (error) { + if (error instanceof InvalidEvidenceFileError) { + io.stderr("INVALID_REPORT_OR_EVIDENCE"); + io.stderr(`- ${error.message}`); + return CHECK_EXIT.INVALID_REPORT_OR_EVIDENCE; + } + throw error; } throw new UsageError( diff --git a/tests/verify/check.unit.ts b/tests/verify/check.unit.ts index 32f6718..f619767 100644 --- a/tests/verify/check.unit.ts +++ b/tests/verify/check.unit.ts @@ -134,6 +134,73 @@ test("a document that is not a v2 report is INVALID_REPORT_OR_EVIDENCE", async ( ); }); +test("single and gate checks reject the other report kind", async () => { + const single = await genuineSingle(passingOffExample); + const gate = await genuineGate(passingOffExample, passingOnExample); + + assert.equal( + (await checkSingle(gate, passingOffExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); + assert.equal( + (await checkGate(single, passingOffExample, passingOnExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); +}); + +test("strict report validation rejects missing, malformed, and unknown fields", async () => { + const genuine = clone(await genuineSingle(brokenOffExample)); + const cases: unknown[] = []; + + const missingAuthority = clone(genuine); + delete missingAuthority.authority; + cases.push(missingAuthority); + + const malformedClause = clone(genuine); + malformedClause.clauses[0].passed = "yes"; + cases.push(malformedClause); + + const malformedViolation = clone(genuine); + malformedViolation.violations[0].code = "PP_NOT_A_REAL_VIOLATION"; + cases.push(malformedViolation); + + const unknownField = clone(genuine); + unknownField.unexpected = true; + cases.push(unknownField); + + const wrongOutcome = clone(genuine); + wrongOutcome.outcome = "INCONCLUSIVE"; + cases.push(wrongOutcome); + + const invalidDigest = clone(genuine); + invalidDigest.inputBinding.sha256 = "not-a-sha256"; + cases.push(invalidDigest); + + const wrongVersion = clone(genuine); + wrongVersion.schemaVersion = "1"; + cases.push(wrongVersion); + + for (const report of cases) { + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); + } +}); + +test("valid-shaped semantic forgeries remain STALE_OR_MISMATCH", async () => { + const report = clone(await genuineSingle(brokenOffExample)); + report.outcome = "PASS"; + report.canonicalVerdict = "pass"; + report.authority.evaluatorSourceSha256 = "0".repeat(64); + report.authority.collectionAttested = true; + + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + test("invalid evidence is INVALID_REPORT_OR_EVIDENCE", async () => { const report = await genuineSingle(passingOffExample); assert.equal( diff --git a/tests/verify/cli.unit.ts b/tests/verify/cli.unit.ts index 784516f..384b8f7 100644 --- a/tests/verify/cli.unit.ts +++ b/tests/verify/cli.unit.ts @@ -438,3 +438,52 @@ test("CLI help succeeds and documents commands and exit codes", () => { assert.match(result.stdout, /gate --off/); assert.match(result.stdout, /3 INVALID_EVIDENCE/); }); + +test("CLI check distinguishes malformed reports from valid semantic mismatches", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "off.json"); + const on = path.join(root, "on.json"); + const output = path.join(root, "output"); + const reportPath = path.join(output, "report.json"); + await writeJson(off, passingOffExample); + await writeJson(on, passingOnExample); + assert.equal( + runCli(["gate", "--off", off, "--on", on, "--out", output]).status, + 0, + ); + + await writeJson(reportPath, { schemaVersion: "2" }); + const malformed = runCli([ + "check", + "--report", + reportPath, + "--off", + off, + "--on", + on, + ]); + assert.equal(malformed.status, 3); + assert.match(malformed.stdout, /INVALID_REPORT_OR_EVIDENCE/); + + const genuineOutput = path.join(root, "genuine"); + assert.equal( + runCli(["gate", "--off", off, "--on", on, "--out", genuineOutput]).status, + 0, + ); + const forged = JSON.parse( + await readFile(path.join(genuineOutput, "report.json"), "utf8"), + ); + forged.outcome = "BROKEN_PROMISE"; + await writeJson(reportPath, forged); + const mismatch = runCli([ + "check", + "--report", + reportPath, + "--off", + off, + "--on", + on, + ]); + assert.equal(mismatch.status, 4); + assert.match(mismatch.stdout, /STALE_OR_MISMATCH/); +}); diff --git a/tests/verify/judge-mode.spec.ts b/tests/verify/judge-mode.spec.ts index 4e06a78..16c12b9 100644 --- a/tests/verify/judge-mode.spec.ts +++ b/tests/verify/judge-mode.spec.ts @@ -1,16 +1,97 @@ -import { execFileSync } from "node:child_process"; -import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { request as httpRequest } from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; import { expect, test, type Page, type Request } from "@playwright/test"; import { EVALUATOR_SOURCE_SHA256 } from "../../src/verify/binding.js"; -import { passingOffExample, passingOnExample } from "../../src/verify/examples.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, +} from "../../src/verify/examples.js"; +import { MAX_INPUT_BYTES, MAX_COLLECTION_ITEMS } from "../../src/verify/schema.js"; const ROUTE = "/verify/"; const REPO = process.cwd(); +function clone(value: T): T { + return structuredClone(value); +} + +async function downloadBytes(page: Page, selector: string): Promise { + const [download] = await Promise.all([ + page.waitForEvent("download"), + page.locator(selector).click(), + ]); + const filename = await download.path(); + if (filename === null) throw new Error("browser download has no local path"); + return readFileSync(filename); +} + +function cliGateReports( + off: unknown, + on: unknown, + expectedExit: 0 | 2, +): { json: Buffer; markdown: Buffer } { + const kit = mkdtempSync(path.join(tmpdir(), "pp-parity-")); + try { + const offPath = path.join(kit, "off.json"); + const onPath = path.join(kit, "on.json"); + writeFileSync(offPath, `${JSON.stringify(off, null, 2)}\n`); + writeFileSync(onPath, `${JSON.stringify(on, null, 2)}\n`); + const result = spawnSync( + process.execPath, + [ + path.join(REPO, "node_modules", "tsx", "dist", "cli.mjs"), + path.join(REPO, "src", "verify", "cli.ts"), + "gate", + "--off", + offPath, + "--on", + onPath, + "--out", + kit, + ], + { cwd: REPO, encoding: "utf8", windowsHide: true }, + ); + if (result.error !== undefined) throw result.error; + if (result.status !== expectedExit) { + throw new Error( + `CLI gate exited ${String(result.status)} instead of ${expectedExit}: ${result.stderr}`, + ); + } + return { + json: readFileSync(path.join(kit, "report.json")), + markdown: readFileSync(path.join(kit, "report.md")), + }; + } finally { + rmSync(kit, { recursive: true, force: true }); + } +} + +function rawStatus(baseURL: string, pathname: string): Promise { + const url = new URL(baseURL); + return new Promise((resolve, reject) => { + const request = httpRequest( + { + hostname: url.hostname, + port: url.port, + method: "GET", + path: pathname, + }, + (response) => { + response.resume(); + response.on("end", () => resolve(response.statusCode ?? 0)); + }, + ); + request.on("error", reject); + request.end(); + }); +} + async function open(page: Page): Promise { await page.goto(ROUTE); await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); @@ -33,35 +114,86 @@ test("Judge Mode opens on PASS + BOUND with the pinned evaluator fingerprint", a await expect(page.getByTestId("violations")).toContainText("no violations"); }); -test("nothing leaves the page: no cross-origin, fetch, socket, or POST traffic", async ({ +test("nothing leaves the page across every Judge and BYO interaction", async ({ page, baseURL, }) => { + await open(page); const offenders: string[] = []; + const evidenceValues = [ + passingOffExample.evidence.subjectId, + passingOnExample.evidence.activity.capturedActivities[0]!.itemId, + ]; const record = (request: Request): void => { const url = request.url(); - const sameOrigin = baseURL !== undefined && url.startsWith(baseURL); + const body = request.postData() ?? ""; const local = url.startsWith("blob:") || url.startsWith("data:") || url.startsWith("about:"); - const chatty = ["fetch", "xhr", "websocket", "eventsource"].includes( + const networkApi = ["fetch", "xhr", "websocket", "eventsource", "ping"].includes( request.resourceType(), ); - if ((!sameOrigin && !local) || chatty || request.method() === "POST") { + const evidenceDerived = evidenceValues.some( + (value) => url.includes(value) || body.includes(value), + ); + if ( + !local && + (networkApi || + request.method() === "POST" || + request.isNavigationRequest() || + evidenceDerived) + ) { offenders.push(`${request.method()} ${request.resourceType()} ${url}`); } }; page.on("request", record); + page.on("websocket", (socket) => offenders.push(`WEBSOCKET ${socket.url()}`)); + page.on("framenavigated", (frame) => { + if (frame === page.mainFrame() && frame.url() !== `${baseURL}${ROUTE}`) { + offenders.push(`NAVIGATION ${frame.url()}`); + } + }); - await open(page); - // Exercise every interactive path that could plausibly phone home. await page.getByTestId("tamper").click(); await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); await page.getByTestId("evaluate").click(); await page.getByTestId("check").click(); - const [download] = await Promise.all([ - page.waitForEvent("download"), - page.locator("#vf-dl-json").click(), - ]); - await download.path(); + await page.getByTestId("seal").click(); + await page.getByTestId("reset").click(); + await downloadBytes(page, "#vf-dl-json"); + await downloadBytes(page, "#vf-dl-md"); + + await page.locator("#vf-file-off").setInputFiles({ + name: "passing-off.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOffExample)), + }); + await page.locator("#vf-file-on").setInputFiles({ + name: "passing-on.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOnExample)), + }); + await expect(page.locator("#vf-name-on")).toHaveText("passing-on.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("PASS"); + + await page.locator("#vf-byo-clear").click(); + await page.locator("#vf-file-off").setInputFiles({ + name: "passing-single.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOffExample)), + }); + await expect(page.locator("#vf-name-off")).toHaveText("passing-single.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("PASS"); + + await page.locator("#vf-byo-clear").click(); + await page.locator("#vf-file-off").setInputFiles({ + name: "invalid.json", + mimeType: "application/json", + buffer: Buffer.from("{not-json"), + }); + await expect(page.locator("#vf-name-off")).toHaveText("invalid.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); await page.waitForTimeout(200); expect(offenders, `unexpected egress:\n${offenders.join("\n")}`).toEqual([]); @@ -76,6 +208,11 @@ test("tampering the OFF evidence flips the same evaluator to a broken promise", await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); await expect(page.getByTestId("repro-pill")).toContainText("STALE_OR_MISMATCH"); await expect(page.getByTestId("violations")).toContainText("PP_IDENTIFIABLE_EVENT_LEAK"); + await expect(page.getByTestId("violations").locator("li")).toHaveCount(1); + await expect(page.locator("#vf-off-facts")).toContainText("Identifiable captured activity1"); + await expect(page.locator("#vf-off-facts")).toContainText("Recommendation-service receipts1"); + await expect(page.getByTestId("clauses").locator(".vf-tag.fail")).toHaveCount(1); + await expect(page.getByTestId("clauses").locator(".vf-tag.pass")).toHaveCount(4); // The failing clause is the identifiable-activity clause, and the OFF panel is flagged. const failing = page.getByTestId("clauses").locator("tr", { @@ -89,8 +226,12 @@ test("you can only seal the truth: re-sealing tampered evidence binds BROKEN, no page, }) => { await open(page); + const originalOn = await page.locator("#vf-on-facts").textContent(); + const originalDigests = await page.locator("#vf-digests").textContent(); + const originalJson = await downloadBytes(page, "#vf-dl-json"); await page.getByTestId("tamper").click(); await expect(page.getByTestId("repro-pill")).toContainText("STALE_OR_MISMATCH"); + await expect(page.locator("#vf-on-facts")).toHaveText(originalOn ?? ""); await page.getByTestId("seal").click(); await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); @@ -100,6 +241,9 @@ test("you can only seal the truth: re-sealing tampered evidence binds BROKEN, no await page.getByTestId("reset").click(); await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + await expect(page.locator("#vf-digests")).toHaveText(originalDigests ?? ""); + await expect(page.locator("#vf-on-facts")).toHaveText(originalOn ?? ""); + expect(await downloadBytes(page, "#vf-dl-json")).toEqual(originalJson); }); test("bring-your-own invalid evidence surfaces validation issues, not a verdict", async ({ @@ -117,28 +261,199 @@ test("bring-your-own invalid evidence surfaces validation issues, not a verdict" await expect(page.locator("#vf-byo-issues")).toBeVisible(); }); -test("parity: the report the browser downloads is byte-identical to the CLI's", async ({ +test("bring-your-own gate rejects swapped scenario slots atomically", async ({ page }) => { + await open(page); + await page.locator("#vf-file-off").setInputFiles({ + name: "wrong-off.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOnExample)), + }); + await page.locator("#vf-file-on").setInputFiles({ + name: "wrong-on.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOffExample)), + }); + await expect(page.locator("#vf-name-on")).toHaveText("wrong-on.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText( + "--off bundle must contain OFF-scenario evidence", + ); + await expect(page.locator("#vf-byo-issues")).toContainText( + "--on bundle must contain ON-scenario evidence", + ); + await expect(page.locator("#vf-byo-table")).toBeHidden(); + await expect(page.locator("#vf-byo-digests")).toBeEmpty(); +}); + +test("passing and sealed-broken JSON and Markdown are byte-identical to CLI output", async ({ page, }) => { await open(page); - const [download] = await Promise.all([ - page.waitForEvent("download"), - page.locator("#vf-dl-json").click(), - ]); - const browserReport = readFileSync(await download.path(), "utf8"); + const passingCli = cliGateReports(passingOffExample, passingOnExample, 0); + expect(await downloadBytes(page, "#vf-dl-json")).toEqual(passingCli.json); + expect(await downloadBytes(page, "#vf-dl-md")).toEqual(passingCli.markdown); - // Produce the CLI report for the same OFF/ON examples the page ships. - const kit = mkdtempSync(path.join(tmpdir(), "pp-parity-")); - const offPath = path.join(kit, "off.json"); - const onPath = path.join(kit, "on.json"); - writeFileSync(offPath, `${JSON.stringify(passingOffExample, null, 2)}\n`); - writeFileSync(onPath, `${JSON.stringify(passingOnExample, null, 2)}\n`); - execFileSync( - "npx", - ["tsx", "src/verify/cli.ts", "gate", "--off", offPath, "--on", onPath, "--out", kit], - { cwd: REPO, stdio: "pipe", shell: process.platform === "win32" }, + await page.getByTestId("tamper").click(); + await page.getByTestId("seal").click(); + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + const brokenCli = cliGateReports(brokenOffExample, passingOnExample, 2); + expect(await downloadBytes(page, "#vf-dl-json")).toEqual(brokenCli.json); + expect(await downloadBytes(page, "#vf-dl-md")).toEqual(brokenCli.markdown); +}); + +test("hostile local files render inertly and invalid input never inherits a verdict", async ({ + page, +}) => { + await open(page); + const hostileKey = ''; + const hostile = clone(passingOffExample) as any; + hostile.evidence.subjectId = "& ` ` | \\ visible text"; + + await page.locator("#vf-file-off").setInputFiles({ + name: `${hostileKey}.json`, + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(hostile)), + }); + await expect(page.locator("#vf-name-off")).toHaveText(`${hostileKey}.json`); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("PASS"); + await expect(page.locator("#pp-injected")).toHaveCount(0); + expect(await page.evaluate(() => (window as any).ppOwned)).toBeUndefined(); + + const unknown = clone(hostile); + unknown.evidence.control[hostileKey] = true; + await page.locator("#vf-file-off").setInputFiles({ + name: "unknown-key.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(unknown)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("Invalid input"); + await expect(page.locator("#pp-injected")).toHaveCount(0); + expect(await page.evaluate(() => (window as any).ppOwned)).toBeUndefined(); + + const bidi = clone(passingOffExample) as any; + bidi.evidence.subjectId = "safe\u202Eunsafe"; + await page.locator("#vf-file-off").setInputFiles({ + name: "bidi.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(bidi)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("forbidden control or bidi"); + + const long = clone(passingOffExample) as any; + long.evidence.subjectId = "x".repeat(257); + await page.locator("#vf-file-off").setInputFiles({ + name: "long.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(long)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + + const collection = clone(passingOffExample) as any; + collection.evidence.recommendations.renderedItemIds = Array.from( + { length: MAX_COLLECTION_ITEMS + 1 }, + (_, index) => `item-${index}`, + ); + await page.locator("#vf-file-off").setInputFiles({ + name: "collection.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(collection)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); +}); + +test("input and drag/drop reject oversized or malformed files without a product verdict", async ({ + page, +}) => { + await open(page); + await page.locator("#vf-file-off").setInputFiles({ + name: "oversized.json", + mimeType: "application/json", + buffer: Buffer.alloc(MAX_INPUT_BYTES + 1, 0x20), + }); + await expect(page.locator("#vf-name-off")).toContainText("rejected"); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("input exceeds"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("PASS"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("BROKEN_PROMISE"); + + await page.locator("#vf-byo-clear").click(); + await page.evaluate(() => { + const file = new File(["{not-json"], "drop-bad.json", { + type: "application/json", + }); + const transfer = new DataTransfer(); + transfer.items.add(file); + document.querySelector("#vf-drop-off")!.dispatchEvent( + new DragEvent("drop", { + bubbles: true, + cancelable: true, + dataTransfer: transfer, + }), + ); + }); + await expect(page.locator("#vf-name-off")).toHaveText("drop-bad.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("malformed JSON"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("PASS"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("BROKEN_PROMISE"); +}); + +test("assembled root, walkthrough, verifier routes, assets, refresh, and traversal boundary", async ({ + page, + request, + baseURL, +}) => { + if (baseURL === undefined) throw new Error("missing test baseURL"); + const root = await request.get("/"); + expect(root.status()).toBe(200); + expect(Buffer.from(await root.body())).toEqual(readFileSync(path.join(REPO, "landing", "index.html"))); + + for (const route of ["/walkthrough/", "/verify/"]) { + const response = await request.get(route); + expect(response.status()).toBe(200); + const html = await response.text(); + const subpath = route.slice(0, -1); + const assets = [...html.matchAll(new RegExp(`${subpath}/assets/[^"'?#]+`, "gu"))].map( + (match) => match[0], + ); + expect(assets.length).toBeGreaterThan(0); + for (const asset of new Set(assets)) { + expect((await request.get(asset)).status()).toBe(200); + } + } + + await page.goto("/verify/"); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); + await page.reload(); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + + expect(await rawStatus(baseURL, "/%2e%2e%2fpackage.json")).toBe(403); + expect([403, 404]).toContain( + await rawStatus(baseURL, "/%2e%2e%5cpackage.json"), ); - const cliReport = readFileSync(path.join(kit, "report.json"), "utf8"); +}); - expect(browserReport).toBe(cliReport); +test("JavaScript-disabled verifier gives an honest fallback", async ({ browser, baseURL }) => { + const context = await browser.newContext({ javaScriptEnabled: false }); + try { + const page = await context.newPage(); + await page.goto(`${baseURL}${ROUTE}`); + await expect(page.locator(".vf-fallback")).toBeVisible(); + await expect(page.locator(".vf-fallback")).toContainText( + "interactive and needs JavaScript", + ); + await expect(page.locator(".vf-fallback")).toContainText("GitHub repository"); + } finally { + await context.close(); + } }); From 6ee51b757ca403ed4fa6402f15a2abbce4e3aa8b Mon Sep 17 00:00:00 2001 From: AlexPaiva Date: Tue, 21 Jul 2026 01:18:55 +0100 Subject: [PATCH 7/7] docs: expose PromiseProof Verify Add a Try PromiseProof Verify section to the README that links the hosted Judge Mode and shows the init, gate, and check commands with their exit codes. Add ADOPTION.md, a developer guide covering the evidence envelope, the CLI, the report format, outcomes, binding semantics, trust boundaries, limits, and platforms. Add a compact bring-your-own-evidence section and a footer link on the landing page so the verifier is discoverable without changing the hero or walkthrough. --- ADOPTION.md | 230 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 40 ++++++++ landing/index.html | 13 +++ 3 files changed, 283 insertions(+) create mode 100644 ADOPTION.md diff --git a/ADOPTION.md b/ADOPTION.md new file mode 100644 index 0000000..09a87fd --- /dev/null +++ b/ADOPTION.md @@ -0,0 +1,230 @@ +# Adopting PromiseProof Verify + +PromiseProof Verify is the reproducible check at the end of the PromiseProof lifecycle. It takes externally supplied evidence for one narrow product promise, runs it through a single unchanged deterministic evaluator, and emits an evidence-bound report you can regenerate byte for byte. No model participates in the verdict. + +This guide covers the external evidence contract, the CLI, the report format, the outcomes and exit codes, the binding semantics, the trust boundaries, and the current limits. + +## Who this is for + +Product, QA, privacy, reliability, and platform engineers who already collect browser and backend evidence for a user-facing control and want a deterministic, re-runnable verdict on one specific promise: when activity-based personalization is off, nothing that identifies the user should reach the recommendation service, and the off preference should survive a reload across UI, storage, and backend. + +You bring the evidence. PromiseProof decides, deterministically, whether that evidence keeps the promise. + +## Supported contract family + +Exactly one: + +``` +activity-personalization/v1 +``` + +Any other `contractFamily` value is rejected as invalid evidence. There is no arbitrary-contract mode. + +## The evidence envelope + +Evidence is a JSON bundle with a fixed shape. A passing OFF bundle: + +```json +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "off", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "off", + "toggleChecked": false, + "storedPreference": "off", + "backendPreference": "off", + "reloadObserved": true + }, + "activity": { + "capturedActivities": [], + "recommendationServiceReceipts": [] + }, + "recommendations": { + "feedFunctional": true, + "renderedSource": "contextual", + "renderedItemIds": ["article-atlas-story-101"], + "recommendationServiceReceipts": [ + { "source": "contextual", "itemIds": ["article-atlas-story-101"] } + ] + } + } +} +``` + +An activity object (used in `capturedActivities` and the activity `recommendationServiceReceipts`) has this shape: + +```json +{ + "runId": "article-atlas-passing-on", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" +} +``` + +`eventType` is the literal `page_view`. `clientSequence` is a non-negative integer. The envelope carries only facts the evaluator reads; it has no verdict field and no free-form cause field. + +### OFF requirements + +An OFF bundle passes when all three clauses hold: + +- `no_identifiable_activity`: zero identifiable captured activity requests and zero identifiable receipts at the recommendation service. +- `contextual_feed_functional`: a functional feed with `renderedSource` `contextual`, a non-empty `renderedItemIds`, and a matching contextual recommendation-service receipt. +- `preference_survives_reload`: a witnessed reload (`reloadObserved` true) followed by off in the UI, toggle, stored preference, and backend. + +The single OFF violation surfaced by the seeded demo is `PP_IDENTIFIABLE_EVENT_LEAK`. + +### ON requirements + +An ON bundle is the control. It passes when: + +- `expected_activity_received`: one correlated identifiable activity for the subject reaches the recommendation service. +- `behavioral_feed_functional`: a functional feed with `renderedSource` `behavioral` and a matching behavioral receipt. + +The ON control proves a fix cannot simply switch recommendations off to satisfy the OFF promise. + +## The CLI + +Scaffold the contract, examples, and a producer template: + +```bash +npm run promiseproof -- init --out .promiseproof +``` + +Verify a single bundle: + +```bash +npm run promiseproof -- verify \ + --evidence .promiseproof/passing-off.example.json \ + --out .promiseproof/report +``` + +Gate an OFF and an ON bundle together (both must pass): + +```bash +npm run promiseproof -- gate \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json \ + --out .promiseproof/report +``` + +Reproduce a report from its evidence: + +```bash +npm run promiseproof -- check \ + --report .promiseproof/report/report.json \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json +``` + +`check` also accepts a single report with `--report ... --evidence ...`. + +## Reports + +`verify` and `gate` write two files to `--out`: + +- `report.json`: the canonical machine-readable report. It carries the contract family, the outcome, the per-clause results, any violations, the evidence input binding (canonicalization id, digest algorithm, and evidence SHA-256), and an `authority` block that records the evidence source, that collection is not attested, and the evaluator source SHA-256. +- `report.md`: the same content as human-readable Markdown. + +## Outcomes and exit codes + +Verdict outcomes from `verify` and `gate`: + +| Outcome | Meaning | Exit | +| --- | --- | --- | +| `PASS` | Every clause held. | `0` | +| `BROKEN_PROMISE` | At least one clause failed; violations are listed. | `2` | +| `INVALID_EVIDENCE` | The bundle failed strict validation and was not evaluated. | `3` | +| usage or execution error | Bad arguments or I/O failure. | `1` | + +Reproduction outcomes from `check`: + +| Outcome | Meaning | Exit | +| --- | --- | --- | +| `BOUND_AND_REPRODUCED` | The supplied report regenerates exactly from the supplied evidence. | `0` | +| `STALE_OR_MISMATCH` | The report is a well-formed report that does not reproduce from the evidence. | `4` | +| `INVALID_REPORT_OR_EVIDENCE` | The report is malformed or the wrong kind, or the evidence is invalid. | `3` | +| usage or execution error | Bad arguments or I/O failure. | `1` | + +The distinction is deliberate: a structurally valid report that lies about its verdict is `STALE_OR_MISMATCH`, not `INVALID`. A malformed or wrong-shape document is `INVALID_REPORT_OR_EVIDENCE`. + +## Binding semantics + +Two content digests connect a report to what produced it: + +- **Evidence digest.** The SHA-256 of the validated bundle serialized with the `promiseproof-canonical-json/v1` canonicalization. Change any load-bearing observation and this digest changes. +- **Evaluator fingerprint.** The SHA-256 of the evaluator source, normalized to LF line endings. It is pinned to the build and checked against the evaluator source by the repository's own tests. + +`check` does not trust these digests on their own. It re-runs the evaluator on the supplied evidence, rebuilds the complete report, and compares the whole canonical document. A forged report that carries the correct digest and fingerprint but an invented verdict, clause, or violation fails to reproduce. + +### Why BOUND does not mean PASS + +`BOUND_AND_REPRODUCED` is an integrity statement: the report faithfully matches the evidence it was computed from. The verdict inside can be `PASS` or `BROKEN_PROMISE`. A broken-promise report is still bound to its evidence. Binding is about whether the report is honest for its evidence, not about whether the promise was kept. + +### Honestly sealing a failing verdict + +Because the report is regenerated from evidence rather than asserted, you can seal a report for changed evidence and it will reproduce, carrying the honest `BROKEN_PROMISE` verdict. What you cannot do is carry an earlier `PASS` onto changed evidence: the old report goes `STALE_OR_MISMATCH`, and any newly sealed report reflects the actual, now-broken, verdict. + +## Collection is not attested + +PromiseProof evaluates evidence that you supply. It does not observe your system and does not claim to know how the evidence was collected or that it is a faithful record of a real session. The `authority` block states this explicitly. Binding proves the report matches the evidence, not that the evidence matches reality. + +## Limits + +Strict validation applies before evaluation: + +- Input is rejected before parsing if it exceeds 256 KiB (262144 bytes). +- Every collection is capped at 100 items. +- String identifiers are capped at 256 characters. +- Unknown fields and wrong single or gate report shapes are rejected. + +In the hosted Judge Mode these limits apply in the browser, before the file is read. + +## Supported platforms + +- Node.js 22.12 or newer and npm 10 or newer for the CLI. +- The CLI is directly verified on Windows 10 x64. It uses cross-platform Node and Web Crypto primitives; macOS and Linux are expected to work but are not yet claimed as verified. +- The hosted Judge Mode runs entirely in a modern browser with no server and no network request in the verdict path. + +## CI example + +The repository workflow runs the verifier suites on pull requests and pushes to `main`: + +```bash +npm run test:external # CLI, bound-report check, strict validation +npm run test:external:pipeline # real evidence-to-evaluation equivalence +npm run build:site # assembles /, /walkthrough/, /verify/ +npm run test:verify:judge # hosted Judge Mode, privacy, browser/CLI parity +``` + +You can run the same commands in your own CI to gate a change against the contract. + +## Trust boundaries + +Four distinct layers, kept separate on purpose: + +1. **Evidence you supply.** You are responsible for the bundle. PromiseProof validates its shape and rejects malformed input, but it does not vouch for the evidence. +2. **Deterministic evaluation.** One fixed evaluator, with no verdict field exposed to any model, decides `PASS` or `BROKEN_PROMISE` from that evidence. +3. **Content binding and complete reproduction.** The report binds the validated evidence by digest to the pinned evaluator source, and `check` regenerates the entire report so a forged verdict cannot survive. +4. **Producer identity and collection integrity.** PromiseProof does not attest who produced the evidence or how it was gathered. That boundary is outside the tool. + +## Current limitations + +- One contract family only: `activity-personalization/v1`. +- The bundled Signal Shelf example is synthetic. This is not a legal or regulatory compliance statement. +- The hosted Judge Mode supports single-bundle evaluation but exposes report download for gate reports; single-report byte parity with the CLI is therefore not part of the browser surface. +- Integrity is repository-level and content-addressed. It is not a signature, notarization, or third-party attestation. + +## Under consideration + +Not promised, and none of it blocks the current release: + +- A repository GitHub Action wrapping the CLI, to make the check easy to drop into another project's CI. +- Verification on macOS and Linux runners. + +PromiseProof does not promise package publication, arbitrary evidence collection, arbitrary promise contracts, or external provenance. diff --git a/README.md b/README.md index 8b33f20..a36e3ff 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,46 @@ npm run test:judge:interface # browser-level walkthrough and accessibili > On Windows systems that block PowerShell's `npm.ps1`, use `npm.cmd` / `npx.cmd`. +## Try PromiseProof Verify + +The lifecycle above repairs a promise without ever letting a model own the verdict. **PromiseProof Verify** exposes that same unchanged evaluator so anyone can re-derive the result from evidence, in the browser or from the CLI. + +**Hosted Judge Mode**, no login and no API key, runs locally in your browser: + +**[▶ Open the verifier](https://promiseproof.alex0paiva0.workers.dev/verify/?judge=1)** + +It opens on a passing OFF/ON gate bound to the pinned evaluator source. Tamper a load-bearing OFF observation and the same evaluator returns `BROKEN_PROMISE` (`PP_IDENTIFIABLE_EVENT_LEAK`), while the report you sealed a moment earlier no longer reproduces (`STALE_OR_MISMATCH`). The files you select stay in the page. + +**Repository-local CLI**, no model call in the verdict path: + +```bash +npm run promiseproof -- init --out .promiseproof + +npm run promiseproof -- gate \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json \ + --out .promiseproof/report + +npm run promiseproof -- check \ + --report .promiseproof/report/report.json \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json +``` + +- `verify` and `gate` exit `0` PASS, `2` BROKEN_PROMISE, `3` INVALID_EVIDENCE, `1` usage or execution error. +- `check` exits `0` BOUND_AND_REPRODUCED, `4` STALE_OR_MISMATCH, `3` INVALID_REPORT_OR_EVIDENCE, `1` usage or execution error. + +What it is, stated exactly: + +- One supported contract family: `activity-personalization/v1`. +- Externally supplied evidence is **not** collection-attested; PromiseProof does not claim to know how it was gathered. +- Reports bind the validated evidence to the pinned evaluator source by content digest. +- `check` re-runs the evaluator and compares the complete regenerated report, not only its hashes, so a report with correct digests but an invented verdict fails to reproduce. +- The verifier path makes no GPT-5.6 or Codex call. +- Signal Shelf still demonstrates the complete investigation and repair lifecycle above; the verifier is the reproducible check at the end of it. + +The full developer guide is in [ADOPTION.md](ADOPTION.md). + ## The two seeded defects Signal Shelf carries two independently selectable defects. Both break the same OFF promise in different ways, produce different evidence, and require different diagnostic actions, so a single flag cannot explain both, and one fix cannot silence the other. diff --git a/landing/index.html b/landing/index.html index 516d484..17dc43a 100644 --- a/landing/index.html +++ b/landing/index.html @@ -460,6 +460,18 @@

    The things a skeptic asks first.

    +
    +
    + Bring your own evidence +

    The proof runs in your browser.

    +

    Run one narrow activity-personalization contract in the browser or from the CLI. PromiseProof recomputes the complete report through the same deterministic evaluator. Tamper a load-bearing observation and the sealed result becomes stale.

    + +
    +
    +
    Explore it yourself @@ -491,6 +503,7 @@

    The verdict isn't ours to give. That's the point.

    Project Interactive walkthrough + In-browser verifier GitHub repository Questions