diff --git a/.github/workflows/submission-hardening.yml b/.github/workflows/submission-hardening.yml index 2623227..0541546 100644 --- a/.github/workflows/submission-hardening.yml +++ b/.github/workflows/submission-hardening.yml @@ -39,6 +39,7 @@ jobs: - run: npx playwright install chromium - run: npm run build - run: npm run evidence:verify + - run: npm run test:external - run: npm run test:evidence:unit - run: npm run test:investigation:unit - run: npm run test:investigation:offline @@ -72,6 +73,7 @@ jobs: - run: npm ci - run: npx playwright install chromium - run: npm test + - run: npm run test:external:pipeline - run: npm run test:expected-red repair-rehearsal: @@ -138,6 +140,7 @@ jobs: - run: npm ci - run: npx playwright install chromium - run: npm run build:site + - run: npm run test:verify:judge - run: npm run test:judge:unit - run: npm run test:judge:interface - run: npm run demo:rehearse diff --git a/ADOPTION.md b/ADOPTION.md new file mode 100644 index 0000000..09a87fd --- /dev/null +++ b/ADOPTION.md @@ -0,0 +1,230 @@ +# Adopting PromiseProof Verify + +PromiseProof Verify is the reproducible check at the end of the PromiseProof lifecycle. It takes externally supplied evidence for one narrow product promise, runs it through a single unchanged deterministic evaluator, and emits an evidence-bound report you can regenerate byte for byte. No model participates in the verdict. + +This guide covers the external evidence contract, the CLI, the report format, the outcomes and exit codes, the binding semantics, the trust boundaries, and the current limits. + +## Who this is for + +Product, QA, privacy, reliability, and platform engineers who already collect browser and backend evidence for a user-facing control and want a deterministic, re-runnable verdict on one specific promise: when activity-based personalization is off, nothing that identifies the user should reach the recommendation service, and the off preference should survive a reload across UI, storage, and backend. + +You bring the evidence. PromiseProof decides, deterministically, whether that evidence keeps the promise. + +## Supported contract family + +Exactly one: + +``` +activity-personalization/v1 +``` + +Any other `contractFamily` value is rejected as invalid evidence. There is no arbitrary-contract mode. + +## The evidence envelope + +Evidence is a JSON bundle with a fixed shape. A passing OFF bundle: + +```json +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "off", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "off", + "toggleChecked": false, + "storedPreference": "off", + "backendPreference": "off", + "reloadObserved": true + }, + "activity": { + "capturedActivities": [], + "recommendationServiceReceipts": [] + }, + "recommendations": { + "feedFunctional": true, + "renderedSource": "contextual", + "renderedItemIds": ["article-atlas-story-101"], + "recommendationServiceReceipts": [ + { "source": "contextual", "itemIds": ["article-atlas-story-101"] } + ] + } + } +} +``` + +An activity object (used in `capturedActivities` and the activity `recommendationServiceReceipts`) has this shape: + +```json +{ + "runId": "article-atlas-passing-on", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" +} +``` + +`eventType` is the literal `page_view`. `clientSequence` is a non-negative integer. The envelope carries only facts the evaluator reads; it has no verdict field and no free-form cause field. + +### OFF requirements + +An OFF bundle passes when all three clauses hold: + +- `no_identifiable_activity`: zero identifiable captured activity requests and zero identifiable receipts at the recommendation service. +- `contextual_feed_functional`: a functional feed with `renderedSource` `contextual`, a non-empty `renderedItemIds`, and a matching contextual recommendation-service receipt. +- `preference_survives_reload`: a witnessed reload (`reloadObserved` true) followed by off in the UI, toggle, stored preference, and backend. + +The single OFF violation surfaced by the seeded demo is `PP_IDENTIFIABLE_EVENT_LEAK`. + +### ON requirements + +An ON bundle is the control. It passes when: + +- `expected_activity_received`: one correlated identifiable activity for the subject reaches the recommendation service. +- `behavioral_feed_functional`: a functional feed with `renderedSource` `behavioral` and a matching behavioral receipt. + +The ON control proves a fix cannot simply switch recommendations off to satisfy the OFF promise. + +## The CLI + +Scaffold the contract, examples, and a producer template: + +```bash +npm run promiseproof -- init --out .promiseproof +``` + +Verify a single bundle: + +```bash +npm run promiseproof -- verify \ + --evidence .promiseproof/passing-off.example.json \ + --out .promiseproof/report +``` + +Gate an OFF and an ON bundle together (both must pass): + +```bash +npm run promiseproof -- gate \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json \ + --out .promiseproof/report +``` + +Reproduce a report from its evidence: + +```bash +npm run promiseproof -- check \ + --report .promiseproof/report/report.json \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json +``` + +`check` also accepts a single report with `--report ... --evidence ...`. + +## Reports + +`verify` and `gate` write two files to `--out`: + +- `report.json`: the canonical machine-readable report. It carries the contract family, the outcome, the per-clause results, any violations, the evidence input binding (canonicalization id, digest algorithm, and evidence SHA-256), and an `authority` block that records the evidence source, that collection is not attested, and the evaluator source SHA-256. +- `report.md`: the same content as human-readable Markdown. + +## Outcomes and exit codes + +Verdict outcomes from `verify` and `gate`: + +| Outcome | Meaning | Exit | +| --- | --- | --- | +| `PASS` | Every clause held. | `0` | +| `BROKEN_PROMISE` | At least one clause failed; violations are listed. | `2` | +| `INVALID_EVIDENCE` | The bundle failed strict validation and was not evaluated. | `3` | +| usage or execution error | Bad arguments or I/O failure. | `1` | + +Reproduction outcomes from `check`: + +| Outcome | Meaning | Exit | +| --- | --- | --- | +| `BOUND_AND_REPRODUCED` | The supplied report regenerates exactly from the supplied evidence. | `0` | +| `STALE_OR_MISMATCH` | The report is a well-formed report that does not reproduce from the evidence. | `4` | +| `INVALID_REPORT_OR_EVIDENCE` | The report is malformed or the wrong kind, or the evidence is invalid. | `3` | +| usage or execution error | Bad arguments or I/O failure. | `1` | + +The distinction is deliberate: a structurally valid report that lies about its verdict is `STALE_OR_MISMATCH`, not `INVALID`. A malformed or wrong-shape document is `INVALID_REPORT_OR_EVIDENCE`. + +## Binding semantics + +Two content digests connect a report to what produced it: + +- **Evidence digest.** The SHA-256 of the validated bundle serialized with the `promiseproof-canonical-json/v1` canonicalization. Change any load-bearing observation and this digest changes. +- **Evaluator fingerprint.** The SHA-256 of the evaluator source, normalized to LF line endings. It is pinned to the build and checked against the evaluator source by the repository's own tests. + +`check` does not trust these digests on their own. It re-runs the evaluator on the supplied evidence, rebuilds the complete report, and compares the whole canonical document. A forged report that carries the correct digest and fingerprint but an invented verdict, clause, or violation fails to reproduce. + +### Why BOUND does not mean PASS + +`BOUND_AND_REPRODUCED` is an integrity statement: the report faithfully matches the evidence it was computed from. The verdict inside can be `PASS` or `BROKEN_PROMISE`. A broken-promise report is still bound to its evidence. Binding is about whether the report is honest for its evidence, not about whether the promise was kept. + +### Honestly sealing a failing verdict + +Because the report is regenerated from evidence rather than asserted, you can seal a report for changed evidence and it will reproduce, carrying the honest `BROKEN_PROMISE` verdict. What you cannot do is carry an earlier `PASS` onto changed evidence: the old report goes `STALE_OR_MISMATCH`, and any newly sealed report reflects the actual, now-broken, verdict. + +## Collection is not attested + +PromiseProof evaluates evidence that you supply. It does not observe your system and does not claim to know how the evidence was collected or that it is a faithful record of a real session. The `authority` block states this explicitly. Binding proves the report matches the evidence, not that the evidence matches reality. + +## Limits + +Strict validation applies before evaluation: + +- Input is rejected before parsing if it exceeds 256 KiB (262144 bytes). +- Every collection is capped at 100 items. +- String identifiers are capped at 256 characters. +- Unknown fields and wrong single or gate report shapes are rejected. + +In the hosted Judge Mode these limits apply in the browser, before the file is read. + +## Supported platforms + +- Node.js 22.12 or newer and npm 10 or newer for the CLI. +- The CLI is directly verified on Windows 10 x64. It uses cross-platform Node and Web Crypto primitives; macOS and Linux are expected to work but are not yet claimed as verified. +- The hosted Judge Mode runs entirely in a modern browser with no server and no network request in the verdict path. + +## CI example + +The repository workflow runs the verifier suites on pull requests and pushes to `main`: + +```bash +npm run test:external # CLI, bound-report check, strict validation +npm run test:external:pipeline # real evidence-to-evaluation equivalence +npm run build:site # assembles /, /walkthrough/, /verify/ +npm run test:verify:judge # hosted Judge Mode, privacy, browser/CLI parity +``` + +You can run the same commands in your own CI to gate a change against the contract. + +## Trust boundaries + +Four distinct layers, kept separate on purpose: + +1. **Evidence you supply.** You are responsible for the bundle. PromiseProof validates its shape and rejects malformed input, but it does not vouch for the evidence. +2. **Deterministic evaluation.** One fixed evaluator, with no verdict field exposed to any model, decides `PASS` or `BROKEN_PROMISE` from that evidence. +3. **Content binding and complete reproduction.** The report binds the validated evidence by digest to the pinned evaluator source, and `check` regenerates the entire report so a forged verdict cannot survive. +4. **Producer identity and collection integrity.** PromiseProof does not attest who produced the evidence or how it was gathered. That boundary is outside the tool. + +## Current limitations + +- One contract family only: `activity-personalization/v1`. +- The bundled Signal Shelf example is synthetic. This is not a legal or regulatory compliance statement. +- The hosted Judge Mode supports single-bundle evaluation but exposes report download for gate reports; single-report byte parity with the CLI is therefore not part of the browser surface. +- Integrity is repository-level and content-addressed. It is not a signature, notarization, or third-party attestation. + +## Under consideration + +Not promised, and none of it blocks the current release: + +- A repository GitHub Action wrapping the CLI, to make the check easy to drop into another project's CI. +- Verification on macOS and Linux runners. + +PromiseProof does not promise package publication, arbitrary evidence collection, arbitrary promise contracts, or external provenance. diff --git a/README.md b/README.md index 8b33f20..a36e3ff 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,46 @@ npm run test:judge:interface # browser-level walkthrough and accessibili > On Windows systems that block PowerShell's `npm.ps1`, use `npm.cmd` / `npx.cmd`. +## Try PromiseProof Verify + +The lifecycle above repairs a promise without ever letting a model own the verdict. **PromiseProof Verify** exposes that same unchanged evaluator so anyone can re-derive the result from evidence, in the browser or from the CLI. + +**Hosted Judge Mode**, no login and no API key, runs locally in your browser: + +**[▶ Open the verifier](https://promiseproof.alex0paiva0.workers.dev/verify/?judge=1)** + +It opens on a passing OFF/ON gate bound to the pinned evaluator source. Tamper a load-bearing OFF observation and the same evaluator returns `BROKEN_PROMISE` (`PP_IDENTIFIABLE_EVENT_LEAK`), while the report you sealed a moment earlier no longer reproduces (`STALE_OR_MISMATCH`). The files you select stay in the page. + +**Repository-local CLI**, no model call in the verdict path: + +```bash +npm run promiseproof -- init --out .promiseproof + +npm run promiseproof -- gate \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json \ + --out .promiseproof/report + +npm run promiseproof -- check \ + --report .promiseproof/report/report.json \ + --off .promiseproof/passing-off.example.json \ + --on .promiseproof/passing-on.example.json +``` + +- `verify` and `gate` exit `0` PASS, `2` BROKEN_PROMISE, `3` INVALID_EVIDENCE, `1` usage or execution error. +- `check` exits `0` BOUND_AND_REPRODUCED, `4` STALE_OR_MISMATCH, `3` INVALID_REPORT_OR_EVIDENCE, `1` usage or execution error. + +What it is, stated exactly: + +- One supported contract family: `activity-personalization/v1`. +- Externally supplied evidence is **not** collection-attested; PromiseProof does not claim to know how it was gathered. +- Reports bind the validated evidence to the pinned evaluator source by content digest. +- `check` re-runs the evaluator and compares the complete regenerated report, not only its hashes, so a report with correct digests but an invented verdict fails to reproduce. +- The verifier path makes no GPT-5.6 or Codex call. +- Signal Shelf still demonstrates the complete investigation and repair lifecycle above; the verifier is the reproducible check at the end of it. + +The full developer guide is in [ADOPTION.md](ADOPTION.md). + ## The two seeded defects Signal Shelf carries two independently selectable defects. Both break the same OFF promise in different ways, produce different evidence, and require different diagnostic actions, so a single flag cannot explain both, and one fix cannot silence the other. diff --git a/artifacts/verify/README.md b/artifacts/verify/README.md new file mode 100644 index 0000000..01b74cb --- /dev/null +++ b/artifacts/verify/README.md @@ -0,0 +1,40 @@ +# PromiseProof external evidence scaffold + +This scaffold supports exactly one contract family: +`activity-personalization/v1`. + +The public `ExternalEvidenceV1` contract contains only facts used by the +unchanged PromiseProof evaluator: + +- scenario and subject identifier; +- UI, toggle, storage, backend, and reload control state; +- captured activities and recommendation-service activity receipts; +- rendered feed state and recommendation-service recommendation receipts. + +Use `broken-off.example.json`, `passing-off.example.json`, and +`passing-on.example.json` as fixed-shape examples. `producer-template.mjs` +wraps an `ExternalEvidenceV1` JSON object; it does not collect or attest +evidence. Then verify one bundle: + +```text +npm run promiseproof -- verify --evidence --out +``` + +Or gate OFF and ON together: + +```text +npm run promiseproof -- gate --off --on --out +``` + +The OFF contract checks zero identifiable recommendation activity, a functional +contextual feed, and witnessed OFF persistence across UI, toggle, storage, and +backend state. The ON control checks correlated identifiable activity and a +functional behavioral feed. + +Evidence is externally supplied. PromiseProof does not attest how it was +collected. The deterministic PromiseProof evaluator alone evaluates a bundle +that passes strict validation. + +These examples prove externally supplied evidence ingestion for one contract +family. They do not prove arbitrary evidence collection or arbitrary promise +support. Article Atlas is a neutral synthetic example product. diff --git a/artifacts/verify/activity-personalization.v1.schema.json b/artifacts/verify/activity-personalization.v1.schema.json new file mode 100644 index 0000000..4df6cfa --- /dev/null +++ b/artifacts/verify/activity-personalization.v1.schema.json @@ -0,0 +1,344 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "schemaVersion": { + "type": "string", + "const": "1" + }, + "contractFamily": { + "type": "string", + "const": "activity-personalization/v1" + }, + "evidence": { + "type": "object", + "properties": { + "scenario": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "subjectId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "control": { + "type": "object", + "properties": { + "uiPreference": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "toggleChecked": { + "type": "boolean" + }, + "storedPreference": { + "anyOf": [ + { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + { + "type": "null" + } + ] + }, + "backendPreference": { + "type": "string", + "enum": [ + "on", + "off" + ] + }, + "reloadObserved": { + "type": "boolean" + } + }, + "required": [ + "uiPreference", + "toggleChecked", + "storedPreference", + "backendPreference", + "reloadObserved" + ], + "additionalProperties": false + }, + "activity": { + "type": "object", + "properties": { + "capturedActivities": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "subjectId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "eventType": { + "type": "string", + "const": "page_view" + }, + "itemId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "clientSequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "runId", + "subjectId", + "eventType", + "itemId", + "clientSequence", + "occurredAt" + ], + "additionalProperties": false + } + }, + "recommendationServiceReceipts": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "subjectId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "eventType": { + "type": "string", + "const": "page_view" + }, + "itemId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "clientSequence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + } + }, + "required": [ + "runId", + "subjectId", + "eventType", + "itemId", + "clientSequence", + "occurredAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "capturedActivities", + "recommendationServiceReceipts" + ], + "additionalProperties": false + }, + "recommendations": { + "type": "object", + "properties": { + "feedFunctional": { + "type": "boolean" + }, + "renderedSource": { + "type": "string", + "enum": [ + "contextual", + "behavioral" + ] + }, + "renderedItemIds": { + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + } + }, + "recommendationServiceReceipts": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "contextual", + "behavioral" + ] + }, + "subjectId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + }, + "itemIds": { + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "allOf": [ + { + "pattern": "\\S" + }, + { + "pattern": "^[^\\u0000-\\u001f\\u007f\\u202a-\\u202e\\u2066-\\u2069]*$" + } + ] + } + } + }, + "required": [ + "source", + "itemIds" + ], + "additionalProperties": false + } + } + }, + "required": [ + "feedFunctional", + "renderedSource", + "renderedItemIds", + "recommendationServiceReceipts" + ], + "additionalProperties": false + } + }, + "required": [ + "scenario", + "subjectId", + "control", + "activity", + "recommendations" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "contractFamily", + "evidence" + ], + "additionalProperties": false, + "title": "PromiseProof activity-personalization/v1 evidence bundle", + "description": "Externally supplied evaluator-relevant facts for the single activity-personalization/v1 contract family." +} diff --git a/artifacts/verify/broken-off.example.json b/artifacts/verify/broken-off.example.json new file mode 100644 index 0000000..2cae644 --- /dev/null +++ b/artifacts/verify/broken-off.example.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "off", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "off", + "toggleChecked": false, + "storedPreference": "off", + "backendPreference": "off", + "reloadObserved": true + }, + "activity": { + "capturedActivities": [ + { + "runId": "article-atlas-broken-off", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + ], + "recommendationServiceReceipts": [ + { + "runId": "article-atlas-broken-off", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + ] + }, + "recommendations": { + "feedFunctional": true, + "renderedSource": "contextual", + "renderedItemIds": [ + "article-atlas-story-101" + ], + "recommendationServiceReceipts": [ + { + "source": "contextual", + "itemIds": [ + "article-atlas-story-101" + ] + } + ] + } + } +} diff --git a/artifacts/verify/passing-off.example.json b/artifacts/verify/passing-off.example.json new file mode 100644 index 0000000..7f9ef81 --- /dev/null +++ b/artifacts/verify/passing-off.example.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "off", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "off", + "toggleChecked": false, + "storedPreference": "off", + "backendPreference": "off", + "reloadObserved": true + }, + "activity": { + "capturedActivities": [], + "recommendationServiceReceipts": [] + }, + "recommendations": { + "feedFunctional": true, + "renderedSource": "contextual", + "renderedItemIds": [ + "article-atlas-story-101" + ], + "recommendationServiceReceipts": [ + { + "source": "contextual", + "itemIds": [ + "article-atlas-story-101" + ] + } + ] + } + } +} diff --git a/artifacts/verify/passing-on.example.json b/artifacts/verify/passing-on.example.json new file mode 100644 index 0000000..ce54a9e --- /dev/null +++ b/artifacts/verify/passing-on.example.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": "1", + "contractFamily": "activity-personalization/v1", + "evidence": { + "scenario": "on", + "subjectId": "article-atlas-reader-001", + "control": { + "uiPreference": "on", + "toggleChecked": true, + "storedPreference": "on", + "backendPreference": "on", + "reloadObserved": true + }, + "activity": { + "capturedActivities": [ + { + "runId": "article-atlas-passing-on", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + ], + "recommendationServiceReceipts": [ + { + "runId": "article-atlas-passing-on", + "subjectId": "article-atlas-reader-001", + "eventType": "page_view", + "itemId": "article-atlas-origin-001", + "clientSequence": 1, + "occurredAt": "2026-01-15T12:00:01.000Z" + } + ] + }, + "recommendations": { + "feedFunctional": true, + "renderedSource": "behavioral", + "renderedItemIds": [ + "article-atlas-story-101" + ], + "recommendationServiceReceipts": [ + { + "source": "behavioral", + "subjectId": "article-atlas-reader-001", + "itemIds": [ + "article-atlas-story-101" + ] + } + ] + } + } +} diff --git a/artifacts/verify/producer-template.mjs b/artifacts/verify/producer-template.mjs new file mode 100644 index 0000000..ead64c3 --- /dev/null +++ b/artifacts/verify/producer-template.mjs @@ -0,0 +1,16 @@ +import { readFile, writeFile } from "node:fs/promises"; + +const [evidenceFile = "external-evidence.json", outputFile = "bundle.json"] = + process.argv.slice(2); +const evidence = JSON.parse(await readFile(evidenceFile, "utf8")); +const bundle = { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence, +}; + +await writeFile( + outputFile, + `${JSON.stringify(bundle, null, 2)}\n`, + { encoding: "utf8", flag: "wx" }, +); diff --git a/landing/index.html b/landing/index.html index 516d484..17dc43a 100644 --- a/landing/index.html +++ b/landing/index.html @@ -460,6 +460,18 @@

The things a skeptic asks first.

+
+
+ Bring your own evidence +

The proof runs in your browser.

+

Run one narrow activity-personalization contract in the browser or from the CLI. PromiseProof recomputes the complete report through the same deterministic evaluator. Tamper a load-bearing observation and the sealed result becomes stale.

+ +
+
+
Explore it yourself @@ -491,6 +503,7 @@

The verdict isn't ours to give. That's the point.

diff --git a/package.json b/package.json index f08fb1f..054604b 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,10 @@ "start:race": "cross-env NODE_ENV=production DEMO_MODE=initialization-race node dist/server.mjs", "start:propagation": "cross-env NODE_ENV=production DEMO_MODE=propagation-failure node dist/server.mjs", "typecheck": "tsc --noEmit", + "promiseproof": "tsx src/verify/cli.ts", + "test:external": "tsx --test tests/verify/core.unit.ts tests/verify/cli.unit.ts tests/verify/check.unit.ts", + "test:external:pipeline": "cross-env PROMISEPROOF_EQUIVALENCE_MODE=initialization-race playwright test --config=tests/verify/pipeline.playwright.config.ts && cross-env PROMISEPROOF_EQUIVALENCE_MODE=propagation-failure playwright test --config=tests/verify/pipeline.playwright.config.ts", + "test:verify:judge": "playwright test --config=tests/verify/judge-mode.playwright.config.ts", "evidence:verify": "tsx scripts/evidence-verify.ts", "demo:rehearse": "tsx scripts/demo-rehearse.ts", "test:judge:unit": "tsx --test tests/judge/rehearsal.unit.ts", diff --git a/scripts/build-site.mjs b/scripts/build-site.mjs index fdbd23d..168f753 100644 --- a/scripts/build-site.mjs +++ b/scripts/build-site.mjs @@ -1,6 +1,7 @@ // Assembles the combined public site into dist/site: // / -> landing/index.html (the front door) // /walkthrough/ -> the judge walkthrough, built with a /walkthrough/ base +// /verify/ -> the hosted deterministic verifier, built with a /verify/ base // // Deploy the result with `npx wrangler deploy` (see wrangler.toml). // @@ -17,39 +18,59 @@ import { const SITE = "dist/site"; const WT = "dist/_walkthrough"; +const VF = "dist/_verify"; -// 1) Build the walkthrough with the /walkthrough/ base so its asset URLs resolve -// under the sub-path. The env var is read by vite.config.ts. +// 1) Build each sub-path page with the base its sub-path needs, so asset URLs +// resolve under the sub-path. The walkthrough reads PP_WALKTHROUGH_BASE from +// vite.config.ts; the verifier passes --base, which overrides the default "/". console.log("• building walkthrough (base=/walkthrough/) ..."); execSync(`npx vite build --outDir ${WT} --emptyOutDir`, { stdio: "inherit", env: { ...process.env, PP_WALKTHROUGH_BASE: "/walkthrough/" }, }); +console.log("• building verifier (base=/verify/) ..."); +execSync(`npx vite build --base=/verify/ --outDir ${VF} --emptyOutDir`, { + stdio: "inherit", +}); + // 2) Fresh output tree. rmSync(SITE, { recursive: true, force: true }); mkdirSync(`${SITE}/walkthrough/assets`, { recursive: true }); +mkdirSync(`${SITE}/verify/assets`, { recursive: true }); -// 3) Landing at the root (already carries relative /walkthrough/ CTAs). +// 3) Landing at the root (already carries relative /walkthrough/ and /verify/ CTAs). copyFileSync("landing/index.html", `${SITE}/index.html`); -// 4) Shared static assets. Root serves the landing's; the walkthrough HTML -// references /walkthrough/favicon.* (Vite rewrote those under the base). +// 4) Shared static assets. Root serves the landing's; the sub-pages reference +// /walkthrough/favicon.* and /verify/favicon.* (Vite rewrote those under each base). const shared = ["favicon.svg", "favicon.ico", "apple-touch-icon.png", "og-card.png"]; for (const f of shared) copyFileSync(`public/${f}`, `${SITE}/${f}`); -for (const f of ["favicon.svg", "favicon.ico", "apple-touch-icon.png"]) { - copyFileSync(`public/${f}`, `${SITE}/walkthrough/${f}`); +for (const sub of ["walkthrough", "verify"]) { + for (const f of ["favicon.svg", "favicon.ico", "apple-touch-icon.png"]) { + copyFileSync(`public/${f}`, `${SITE}/${sub}/${f}`); + } } -// 5) Walkthrough page + only the assets it actually references. -copyFileSync(`${WT}/judge.html`, `${SITE}/walkthrough/index.html`); -const wtHtml = readFileSync(`${SITE}/walkthrough/index.html`, "utf8"); -const assets = [...wtHtml.matchAll(/\/walkthrough\/assets\/([^"?#]+)/g)].map((m) => m[1]); -for (const a of new Set(assets)) { - copyFileSync(`${WT}/assets/${a}`, `${SITE}/walkthrough/assets/${a}`); +// 5) Each sub-page + only the assets it actually references. +function assemble(intermediateDir, htmlName, sub) { + copyFileSync(`${intermediateDir}/${htmlName}`, `${SITE}/${sub}/index.html`); + const html = readFileSync(`${SITE}/${sub}/index.html`, "utf8"); + const pattern = new RegExp(`/${sub}/assets/([^"?#]+)`, "g"); + const assets = [...html.matchAll(pattern)].map((m) => m[1]); + for (const a of new Set(assets)) { + copyFileSync(`${intermediateDir}/assets/${a}`, `${SITE}/${sub}/assets/${a}`); + } + return new Set(assets).size; } -// 6) Drop the intermediate build. +const wtCount = assemble(WT, "judge.html", "walkthrough"); +const vfCount = assemble(VF, "verify.html", "verify"); + +// 6) Drop the intermediate builds. rmSync(WT, { recursive: true, force: true }); +rmSync(VF, { recursive: true, force: true }); -console.log(`✓ built ${SITE} (landing + /walkthrough/, ${assets.length} walkthrough assets)`); +console.log( + `✓ built ${SITE} (landing + /walkthrough/ [${wtCount} assets] + /verify/ [${vfCount} assets])`, +); diff --git a/scripts/serve-site.mjs b/scripts/serve-site.mjs new file mode 100644 index 0000000..951da7a --- /dev/null +++ b/scripts/serve-site.mjs @@ -0,0 +1,47 @@ +// Minimal static file server for the assembled site (dist/site). Used only by +// the Judge Mode Playwright harness; the real site is served by Cloudflare. +// +// node scripts/serve-site.mjs [rootDir] [port] + +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { extname, join, normalize, resolve, sep } from "node:path"; + +const ROOT = resolve(process.argv[2] ?? "dist/site"); +const PORT = Number(process.argv[3] ?? 4188); + +const TYPES = { + ".html": "text/html; charset=utf-8", + ".js": "text/javascript; charset=utf-8", + ".mjs": "text/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json; charset=utf-8", + ".svg": "image/svg+xml", + ".ico": "image/x-icon", + ".png": "image/png", + ".map": "application/json", +}; + +const server = createServer(async (req, res) => { + try { + let pathname = decodeURIComponent((req.url ?? "/").split("?")[0]); + if (pathname.endsWith("/")) pathname += "index.html"; + const full = normalize(join(ROOT, pathname)); + if (full !== ROOT && !full.startsWith(ROOT + sep)) { + res.writeHead(403).end("forbidden"); + return; + } + const body = await readFile(full); + res.writeHead(200, { + "content-type": TYPES[extname(full)] ?? "application/octet-stream", + "cache-control": "no-store", + }); + res.end(body); + } catch { + res.writeHead(404).end("not found"); + } +}); + +server.listen(PORT, "127.0.0.1", () => { + console.log(`serving ${ROOT} on http://127.0.0.1:${PORT}`); +}); diff --git a/src/verify-ui/main.ts b/src/verify-ui/main.ts new file mode 100644 index 0000000..f55dbe7 --- /dev/null +++ b/src/verify-ui/main.ts @@ -0,0 +1,739 @@ +import "./styles.css"; + +import { + CANONICAL_JSON_ID, + EVALUATOR_SOURCE_SHA256, + SHA256_ALGORITHM, +} from "../verify/binding.js"; +import { checkGate } from "../verify/check.js"; +import { passingOffExample, passingOnExample } from "../verify/examples.js"; +import { REPORT_SCHEMA_VERSION } from "../verify/outcome.js"; +import { + createGateReport, + createVerifyReport, + serializeGateReportMarkdown, + serializeReportJson, + type GateReport, +} from "../verify/report.js"; +import { + MAX_INPUT_BYTES, + type ExternalActivity, +} from "../verify/schema.js"; +import { runGate, verifyBundle, type VerifyResult } from "../verify/verify.js"; + +// --------------------------------------------------------------------------- +// Working evidence. runGate/verifyBundle/checkGate take `unknown`, so the page +// keeps mutable working copies of the shipped examples and never re-implements +// any evaluator rule. Tampering the OFF bundle just adds the one identifiable +// activity + receipt that the shipped broken-off example carries, so the +// tampered digest matches what the CLI would compute for the same evidence. +// --------------------------------------------------------------------------- +interface WorkEvidence { + scenario: "off" | "on"; + subjectId: string; + control: { + uiPreference: string; + toggleChecked: boolean; + storedPreference: string; + backendPreference: string; + reloadObserved: boolean; + }; + activity: { + capturedActivities: ExternalActivity[]; + recommendationServiceReceipts: ExternalActivity[]; + }; + recommendations: { + feedFunctional: boolean; + renderedSource: string; + renderedItemIds: string[]; + recommendationServiceReceipts: unknown[]; + }; +} +interface WorkBundle { + schemaVersion: string; + contractFamily: string; + evidence: WorkEvidence; +} + +const LEAK_ACTIVITY: ExternalActivity = { + runId: "article-atlas-broken-off", + subjectId: "article-atlas-reader-001", + eventType: "page_view", + itemId: "article-atlas-origin-001", + clientSequence: 1, + occurredAt: "2026-01-15T12:00:01.000Z", +}; + +function freshOff(): WorkBundle { + return structuredClone(passingOffExample as unknown as WorkBundle); +} +function freshOn(): WorkBundle { + return structuredClone(passingOnExample as unknown as WorkBundle); +} +function withLeak(base: WorkBundle): WorkBundle { + const next = structuredClone(base); + next.evidence.activity.capturedActivities = [{ ...LEAK_ACTIVITY }]; + next.evidence.activity.recommendationServiceReceipts = [{ ...LEAK_ACTIVITY }]; + return next; +} + +interface State { + off: WorkBundle; + on: WorkBundle; + sealed: GateReport; + tampered: boolean; +} + +// --------------------------------------------------------------------------- +// Small DOM helpers. Every value derived from evidence is written with +// textContent, never innerHTML, so a hostile BYO file cannot inject markup. +// --------------------------------------------------------------------------- +function byId(id: string): T { + const node = document.getElementById(id); + if (node === null) { + throw new Error(`missing element #${id}`); + } + return node as T; +} +function tag( + name: K, + className?: string, + text?: string, +): HTMLElementTagNameMap[K] { + const node = document.createElement(name); + if (className !== undefined) node.className = className; + if (text !== undefined) node.textContent = text; + return node; +} +function download(name: string, text: string, mime: string): void { + const url = URL.createObjectURL(new Blob([text], { type: mime })); + const anchor = tag("a"); + anchor.href = url; + anchor.download = name; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + setTimeout(() => URL.revokeObjectURL(url), 0); +} + +interface Fact { + k: string; + v: string; + leak?: boolean; +} +function offFacts(ev: WorkEvidence): Fact[] { + const captured = ev.activity.capturedActivities.length; + const receipts = ev.activity.recommendationServiceReceipts.length; + return [ + { k: "Scenario", v: "OFF — personalization off" }, + { k: "Subject", v: ev.subjectId }, + { + k: "Preference (ui / stored / backend)", + v: `${ev.control.uiPreference} / ${ev.control.storedPreference} / ${ev.control.backendPreference}`, + }, + { k: "Reload witnessed", v: ev.control.reloadObserved ? "yes" : "no" }, + { k: "Identifiable captured activity", v: String(captured), leak: captured > 0 }, + { k: "Recommendation-service receipts", v: String(receipts), leak: receipts > 0 }, + { + k: "Feed", + v: `${ev.recommendations.renderedSource}, ${ev.recommendations.renderedItemIds.length} item(s)`, + }, + ]; +} +function onFacts(ev: WorkEvidence): Fact[] { + return [ + { k: "Scenario", v: "ON — personalization on" }, + { k: "Subject", v: ev.subjectId }, + { + k: "Preference (ui / stored / backend)", + v: `${ev.control.uiPreference} / ${ev.control.storedPreference} / ${ev.control.backendPreference}`, + }, + { + k: "Correlated activity captured", + v: String(ev.activity.capturedActivities.length), + }, + { + k: "Feed", + v: `${ev.recommendations.renderedSource}, ${ev.recommendations.renderedItemIds.length} item(s)`, + }, + ]; +} +function renderFacts(container: HTMLElement, facts: Fact[]): void { + container.replaceChildren(); + for (const fact of facts) { + const row = tag("div", `vf-fact${fact.leak ? " leak" : ""}`); + row.append(tag("span", "vf-fact-k", fact.k), tag("span", "vf-fact-v", fact.v)); + container.append(row); + } +} + +interface ClauseLike { + id: string; + passed: boolean; + expected: string; + observed: string; +} +function renderClauses( + tbody: HTMLElement, + off: readonly ClauseLike[], + on: readonly ClauseLike[], +): void { + tbody.replaceChildren(); + const scoped: Array<{ scope: string; clause: ClauseLike }> = [ + ...off.map((clause) => ({ scope: "OFF", clause })), + ...on.map((clause) => ({ scope: "ON", clause })), + ]; + for (const { scope, clause } of scoped) { + const row = tag("tr"); + + const idCell = tag("td"); + idCell.append( + tag("span", "vf-clause-scope", scope), + tag("span", "vf-clause-id", clause.id), + ); + + const resultCell = tag("td"); + resultCell.append( + tag( + "span", + `vf-tag ${clause.passed ? "pass" : "fail"}`, + clause.passed ? "PASS" : "FAIL", + ), + ); + + const detailCell = tag("td", "vf-clause-detail"); + detailCell.append( + tag("b", undefined, "Expected "), + document.createTextNode(clause.expected), + tag("br"), + tag("b", undefined, "Observed "), + document.createTextNode(clause.observed), + ); + + row.append(idCell, resultCell, detailCell); + tbody.append(row); + } +} + +interface ViolationLike { + code: string; + clause: string; + message: string; +} +function renderViolations(list: HTMLElement, violations: readonly ViolationLike[]): void { + list.replaceChildren(); + if (violations.length === 0) { + list.className = "vf-violations empty"; + list.append(tag("li", undefined, "Promise verified — no violations.")); + return; + } + list.className = "vf-violations"; + for (const violation of violations) { + const item = tag("li"); + item.append( + tag("code", undefined, violation.code), + document.createTextNode(` (${violation.clause}) — ${violation.message}`), + ); + list.append(item); + } +} + +function setPill(pill: HTMLElement, text: string, kind: "good" | "bad" | "warn" | "idle"): void { + const label = pill.querySelector(".vf-pill-label"); + pill.className = `vf-pill${kind === "idle" ? "" : ` ${kind}`}`; + pill.replaceChildren(); + if (label !== null) pill.append(label); + pill.append(document.createTextNode(text)); +} + +function outcomeKind(outcome: string): "good" | "bad" | "warn" { + if (outcome === "PASS") return "good"; + if (outcome === "BROKEN_PROMISE") return "bad"; + return "warn"; +} +function reproKind(status: string): "good" | "warn" { + return status === "BOUND_AND_REPRODUCED" ? "good" : "warn"; +} +function noteFor(outcome: string, repro: string, tampered: boolean): string { + if (outcome === "INVALID_EVIDENCE") { + return "This evidence does not pass strict validation, so the deterministic evaluator does not run on it."; + } + if (outcome === "PASS" && repro === "BOUND_AND_REPRODUCED") { + return "PASS. The deterministic evaluator verified the promise over the recorded evidence, and the sealed report reproduces exactly from it. No model graded this — an unchanged evaluator did."; + } + if (outcome === "BROKEN_PROMISE" && repro !== "BOUND_AND_REPRODUCED") { + return "Broken promise. Identifiable activity is present in the OFF evidence, so the same evaluator now fails no_identifiable_activity and raises PP_IDENTIFIABLE_EVENT_LEAK. The report sealed a moment ago no longer reproduces from this evidence — a PASS cannot be silently carried onto changed evidence."; + } + if (outcome === "BROKEN_PROMISE" && repro === "BOUND_AND_REPRODUCED") { + return "Broken promise, sealed honestly. A new report was sealed for the current evidence and it reproduces — as BROKEN_PROMISE. Sealing binds whatever is true; it cannot manufacture a PASS."; + } + if (tampered) { + return "The evidence changed. Re-evaluate it, and verify the sealed report against it."; + } + return "PASS on this evidence, but the sealed report was bound to different evidence, so it does not reproduce here."; +} + +// --------------------------------------------------------------------------- +// Page assembly. +// --------------------------------------------------------------------------- +const APP_HTML = ` +
+
+ Deterministic verifier · running in this page + contract activity-personalization/v1 +
+
+
+

Evaluator source SHA-256

+

+

Pinned to this build and verified against evaluator.ts by repository tests.

+
+
+

Canonicalization / digest

+

+

Evidence digests below are recomputed here from the evidence shown.

+
+
+

Report schema

+

+

The same verifier modules the CLI ships. No evaluator rule runs in this UI.

+
+
+
+
+
+
OFF evidencethe promise under test
+
+
+
+
ON controlthe counter-example
+
+
+
+ +
+ + + + + + + +
+ +
+
+
+ Current evidence + Sealed report +
+

+
+ + + +
ClauseResultExpected vs observed
+
    +
    +

    +
    +
    +
    + +
    +
    Reproduce it from a terminalbyte-identical report
    +
    +

    The browser and the CLI run the same modules, so the report you download here is byte-identical to the one the CLI writes for the same evidence.

    +
    +
    passing example
    +
    +
    $ npm run promiseproof -- gate --off passing-off.json --on passing-on.json
    +
    PASS # exit 0
    +
    $ npm run promiseproof -- check --report report.json --off passing-off.json --on passing-on.json
    +
    BOUND_AND_REPRODUCED # exit 0
    +
    +
    +
    +
    after tampering the OFF evidence
    +
    +
    $ npm run promiseproof -- gate --off tampered-off.json --on passing-on.json
    +
    BROKEN_PROMISE # exit 2 · PP_IDENTIFIABLE_EVENT_LEAK
    +
    $ npm run promiseproof -- check --report report.json --off tampered-off.json --on passing-on.json
    +
    STALE_OR_MISMATCH # exit 4
    +
    +
    +
    +
    + +
    +
    Bring your own evidenceparsed locally, never uploaded
    +
    +
    +
    + + +

    +
    +
    + + +

    +
    +
    +
    + + +
    + +

    Provide one bundle for a single verification, or both an OFF and ON bundle to run the gate. Files are read with the browser's local file API; nothing is sent anywhere.

    +
    +
    + +
    +
    What this proves, and what it doesn't
    +
    +

    What it proves. The verdict comes from one deterministic evaluator whose source is fingerprinted above. The AI that diagnoses and repairs the promise never writes PASS; this unchanged evaluator does, and you can re-derive it here or from the CLI and get the same answer.

    +

    Tamper-evidence. A sealed report is bound to the exact evidence it was computed from. Change the evidence and the report no longer reproduces (STALE_OR_MISMATCH); you can only ever seal what is actually true.

    +

    What it does not claim. Evidence is externally supplied and, here, synthetic. PromiseProof does not attest how evidence was collected — only that a bundle which passes strict validation is evaluated deterministically. Full loop and source: github.com/AlexPaiva/PromiseProof.

    +
    +
    +`; + +const state: State = { + off: freshOff(), + on: freshOn(), + sealed: undefined as unknown as GateReport, + tampered: false, +}; + +// Cached refs, populated after the shell renders. +let offFactsEl: HTMLElement; +let onFactsEl: HTMLElement; +let evOffEl: HTMLElement; +let outcomePill: HTMLElement; +let reproPill: HTMLElement; +let noteEl: HTMLElement; +let clausesEl: HTMLElement; +let violationsEl: HTMLElement; +let digestsEl: HTMLElement; +let toastEl: HTMLElement; +let tamperBtn: HTMLButtonElement; + +function toast(message: string): void { + toastEl.textContent = message; +} + +function renderDigestPairs(target: HTMLElement, entries: Array<[string, string]>): void { + target.replaceChildren(); + for (const [k, v] of entries) { + const cell = tag("div"); + cell.append(tag("p", "vf-digest-k", k), tag("p", "vf-digest-v", v)); + target.append(cell); + } +} +function gateDigestPairs(report: GateReport): Array<[string, string]> { + return [ + ["OFF evidence SHA-256", report.inputBindings.off.sha256], + ["ON evidence SHA-256", report.inputBindings.on.sha256], + ]; +} + +async function refresh(): Promise<{ outcome: string; repro: string }> { + const gate = runGate(state.off, state.on); + renderFacts(offFactsEl, offFacts(state.off.evidence)); + renderFacts(onFactsEl, onFacts(state.on.evidence)); + evOffEl.classList.toggle("tampered", state.tampered); + + setPill(outcomePill, gate.outcome, outcomeKind(gate.outcome)); + renderClauses( + clausesEl, + gate.off.evaluation?.clauses ?? [], + gate.on.evaluation?.clauses ?? [], + ); + renderViolations(violationsEl, [ + ...(gate.off.evaluation?.violations ?? []), + ...(gate.on.evaluation?.violations ?? []), + ]); + + const repro = await checkGate(state.sealed, state.off, state.on); + setPill(reproPill, repro.status, reproKind(repro.status)); + + if (gate.outcome !== "INVALID_EVIDENCE") { + renderDigestPairs(digestsEl, gateDigestPairs(await createGateReport(gate))); + } else { + digestsEl.replaceChildren(); + } + + noteEl.textContent = noteFor(gate.outcome, repro.status, state.tampered); + tamperBtn.disabled = state.tampered; + return { outcome: gate.outcome, repro: repro.status }; +} + +async function seal(): Promise { + const gate = runGate(state.off, state.on); + if (gate.outcome === "INVALID_EVIDENCE") { + toast("Cannot seal invalid evidence."); + return; + } + state.sealed = await createGateReport(gate); + const { outcome, repro } = await refresh(); + toast(`Sealed the current result: ${outcome} · ${repro}`); +} + +async function currentReport(): Promise { + const gate = runGate(state.off, state.on); + if (gate.outcome === "INVALID_EVIDENCE") return null; + return createGateReport(gate); +} + +// ---- BYO ---- +let byoOffText: string | null = null; +let byoOnText: string | null = null; + +function parseBundle(text: string): unknown { + return JSON.parse(text); +} + +function renderByo( + off: VerifyResult, + on: VerifyResult | null, + digests: Array<[string, string]>, + gateIssues: readonly string[] = [], +): void { + const result = byId("vf-byo-result"); + const outcome = byId("vf-byo-outcome"); + const issues = byId("vf-byo-issues"); + const table = byId("vf-byo-table"); + const violations = byId("vf-byo-violations"); + const digestBox = byId("vf-byo-digests"); + result.hidden = false; + + const combinedOutcome = + on === null + ? off.outcome + : off.outcome === "INVALID_EVIDENCE" || on.outcome === "INVALID_EVIDENCE" + ? "INVALID_EVIDENCE" + : off.outcome === "PASS" && on.outcome === "PASS" + ? "PASS" + : "BROKEN_PROMISE"; + setPill(outcome, combinedOutcome, outcomeKind(combinedOutcome)); + + const allIssues = [ + ...gateIssues, + ...off.issues.map((issue) => (on === null ? issue : `off · ${issue}`)), + ...(on?.issues ?? []).map((issue) => `on · ${issue}`), + ]; + issues.replaceChildren(); + issues.hidden = allIssues.length === 0; + for (const issue of allIssues) issues.append(tag("li", undefined, issue)); + + const offClauses = off.evaluation?.clauses ?? []; + const onClauses = on?.evaluation?.clauses ?? []; + const hasClauses = offClauses.length + onClauses.length > 0; + table.hidden = !hasClauses; + if (hasClauses) { + renderClauses(byId("vf-byo-clauses"), offClauses, onClauses); + } + renderViolations(violations, [ + ...(off.evaluation?.violations ?? []), + ...(on?.evaluation?.violations ?? []), + ]); + renderDigestPairs(digestBox, digests); +} + +async function runByo(): Promise { + if (byoOffText === null && byoOnText === null) { + toast("Select at least one bundle file first."); + return; + } + try { + if (byoOffText !== null && byoOnText !== null) { + const gate = runGate(parseBundle(byoOffText), parseBundle(byoOnText)); + const digests = + gate.outcome === "INVALID_EVIDENCE" + ? [] + : gateDigestPairs(await createGateReport(gate)); + renderByo(gate.off, gate.on, digests, gate.issues); + } else { + const single = verifyBundle(parseBundle((byoOffText ?? byoOnText) as string)); + const digests = + single.outcome === "INVALID_EVIDENCE" + ? [] + : [ + [ + "Evidence SHA-256", + (await createVerifyReport(single)).inputBinding.sha256, + ] as [string, string], + ]; + renderByo(single, null, digests); + } + toast("Verified local evidence."); + } catch { + renderByoInvalid(": malformed JSON"); + toast("That file is not valid JSON."); + } +} + +function renderByoInvalid(issue: string): void { + const result = byId("vf-byo-result"); + const outcome = byId("vf-byo-outcome"); + const issues = byId("vf-byo-issues"); + result.hidden = false; + setPill(outcome, "INVALID_EVIDENCE", "warn"); + issues.replaceChildren(tag("li", undefined, issue)); + issues.hidden = false; + byId("vf-byo-table").hidden = true; + byId("vf-byo-clauses").replaceChildren(); + byId("vf-byo-violations").replaceChildren(); + byId("vf-byo-digests").replaceChildren(); +} + +function wireFile( + inputId: string, + dropId: string, + nameId: string, + assign: (text: string | null, name: string) => void, +): void { + const input = byId(inputId); + const drop = byId(dropId); + const nameEl = byId(nameId); + const load = (file: File): void => { + assign(null, file.name); + byId("vf-byo-result").hidden = true; + if (file.size > MAX_INPUT_BYTES) { + const issue = `: input exceeds ${MAX_INPUT_BYTES}-byte limit`; + nameEl.textContent = `${file.name} — rejected`; + input.value = ""; + renderByoInvalid(issue); + toast("That file exceeds the verifier input-size limit."); + return; + } + const reader = new FileReader(); + reader.onload = () => { + assign(String(reader.result), file.name); + nameEl.textContent = file.name; + }; + reader.onerror = () => { + assign(null, file.name); + renderByoInvalid(": file could not be read"); + toast("Could not read that file."); + }; + reader.readAsText(file); + }; + input.addEventListener("change", () => { + const file = input.files?.[0]; + if (file) load(file); + }); + drop.addEventListener("dragover", (event) => { + event.preventDefault(); + drop.classList.add("drag"); + }); + drop.addEventListener("dragleave", () => drop.classList.remove("drag")); + drop.addEventListener("drop", (event) => { + event.preventDefault(); + drop.classList.remove("drag"); + const file = event.dataTransfer?.files?.[0]; + if (file) load(file); + }); +} + +async function init(): Promise { + const appRoot = byId("vf-app"); + appRoot.innerHTML = APP_HTML; + + offFactsEl = byId("vf-off-facts"); + onFactsEl = byId("vf-on-facts"); + evOffEl = byId("vf-ev-off"); + outcomePill = byId("vf-outcome"); + reproPill = byId("vf-repro"); + noteEl = byId("vf-note"); + clausesEl = byId("vf-clauses"); + violationsEl = byId("vf-violations"); + digestsEl = byId("vf-digests"); + toastEl = byId("vf-toast"); + tamperBtn = byId("vf-tamper"); + + byId("vf-fp").textContent = EVALUATOR_SOURCE_SHA256; + byId("vf-canon").textContent = `${CANONICAL_JSON_ID} · ${SHA256_ALGORITHM}`; + byId("vf-schema").textContent = `report schema v${REPORT_SCHEMA_VERSION}`; + + // Seal the passing example on load so Judge Mode opens on PASS + BOUND. + state.sealed = await createGateReport(runGate(state.off, state.on)); + + tamperBtn.addEventListener("click", async () => { + state.off = withLeak(state.off); + state.tampered = true; + const { outcome, repro } = await refresh(); + toast(`Added one identifiable page_view + receipt to OFF evidence → ${outcome} · sealed report ${repro}`); + }); + byId("vf-evaluate").addEventListener("click", async () => { + const { outcome } = await refresh(); + toast(`Evaluated current evidence: ${outcome}`); + }); + byId("vf-check").addEventListener("click", async () => { + const check = await checkGate(state.sealed, state.off, state.on); + await refresh(); + toast(`Loaded report vs current evidence: ${check.status}`); + }); + byId("vf-seal").addEventListener("click", () => { + void seal(); + }); + byId("vf-reset").addEventListener("click", async () => { + state.off = freshOff(); + state.on = freshOn(); + state.tampered = false; + state.sealed = await createGateReport(runGate(state.off, state.on)); + await refresh(); + toast("Reset to the passing example."); + }); + byId("vf-dl-json").addEventListener("click", async () => { + const report = await currentReport(); + if (report === null) { + toast("No report for invalid evidence."); + return; + } + download("promiseproof-report.json", serializeReportJson(report), "application/json"); + }); + byId("vf-dl-md").addEventListener("click", async () => { + const report = await currentReport(); + if (report === null) { + toast("No report for invalid evidence."); + return; + } + download("promiseproof-report.md", serializeGateReportMarkdown(report), "text/markdown"); + }); + + wireFile("vf-file-off", "vf-drop-off", "vf-name-off", (text) => { + byoOffText = text; + }); + wireFile("vf-file-on", "vf-drop-on", "vf-name-on", (text) => { + byoOnText = text; + }); + byId("vf-byo-run").addEventListener("click", () => { + void runByo(); + }); + byId("vf-byo-clear").addEventListener("click", () => { + byoOffText = null; + byoOnText = null; + byId("vf-name-off").textContent = ""; + byId("vf-name-on").textContent = ""; + byId("vf-file-off").value = ""; + byId("vf-file-on").value = ""; + byId("vf-byo-result").hidden = true; + }); + + await refresh(); +} + +void init(); diff --git a/src/verify-ui/styles.css b/src/verify-ui/styles.css new file mode 100644 index 0000000..9ce33fb --- /dev/null +++ b/src/verify-ui/styles.css @@ -0,0 +1,696 @@ +/* PromiseProof Verify — Judge Mode. Dark control-console styling on the same + palette and system type as the public landing, so /verify/ reads as one site. + No web fonts (CSP-safe, offline-safe); state is carried by semantic colour. */ + +:root { + --bg: #080b12; + --bg2: #0c111c; + --panel: #0e1524; + --panel2: #121a2b; + --line: rgba(255, 255, 255, 0.1); + --line-soft: rgba(255, 255, 255, 0.06); + --text: #e8edf6; + --muted: #8ea3c4; + --muted2: #7c8ca6; + --blue: #3f6bf0; + --blue-bright: #6f9bff; + --red: #e5484d; + --red-bright: #ff8f93; + --green: #2fa968; + --green-bright: #5ad39a; + --amber: #e0a63b; + --amber-bright: #f2c66b; + --mono: ui-monospace, "SF Mono", Menlo, Consolas, monospace; + --sans: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + --maxw: 1080px; + --radius: 14px; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + font-family: var(--sans); + background: + radial-gradient(1200px 600px at 50% -8%, rgba(63, 107, 240, 0.14), transparent 60%), + var(--bg); + color: var(--text); + line-height: 1.55; + -webkit-font-smoothing: antialiased; + overflow-x: hidden; +} + +.mono { + font-family: var(--mono); +} + +a { + color: var(--blue-bright); + text-decoration: none; +} +a:hover { + text-decoration: underline; +} + +:focus-visible { + outline: 2px solid var(--blue-bright); + outline-offset: 2px; + border-radius: 6px; +} + +.vf-shell { + max-width: var(--maxw); + margin: 0 auto; + padding: 0 22px 72px; +} + +/* ---- Topbar ---- */ +.vf-topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + flex-wrap: wrap; + padding: 20px 0 18px; + border-bottom: 1px solid var(--line-soft); +} +.vf-brand { + display: inline-flex; + align-items: center; + gap: 12px; + color: var(--text); +} +.vf-brand:hover { + text-decoration: none; +} +.vf-mark { + width: 30px; + height: 30px; + border-radius: 8px; + background: linear-gradient(160deg, #12203b, #0a1120); + display: grid; + place-items: center; + gap: 2px; + grid-auto-flow: column; + box-shadow: inset 0 0 0 1px rgba(255, 255, 255, 0.08); +} +.vf-mark span { + width: 3px; + height: 12px; + border-radius: 2px; + background: var(--green-bright); +} +.vf-mark span:nth-child(2) { + height: 8px; + background: var(--blue-bright); +} +.vf-mark span:nth-child(3) { + height: 15px; + background: var(--green); +} +.vf-brand-text strong { + display: block; + font-size: 0.98rem; + letter-spacing: 0.01em; +} +.vf-brand-text small { + display: block; + color: var(--muted); + font-size: 0.72rem; +} +.vf-topbar-actions { + display: flex; + align-items: center; + gap: 14px; + flex-wrap: wrap; +} +.vf-chip { + margin: 0; + display: inline-flex; + align-items: center; + gap: 8px; + font-family: var(--mono); + font-size: 0.66rem; + letter-spacing: 0.04em; + color: #93c2a8; + border: 1px solid rgba(90, 211, 154, 0.25); + padding: 6px 12px; + border-radius: 999px; +} +.vf-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--green-bright); + box-shadow: 0 0 0 3px rgba(90, 211, 154, 0.18); +} +.vf-toplink { + font-family: var(--mono); + font-size: 0.72rem; + letter-spacing: 0.03em; + color: var(--muted); +} + +/* ---- Hero ---- */ +.vf-hero { + padding: 44px 0 30px; +} +.vf-eyebrow { + display: inline-flex; + align-items: center; + gap: 9px; + font-family: var(--mono); + font-size: 0.72rem; + letter-spacing: 0.1em; + text-transform: uppercase; + color: #93c2a8; + border: 1px solid rgba(90, 211, 154, 0.25); + padding: 6px 13px; + border-radius: 999px; + margin: 0 0 22px; +} +.vf-h1 { + font-size: clamp(2.1rem, 5.4vw, 3.2rem); + line-height: 1.04; + letter-spacing: -0.02em; + margin: 0 0 18px; + text-wrap: balance; +} +.vf-h1 .g { + background: linear-gradient(100deg, #9db8ff, #6f9bff 60%, #5ad39a); + -webkit-background-clip: text; + background-clip: text; + color: transparent; +} +.vf-lede { + max-width: 68ch; + font-size: 1.06rem; + color: #cfd8ea; + margin: 0 0 16px; +} +.vf-thesis { + max-width: 62ch; + font-family: var(--mono); + font-size: 0.82rem; + color: var(--green-bright); + border-left: 2px solid rgba(90, 211, 154, 0.4); + padding-left: 14px; + margin: 0; +} + +/* ---- App container ---- */ +.vf-app { + display: grid; + gap: 22px; +} +.vf-fallback { + border: 1px solid var(--line); + border-radius: var(--radius); + padding: 22px; + color: var(--muted); + background: var(--panel); +} + +.vf-panel { + border: 1px solid var(--line); + border-radius: 16px; + background: linear-gradient(180deg, rgba(18, 26, 42, 0.72), rgba(10, 15, 25, 0.72)); + box-shadow: 0 30px 70px -46px rgba(0, 0, 0, 0.8); + overflow: hidden; +} +.vf-panel-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + flex-wrap: wrap; + padding: 13px 20px; + border-bottom: 1px solid var(--line-soft); + font-family: var(--mono); + font-size: 0.62rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted); +} +.vf-panel-body { + padding: 20px; +} +.vf-section-title { + font-size: 1.02rem; + margin: 0; + letter-spacing: -0.01em; +} + +/* ---- Authority strip ---- */ +.vf-authority { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); + gap: 1px; + background: var(--line-soft); +} +.vf-auth-cell { + background: var(--panel); + padding: 14px 18px; +} +.vf-auth-k { + font-family: var(--mono); + font-size: 0.58rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted2); + margin: 0 0 5px; +} +.vf-auth-v { + font-family: var(--mono); + font-size: 0.76rem; + color: var(--text); + word-break: break-all; + line-height: 1.5; +} +.vf-auth-note { + color: var(--muted); + font-size: 0.68rem; + margin-top: 4px; + font-family: var(--sans); + letter-spacing: 0; +} + +/* ---- Evidence panels ---- */ +.vf-evidence { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 16px; +} +.vf-ev { + border: 1px solid var(--line); + border-radius: 13px; + background: var(--panel); + overflow: hidden; +} +.vf-ev-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + padding: 11px 16px; + border-bottom: 1px solid var(--line-soft); + font-family: var(--mono); + font-size: 0.64rem; + letter-spacing: 0.09em; + text-transform: uppercase; + color: var(--muted); +} +.vf-ev-badge { + font-weight: 700; + color: var(--blue-bright); +} +.vf-ev-body { + padding: 6px 16px 14px; +} +.vf-fact { + display: flex; + justify-content: space-between; + gap: 12px; + padding: 7px 0; + border-bottom: 1px dashed var(--line-soft); + font-size: 0.82rem; +} +.vf-fact:last-child { + border-bottom: none; +} +.vf-fact-k { + color: var(--muted); +} +.vf-fact-v { + font-family: var(--mono); + font-size: 0.78rem; + text-align: right; + word-break: break-word; +} +.vf-fact.leak { + color: var(--red-bright); +} +.vf-fact.leak .vf-fact-k { + color: var(--red-bright); +} +.vf-fact.leak .vf-fact-v { + color: var(--red-bright); + font-weight: 700; +} +.vf-ev.tampered { + border-color: rgba(229, 72, 77, 0.55); + box-shadow: 0 0 0 1px rgba(229, 72, 77, 0.25); +} + +/* ---- Actions ---- */ +.vf-actions { + display: flex; + flex-wrap: wrap; + gap: 10px; +} +.vf-btn { + appearance: none; + font: inherit; + font-size: 0.84rem; + font-weight: 600; + cursor: pointer; + border-radius: 10px; + padding: 10px 16px; + border: 1px solid var(--line); + background: rgba(255, 255, 255, 0.04); + color: var(--text); + transition: transform 0.14s ease, border-color 0.14s ease, background 0.14s ease; +} +.vf-btn:hover { + border-color: rgba(255, 255, 255, 0.24); + transform: translateY(-1px); +} +.vf-btn:active { + transform: translateY(0); +} +.vf-btn:disabled { + opacity: 0.42; + cursor: not-allowed; + transform: none; +} +.vf-btn-tamper { + border-color: rgba(229, 72, 77, 0.5); + background: linear-gradient(180deg, rgba(229, 72, 77, 0.18), rgba(229, 72, 77, 0.06)); + color: #ffd9da; +} +.vf-btn-tamper:hover { + border-color: rgba(229, 72, 77, 0.8); +} +.vf-btn-primary { + border-color: rgba(111, 155, 255, 0.5); + background: linear-gradient(180deg, rgba(63, 107, 240, 0.28), rgba(63, 107, 240, 0.1)); + color: #dfe8ff; +} +.vf-btn-quiet { + color: var(--muted); + background: transparent; +} + +/* ---- Result ---- */ +.vf-verdict { + display: flex; + align-items: center; + gap: 12px; + flex-wrap: wrap; + margin-bottom: 6px; +} +.vf-pill { + display: inline-flex; + align-items: center; + gap: 8px; + font-family: var(--mono); + font-size: 0.74rem; + font-weight: 700; + letter-spacing: 0.06em; + padding: 7px 13px; + border-radius: 999px; + border: 1px solid var(--line); + color: var(--muted); + background: rgba(255, 255, 255, 0.03); +} +.vf-pill-label { + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--muted2); + margin-right: 2px; +} +.vf-pill.good { + color: var(--green-bright); + border-color: rgba(90, 211, 154, 0.45); + background: rgba(47, 169, 104, 0.12); +} +.vf-pill.bad { + color: var(--red-bright); + border-color: rgba(229, 72, 77, 0.5); + background: rgba(229, 72, 77, 0.12); +} +.vf-pill.warn { + color: var(--amber-bright); + border-color: rgba(224, 166, 59, 0.5); + background: rgba(224, 166, 59, 0.12); +} +.vf-result-note { + color: #cfd8ea; + font-size: 0.9rem; + margin: 10px 0 16px; +} +.vf-result-note strong { + color: var(--text); +} + +.vf-clauses { + width: 100%; + border-collapse: collapse; + font-size: 0.82rem; + margin: 4px 0 6px; +} +.vf-clauses th { + text-align: left; + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted2); + font-weight: 600; + padding: 6px 10px; + border-bottom: 1px solid var(--line-soft); +} +.vf-clauses td { + padding: 9px 10px; + border-bottom: 1px solid var(--line-soft); + vertical-align: top; +} +.vf-clauses tr:last-child td { + border-bottom: none; +} +.vf-clause-id { + font-family: var(--mono); + font-size: 0.76rem; + color: var(--text); +} +.vf-clause-scope { + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--muted2); + display: block; +} +.vf-clause-detail { + color: var(--muted); + font-size: 0.74rem; + font-family: var(--mono); + line-height: 1.5; +} +.vf-clause-detail b { + color: #c7d2e6; + font-weight: 600; +} +.vf-tag { + font-family: var(--mono); + font-size: 0.6rem; + font-weight: 700; + letter-spacing: 0.06em; + padding: 3px 8px; + border-radius: 6px; + white-space: nowrap; +} +.vf-tag.pass { + color: var(--green-bright); + background: rgba(47, 169, 104, 0.14); +} +.vf-tag.fail { + color: var(--red-bright); + background: rgba(229, 72, 77, 0.16); +} + +.vf-violations { + list-style: none; + margin: 12px 0 0; + padding: 0; + display: grid; + gap: 8px; +} +.vf-violations li { + border: 1px solid rgba(229, 72, 77, 0.32); + background: rgba(229, 72, 77, 0.08); + border-radius: 10px; + padding: 10px 13px; + font-size: 0.82rem; +} +.vf-violations code { + font-family: var(--mono); + font-size: 0.76rem; + color: var(--red-bright); + font-weight: 700; +} +.vf-violations.empty li { + border-color: rgba(90, 211, 154, 0.3); + background: rgba(47, 169, 104, 0.08); + color: var(--green-bright); +} + +.vf-digests { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 10px; + margin-top: 16px; + padding-top: 14px; + border-top: 1px solid var(--line-soft); +} +.vf-digest-k { + font-family: var(--mono); + font-size: 0.56rem; + letter-spacing: 0.11em; + text-transform: uppercase; + color: var(--muted2); +} +.vf-digest-v { + font-family: var(--mono); + font-size: 0.72rem; + color: #c7d2e6; + word-break: break-all; +} + +/* ---- CLI mirror ---- */ +.vf-cli { + background: #0b111c; + border: 1px solid var(--line); + border-radius: 13px; + overflow: hidden; + font-family: var(--mono); + font-size: 0.78rem; +} +.vf-cli-head { + padding: 9px 15px; + border-bottom: 1px solid var(--line-soft); + color: var(--muted2); + font-size: 0.6rem; + letter-spacing: 0.11em; + text-transform: uppercase; +} +.vf-cli-body { + padding: 14px 16px; + display: grid; + gap: 6px; + overflow-x: auto; +} +.vf-cli-line { + white-space: pre; + color: #c7d2e6; +} +.vf-cli-line .p { + color: var(--green-bright); +} +.vf-cli-line .c { + color: var(--muted2); +} +.vf-cli-line .ok { + color: var(--green-bright); +} +.vf-cli-line .no { + color: var(--red-bright); +} + +/* ---- BYO ---- */ +.vf-byo-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 14px; + margin-bottom: 14px; +} +.vf-drop { + border: 1px dashed rgba(255, 255, 255, 0.22); + border-radius: 12px; + padding: 16px; + background: rgba(255, 255, 255, 0.02); + transition: border-color 0.14s ease, background 0.14s ease; +} +.vf-drop.drag { + border-color: var(--blue-bright); + background: rgba(63, 107, 240, 0.1); +} +.vf-drop label { + display: block; + font-family: var(--mono); + font-size: 0.62rem; + letter-spacing: 0.09em; + text-transform: uppercase; + color: var(--muted); + margin-bottom: 8px; +} +.vf-drop input[type="file"] { + font-family: var(--mono); + font-size: 0.74rem; + color: var(--muted); + max-width: 100%; +} +.vf-drop-name { + margin-top: 8px; + font-family: var(--mono); + font-size: 0.72rem; + color: var(--green-bright); + word-break: break-all; +} +.vf-issues { + list-style: none; + margin: 10px 0 0; + padding: 0; + display: grid; + gap: 6px; +} +.vf-issues li { + font-family: var(--mono); + font-size: 0.72rem; + color: var(--amber-bright); + border-left: 2px solid rgba(224, 166, 59, 0.5); + padding-left: 10px; +} + +.vf-muted { + color: var(--muted); + font-size: 0.82rem; +} +.vf-disclose p { + color: #cfd8ea; + font-size: 0.88rem; +} +.vf-disclose strong { + color: var(--text); +} + +.vf-footer { + margin-top: 20px; + padding-top: 18px; + border-top: 1px solid var(--line-soft); + color: var(--muted2); + font-size: 0.78rem; + max-width: 76ch; +} + +/* ---- Responsive ---- */ +@media (max-width: 720px) { + .vf-evidence, + .vf-byo-grid { + grid-template-columns: 1fr; + } + .vf-shell { + padding: 0 16px 56px; + } +} + +@media (prefers-reduced-motion: reduce) { + * { + transition: none !important; + animation: none !important; + } +} diff --git a/src/verify/adapter.ts b/src/verify/adapter.ts new file mode 100644 index 0000000..297ac5d --- /dev/null +++ b/src/verify/adapter.ts @@ -0,0 +1,104 @@ +import type { + ActivityPayload, + PromiseEvidence, + RecommendationReceipt, +} from "../shared/types.js"; +import type { + ExternalActivity, + ExternalEvidenceV1, + ExternalRecommendationReceipt, +} from "./schema.js"; + +const PLACEHOLDER_RUN_ID = "promiseproof-external-adapter"; +const PLACEHOLDER_TIMESTAMP = "2000-01-01T00:00:00.000Z"; +const PLACEHOLDER_TEXT = "Not part of externally supplied evidence"; + +function adaptActivity(activity: ExternalActivity): ActivityPayload { + return { + runId: activity.runId, + userId: activity.subjectId, + eventType: activity.eventType, + itemId: activity.itemId, + clientSequence: activity.clientSequence, + occurredAt: activity.occurredAt, + }; +} + +function adaptRecommendationReceipt( + receipt: ExternalRecommendationReceipt, + index: number, +): RecommendationReceipt { + return { + kind: "recommendation", + receiptId: `external-recommendation-receipt-${index + 1}`, + sequence: index + 1, + receivedAt: PLACEHOLDER_TIMESTAMP, + source: receipt.source, + ...(receipt.subjectId === undefined + ? {} + : { userId: receipt.subjectId }), + items: receipt.itemIds.map((id) => ({ + id, + title: PLACEHOLDER_TEXT, + description: PLACEHOLDER_TEXT, + eyebrow: PLACEHOLDER_TEXT, + })), + }; +} + +export function adaptExternalEvidence( + external: ExternalEvidenceV1, +): PromiseEvidence { + return { + scenario: external.scenario, + runId: PLACEHOLDER_RUN_ID, + userId: external.subjectId, + ui: { + preference: external.control.uiPreference, + toggleChecked: external.control.toggleChecked, + feedFunctional: external.recommendations.feedFunctional, + }, + storage: { + preference: external.control.storedPreference, + }, + request: { + activityPayloads: external.activity.capturedActivities.map(adaptActivity), + preferenceUpdates: [], + }, + response: { + preferenceUpdates: [], + }, + backend: { + preference: external.control.backendPreference, + activityReceipts: + external.activity.recommendationServiceReceipts.map( + (activity, index) => ({ + kind: "activity", + service: "recommendation", + receiptId: `external-activity-receipt-${index + 1}`, + sequence: index + 1, + receivedAt: PLACEHOLDER_TIMESTAMP, + payload: adaptActivity(activity), + }), + ), + recommendationReceipts: + external.recommendations.recommendationServiceReceipts.map( + adaptRecommendationReceipt, + ), + preferenceReceipts: [], + }, + recommendation: { + source: external.recommendations.renderedSource, + itemIds: [...external.recommendations.renderedItemIds], + }, + timestamps: { + clientTimeline: [], + activityReceivedAt: [], + preferenceReceivedAt: [], + recommendationReceivedAt: [], + }, + journey: { + reloadObserved: external.control.reloadObserved, + }, + }; +} diff --git a/src/verify/binding.ts b/src/verify/binding.ts new file mode 100644 index 0000000..4d0e3b3 --- /dev/null +++ b/src/verify/binding.ts @@ -0,0 +1,105 @@ +import type { ExternalBundle } from "./schema.js"; + +export const CANONICAL_JSON_ID = "promiseproof-canonical-json/v1" as const; +export const SHA256_ALGORITHM = "SHA-256" as const; + +/** SHA-256 of src/shared/evaluator.ts after UTF-8 decoding and LF normalization. */ +export const EVALUATOR_SOURCE_SHA256 = + "fca20925861d1de2772ad0297a2465029678c4f34faa7a4755592f37aef9f87f" as const; + +export interface InputBinding { + readonly canonicalization: typeof CANONICAL_JSON_ID; + readonly algorithm: typeof SHA256_ALGORITHM; + readonly sha256: string; +} + +type CanonicalJson = + | null + | boolean + | number + | string + | readonly CanonicalJson[] + | { readonly [key: string]: CanonicalJson }; + +function compareCodeUnits(left: string, right: string): number { + if (left < right) { + return -1; + } + if (left > right) { + return 1; + } + return 0; +} + +function normalizeCanonicalJson(value: unknown): CanonicalJson { + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new TypeError("Canonical JSON accepts only finite numbers."); + } + return value; + } + if (Array.isArray(value)) { + return value.map(normalizeCanonicalJson); + } + if (typeof value === "object") { + const normalized: Record = {}; + for (const key of Object.keys(value).sort(compareCodeUnits)) { + normalized[key] = normalizeCanonicalJson( + (value as Record)[key], + ); + } + return normalized; + } + throw new TypeError(`Canonical JSON does not accept ${typeof value}.`); +} + +export function canonicalizeJson(value: unknown): string { + return JSON.stringify(normalizeCanonicalJson(value)); +} + +function lowercaseHex(bytes: ArrayBuffer): string { + return Array.from(new Uint8Array(bytes), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); +} + +export async function sha256Utf8(value: string): Promise { + const bytes = new TextEncoder().encode(value); + return lowercaseHex(await globalThis.crypto.subtle.digest(SHA256_ALGORITHM, bytes)); +} + +export async function bindExternalBundle( + bundle: ExternalBundle, +): Promise { + return bindCanonicalJson(canonicalizeJson(bundle)); +} + +export async function bindCanonicalJson( + canonicalJson: string, +): Promise { + return { + canonicalization: CANONICAL_JSON_ID, + algorithm: SHA256_ALGORITHM, + sha256: await sha256Utf8(canonicalJson), + }; +} + +export function canonicalizeSourceLineEndings(source: string): string { + return source.replaceAll("\r\n", "\n").replaceAll("\r", "\n"); +} + +export async function evaluatorSourceMatchesFingerprint( + source: string, + expected: string = EVALUATOR_SOURCE_SHA256, +): Promise { + return ( + (await sha256Utf8(canonicalizeSourceLineEndings(source))) === expected + ); +} diff --git a/src/verify/check.ts b/src/verify/check.ts new file mode 100644 index 0000000..62aa516 --- /dev/null +++ b/src/verify/check.ts @@ -0,0 +1,174 @@ +// Reproducible report check. A bound report is only trustworthy if it can be +// REPRODUCED: re-run the unchanged evaluator on the supplied evidence, regenerate +// the complete report, and compare the whole canonical document. Comparing only +// the embedded hashes is insufficient, a fabricated report could carry the correct +// evidence digest and evaluator fingerprint while inventing the verdict, clauses, +// or violations. Full reproduction catches that. +import { z } from "zod"; +import { + CANONICAL_JSON_ID, + canonicalizeJson, + SHA256_ALGORITHM, +} from "./binding.js"; +import { + REPORT_SCHEMA_VERSION, + SUPPORTED_CONTRACT_FAMILY, +} from "./outcome.js"; +import { createGateReport, createVerifyReport } from "./report.js"; +import { runGate, verifyBundle } from "./verify.js"; + +export const CHECK_EXIT = { + BOUND_AND_REPRODUCED: 0, + USAGE_OR_EXECUTION_ERROR: 1, + INVALID_REPORT_OR_EVIDENCE: 3, + STALE_OR_MISMATCH: 4, +} as const; + +export type CheckStatus = keyof typeof CHECK_EXIT; + +export interface CheckResult { + readonly status: CheckStatus; + readonly detail: string; +} + +const sha256 = z.string().regex(/^[0-9a-f]{64}$/u); +const inputBindingSchema = z + .object({ + canonicalization: z.literal(CANONICAL_JSON_ID), + algorithm: z.literal(SHA256_ALGORITHM), + sha256, + }) + .strict(); +const clauseIdSchema = z.enum([ + "no_identifiable_activity", + "contextual_feed_functional", + "preference_survives_reload", + "expected_activity_received", + "behavioral_feed_functional", +]); +const violationCodeSchema = z.enum([ + "PP_IDENTIFIABLE_EVENT_LEAK", + "PP_CONTEXTUAL_FEED_MISSING", + "PP_PREFERENCE_NOT_PERSISTED", + "PP_EXPECTED_ACTIVITY_MISSING", + "PP_BEHAVIORAL_FEED_MISSING", +]); +const clauseSchema = z + .object({ + id: clauseIdSchema, + passed: z.boolean(), + expected: z.string(), + observed: z.string(), + }) + .strict(); +const violationSchema = z + .object({ + code: violationCodeSchema, + clause: clauseIdSchema, + message: z.string(), + }) + .strict(); +const evaluationShape = { + scenario: z.enum(["off", "on"]), + canonicalVerdict: z.enum(["pass", "fail"]), + clauses: z.array(clauseSchema), + violations: z.array(violationSchema), +} as const; +const authoritySchema = z + .object({ + evidenceSource: z.string(), + collectionAttested: z.boolean(), + evaluation: z.string(), + evaluatorSourceSha256: sha256, + }) + .strict(); +const reportHeaderShape = { + schemaVersion: z.literal(REPORT_SCHEMA_VERSION), + contractFamily: z.literal(SUPPORTED_CONTRACT_FAMILY), + outcome: z.enum(["PASS", "BROKEN_PROMISE"]), +} as const; + +// Report parsing validates only the versioned public report structure. Values +// that are structurally valid but forged (for example, a valid-shaped PASS over +// broken evidence or a different valid-shaped evaluator fingerprint) proceed to +// full reproduction and become STALE_OR_MISMATCH. They are not mistaken for a +// malformed report. +const verifyReportSchema = z + .object({ + ...reportHeaderShape, + inputBinding: inputBindingSchema, + ...evaluationShape, + authority: authoritySchema, + }) + .strict(); +const gateReportSchema = z + .object({ + ...reportHeaderShape, + inputBindings: z + .object({ off: inputBindingSchema, on: inputBindingSchema }) + .strict(), + evaluations: z + .object({ + off: z.object(evaluationShape).strict(), + on: z.object(evaluationShape).strict(), + }) + .strict(), + authority: authoritySchema, + }) + .strict(); + +function invalid(detail: string): CheckResult { + return { status: "INVALID_REPORT_OR_EVIDENCE", detail }; +} + +function compare(regenerated: unknown, supplied: unknown): CheckResult { + if (canonicalizeJson(regenerated) === canonicalizeJson(supplied)) { + return { + status: "BOUND_AND_REPRODUCED", + detail: + "the report is reproduced exactly by re-running the unchanged evaluator on the supplied evidence", + }; + } + return { + status: "STALE_OR_MISMATCH", + detail: + "the supplied report does not match the report reproduced from the supplied evidence", + }; +} + +export async function checkSingle( + rawReport: unknown, + rawEvidence: unknown, +): Promise { + if (!verifyReportSchema.safeParse(rawReport).success) { + return invalid( + `report is not a schemaVersion ${REPORT_SCHEMA_VERSION} verification report`, + ); + } + const result = verifyBundle(rawEvidence); + if (result.outcome === "INVALID_EVIDENCE") { + return invalid( + "evidence is invalid; no canonical report can be reproduced from it", + ); + } + return compare(await createVerifyReport(result), rawReport); +} + +export async function checkGate( + rawReport: unknown, + rawOff: unknown, + rawOn: unknown, +): Promise { + if (!gateReportSchema.safeParse(rawReport).success) { + return invalid( + `report is not a schemaVersion ${REPORT_SCHEMA_VERSION} gate report`, + ); + } + const result = runGate(rawOff, rawOn); + if (result.outcome === "INVALID_EVIDENCE") { + return invalid( + "gate evidence is invalid; no canonical report can be reproduced from it", + ); + } + return compare(await createGateReport(result), rawReport); +} diff --git a/src/verify/cli.ts b/src/verify/cli.ts new file mode 100644 index 0000000..cc4ac46 --- /dev/null +++ b/src/verify/cli.ts @@ -0,0 +1,454 @@ +import { + lstat, + mkdir, + readFile, + stat, + writeFile, +} from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +import { + CHECK_EXIT, + checkGate, + checkSingle, +} from "./check.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, + producerTemplate, + scaffoldReadme, +} from "./examples.js"; +import { + EXIT_CODE, + type ExternalOutcome, +} from "./outcome.js"; +import { + createGateReport, + createVerifyReport, + serializeGateReportMarkdown, + serializeReportJson, + serializeVerifyReportMarkdown, +} from "./report.js"; +import { + externalBundleJsonSchema, + MAX_INPUT_BYTES, +} from "./schema.js"; +import { + gateValidationIssues, + runGate, + verifyBundle, +} from "./verify.js"; + +const HELP = `PromiseProof external evidence verifier + +Usage: + npm run promiseproof -- init --out + npm run promiseproof -- verify --evidence --out + npm run promiseproof -- gate --off --on --out + npm run promiseproof -- check --report --evidence + npm run promiseproof -- check --report --off --on + npm run promiseproof -- --help + +Exit codes (verify / gate): + 0 PASS + 1 usage, I/O, or unexpected execution error + 2 BROKEN_PROMISE + 3 INVALID_EVIDENCE + +Exit codes (check): the report is reproduced by re-running the unchanged +evaluator on the supplied evidence, not merely hash-compared. + 0 BOUND_AND_REPRODUCED + 1 usage, I/O, or unexpected execution error + 3 INVALID_REPORT_OR_EVIDENCE + 4 STALE_OR_MISMATCH + +Supported contract family: activity-personalization/v1 +`; + +const REPORT_FILENAMES = ["report.json", "report.md"] as const; +const SCAFFOLD_FILES = { + "activity-personalization.v1.schema.json": () => + serializeJson(externalBundleJsonSchema()), + "broken-off.example.json": () => serializeJson(brokenOffExample), + "passing-off.example.json": () => serializeJson(passingOffExample), + "passing-on.example.json": () => serializeJson(passingOnExample), + "producer-template.mjs": () => producerTemplate, + "README.md": () => scaffoldReadme, +} as const; + +class UsageError extends Error {} +class InvalidEvidenceFileError extends Error {} + +interface CliIo { + readonly stdout: (message: string) => void; + readonly stderr: (message: string) => void; +} + +interface ParsedOptions { + readonly [name: string]: string; +} + +function serializeJson(value: unknown): string { + return `${JSON.stringify(value, null, 2)}\n`; +} + +function parseOptions( + args: readonly string[], + allowed: readonly string[], +): ParsedOptions { + const options: Record = {}; + const allowedSet = new Set(allowed); + + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if ( + name === undefined || + value === undefined || + !name.startsWith("--") || + value.startsWith("--") + ) { + throw new UsageError("Options must be supplied as --name pairs."); + } + + const key = name.slice(2); + if (!allowedSet.has(key)) { + throw new UsageError(`Unknown option: ${name}`); + } + if (options[key] !== undefined) { + throw new UsageError(`Duplicate option: ${name}`); + } + options[key] = value; + } + + for (const required of allowed) { + if (options[required] === undefined) { + throw new UsageError(`Missing required option: --${required}`); + } + } + + return options; +} + +function safeOutputDirectory(rawPath: string): string { + if (rawPath.trim().length === 0) { + throw new UsageError("--out must not be empty."); + } + + const normalizedSegments = rawPath.replaceAll("\\", "/").split("/"); + if (normalizedSegments.includes("..")) { + throw new UsageError("--out must not contain path traversal."); + } + + const resolved = path.resolve(rawPath); + if (resolved === path.parse(resolved).root) { + throw new UsageError("--out must not be a filesystem root."); + } + return resolved; +} + +function safeChildPath(directory: string, filename: string): string { + if ( + path.basename(filename) !== filename || + filename === "." || + filename === ".." + ) { + throw new Error("Generated filename is unsafe."); + } + + const child = path.resolve(directory, filename); + if (path.dirname(child) !== directory) { + throw new Error("Generated path escaped the output directory."); + } + return child; +} + +async function ensureOutputDirectory(directory: string): Promise { + try { + const existing = await lstat(directory); + if (existing.isSymbolicLink() || !existing.isDirectory()) { + throw new Error("Output path must be a real directory."); + } + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + await mkdir(directory, { recursive: true }); + return; + } + throw error; + } +} + +async function refuseUnsafeReportTarget(target: string): Promise { + try { + const existing = await lstat(target); + if (existing.isSymbolicLink() || !existing.isFile()) { + throw new Error(`Refusing unsafe report target: ${path.basename(target)}`); + } + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + return; + } + throw error; + } +} + +async function writeReports( + outputDirectory: string, + json: string, + markdown: string, +): Promise { + await ensureOutputDirectory(outputDirectory); + const jsonPath = safeChildPath(outputDirectory, REPORT_FILENAMES[0]); + const markdownPath = safeChildPath(outputDirectory, REPORT_FILENAMES[1]); + await refuseUnsafeReportTarget(jsonPath); + await refuseUnsafeReportTarget(markdownPath); + await writeFile(jsonPath, json, "utf8"); + await writeFile(markdownPath, markdown, "utf8"); +} + +async function writeScaffold(outputDirectory: string): Promise { + await ensureOutputDirectory(outputDirectory); + const entries = Object.entries(SCAFFOLD_FILES); + const targets = entries.map(([filename]) => + safeChildPath(outputDirectory, filename), + ); + + for (const target of targets) { + try { + await lstat(target); + throw new Error( + `Refusing to overwrite existing scaffold file: ${path.basename(target)}`, + ); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ) { + continue; + } + throw error; + } + } + + const created: string[] = []; + try { + for (const [filename, content] of entries) { + const target = safeChildPath(outputDirectory, filename); + await writeFile(target, content(), { encoding: "utf8", flag: "wx" }); + created.push(target); + } + } catch (error) { + await Promise.all( + created.map(async (target) => { + const { unlink } = await import("node:fs/promises"); + await unlink(target); + }), + ); + throw error; + } +} + +async function readExternalJson(filename: string): Promise { + let metadata; + try { + metadata = await stat(filename); + } catch { + throw new Error("Unable to read evidence file."); + } + + if (!metadata.isFile()) { + throw new Error("Evidence path must identify a regular file."); + } + if (metadata.size > MAX_INPUT_BYTES) { + throw new InvalidEvidenceFileError( + `: input exceeds ${MAX_INPUT_BYTES}-byte limit`, + ); + } + + const bytes = await readFile(filename); + if (bytes.byteLength > MAX_INPUT_BYTES) { + throw new InvalidEvidenceFileError( + `: input exceeds ${MAX_INPUT_BYTES}-byte limit`, + ); + } + + try { + return JSON.parse(bytes.toString("utf8")) as unknown; + } catch { + throw new InvalidEvidenceFileError(": malformed JSON"); + } +} + +function printInvalid(io: CliIo, issues: readonly string[]): number { + io.stderr("INVALID_EVIDENCE"); + for (const issue of [...issues].sort()) { + io.stderr(`- ${issue}`); + } + return EXIT_CODE.INVALID_EVIDENCE; +} + +function printOutcome(io: CliIo, outcome: ExternalOutcome): void { + if (outcome === "PASS") { + io.stdout("PASS — report.json and report.md written."); + } else if (outcome === "BROKEN_PROMISE") { + io.stdout("BROKEN_PROMISE — report.json and report.md written."); + } +} + +async function runInit(args: readonly string[], io: CliIo): Promise { + const options = parseOptions(args, ["out"]); + await writeScaffold(safeOutputDirectory(options.out!)); + io.stdout("Initialized activity-personalization/v1 scaffold (6 files)."); + return EXIT_CODE.PASS; +} + +async function runVerify(args: readonly string[], io: CliIo): Promise { + const options = parseOptions(args, ["evidence", "out"]); + const raw = await readExternalJson(options.evidence!); + const result = verifyBundle(raw); + if (result.outcome === "INVALID_EVIDENCE") { + return printInvalid(io, result.issues); + } + + const report = await createVerifyReport(result); + await writeReports( + safeOutputDirectory(options.out!), + serializeReportJson(report), + serializeVerifyReportMarkdown(report), + ); + printOutcome(io, result.outcome); + return EXIT_CODE[result.outcome]; +} + +async function runGateCommand( + args: readonly string[], + io: CliIo, +): Promise { + const options = parseOptions(args, ["off", "on", "out"]); + const rawOff = await readExternalJson(options.off!); + const rawOn = await readExternalJson(options.on!); + const result = runGate(rawOff, rawOn); + if (result.outcome === "INVALID_EVIDENCE") { + return printInvalid(io, gateValidationIssues(result)); + } + + const report = await createGateReport(result); + await writeReports( + safeOutputDirectory(options.out!), + serializeReportJson(report), + serializeGateReportMarkdown(report), + ); + printOutcome(io, result.outcome); + return EXIT_CODE[result.outcome]; +} + +async function runCheck(args: readonly string[], io: CliIo): Promise { + const hasEvidence = args.includes("--evidence"); + const hasGate = args.includes("--off") || args.includes("--on"); + + try { + if (hasEvidence && !hasGate) { + const options = parseOptions(args, ["report", "evidence"]); + const [report, evidence] = await Promise.all([ + readExternalJson(options.report!), + readExternalJson(options.evidence!), + ]); + const result = await checkSingle(report, evidence); + io.stdout(`${result.status}: ${result.detail}`); + return CHECK_EXIT[result.status]; + } + + if (hasGate && !hasEvidence) { + const options = parseOptions(args, ["report", "off", "on"]); + const [report, off, on] = await Promise.all([ + readExternalJson(options.report!), + readExternalJson(options.off!), + readExternalJson(options.on!), + ]); + const result = await checkGate(report, off, on); + io.stdout(`${result.status}: ${result.detail}`); + return CHECK_EXIT[result.status]; + } + } catch (error) { + if (error instanceof InvalidEvidenceFileError) { + io.stderr("INVALID_REPORT_OR_EVIDENCE"); + io.stderr(`- ${error.message}`); + return CHECK_EXIT.INVALID_REPORT_OR_EVIDENCE; + } + throw error; + } + + throw new UsageError( + "check requires --report with either --evidence, or both --off and --on.", + ); +} + +export async function runCli( + args: readonly string[], + io: CliIo = { + stdout: (message) => console.log(message), + stderr: (message) => console.error(message), + }, +): Promise { + if (args.length === 0) { + io.stderr(HELP); + return EXIT_CODE.EXECUTION_ERROR; + } + if (args[0] === "--help" || args[0] === "-h") { + io.stdout(HELP); + return EXIT_CODE.PASS; + } + + const [command, ...commandArgs] = args; + try { + if (command === "init") { + return await runInit(commandArgs, io); + } + if (command === "verify") { + return await runVerify(commandArgs, io); + } + if (command === "gate") { + return await runGateCommand(commandArgs, io); + } + if (command === "check") { + return await runCheck(commandArgs, io); + } + throw new UsageError(`Unknown command: ${String(command)}`); + } catch (error) { + if (error instanceof InvalidEvidenceFileError) { + return printInvalid(io, [error.message]); + } + if (error instanceof UsageError) { + io.stderr(`USAGE_ERROR: ${error.message}`); + io.stderr("Run npm run promiseproof -- --help for usage."); + return EXIT_CODE.EXECUTION_ERROR; + } + + io.stderr( + `EXECUTION_ERROR: ${error instanceof Error ? error.message : String(error)}`, + ); + return EXIT_CODE.EXECUTION_ERROR; + } +} + +const invokedPath = process.argv[1] + ? path.resolve(process.argv[1]) + : undefined; +if (invokedPath === fileURLToPath(import.meta.url)) { + process.exitCode = await runCli(process.argv.slice(2)); +} diff --git a/src/verify/examples.ts b/src/verify/examples.ts new file mode 100644 index 0000000..70839cc --- /dev/null +++ b/src/verify/examples.ts @@ -0,0 +1,164 @@ +import type { + ExternalActivity, + ExternalBundle, + ExternalEvidenceV1, +} from "./schema.js"; +import { + SUPPORTED_CONTRACT_FAMILY, + SUPPORTED_SCHEMA_VERSION, +} from "./outcome.js"; + +const SUBJECT_ID = "article-atlas-reader-001"; +const ITEM_ID = "article-atlas-story-101"; +const TIME_ACTIVITY = "2026-01-15T12:00:01.000Z"; + +function activity(runId: string): ExternalActivity { + return { + runId, + subjectId: SUBJECT_ID, + eventType: "page_view", + itemId: "article-atlas-origin-001", + clientSequence: 1, + occurredAt: TIME_ACTIVITY, + }; +} + +function bundle(evidence: ExternalEvidenceV1): ExternalBundle { + return { + schemaVersion: SUPPORTED_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + evidence, + }; +} + +function offEvidence(includeActivity: boolean): ExternalEvidenceV1 { + const runId = includeActivity + ? "article-atlas-broken-off" + : "article-atlas-passing-off"; + const observedActivity = activity(runId); + + return { + scenario: "off", + subjectId: SUBJECT_ID, + control: { + uiPreference: "off", + toggleChecked: false, + storedPreference: "off", + backendPreference: "off", + reloadObserved: true, + }, + activity: { + capturedActivities: includeActivity ? [{ ...observedActivity }] : [], + recommendationServiceReceipts: includeActivity + ? [{ ...observedActivity }] + : [], + }, + recommendations: { + feedFunctional: true, + renderedSource: "contextual", + renderedItemIds: [ITEM_ID], + recommendationServiceReceipts: [ + { + source: "contextual", + itemIds: [ITEM_ID], + }, + ], + }, + }; +} + +function onEvidence(): ExternalEvidenceV1 { + const observedActivity = activity("article-atlas-passing-on"); + + return { + scenario: "on", + subjectId: SUBJECT_ID, + control: { + uiPreference: "on", + toggleChecked: true, + storedPreference: "on", + backendPreference: "on", + reloadObserved: true, + }, + activity: { + capturedActivities: [{ ...observedActivity }], + recommendationServiceReceipts: [{ ...observedActivity }], + }, + recommendations: { + feedFunctional: true, + renderedSource: "behavioral", + renderedItemIds: [ITEM_ID], + recommendationServiceReceipts: [ + { + source: "behavioral", + subjectId: SUBJECT_ID, + itemIds: [ITEM_ID], + }, + ], + }, + }; +} + +export const brokenOffExample = bundle(offEvidence(true)); +export const passingOffExample = bundle(offEvidence(false)); +export const passingOnExample = bundle(onEvidence()); + +export const producerTemplate = `import { readFile, writeFile } from "node:fs/promises"; + +const [evidenceFile = "external-evidence.json", outputFile = "bundle.json"] = + process.argv.slice(2); +const evidence = JSON.parse(await readFile(evidenceFile, "utf8")); +const bundle = { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence, +}; + +await writeFile( + outputFile, + \`\${JSON.stringify(bundle, null, 2)}\\n\`, + { encoding: "utf8", flag: "wx" }, +); +`; + +export const scaffoldReadme = `# PromiseProof external evidence scaffold + +This scaffold supports exactly one contract family: +\`activity-personalization/v1\`. + +The public \`ExternalEvidenceV1\` contract contains only facts used by the +unchanged PromiseProof evaluator: + +- scenario and subject identifier; +- UI, toggle, storage, backend, and reload control state; +- captured activities and recommendation-service activity receipts; +- rendered feed state and recommendation-service recommendation receipts. + +Use \`broken-off.example.json\`, \`passing-off.example.json\`, and +\`passing-on.example.json\` as fixed-shape examples. \`producer-template.mjs\` +wraps an \`ExternalEvidenceV1\` JSON object; it does not collect or attest +evidence. Then verify one bundle: + +\`\`\`text +npm run promiseproof -- verify --evidence --out +\`\`\` + +Or gate OFF and ON together: + +\`\`\`text +npm run promiseproof -- gate --off --on --out +\`\`\` + +The OFF contract checks zero identifiable recommendation activity, a functional +contextual feed, and witnessed OFF persistence across UI, toggle, storage, and +backend state. The ON control checks correlated identifiable activity and a +functional behavioral feed. + +Evidence is externally supplied. PromiseProof does not attest how it was +collected. The deterministic PromiseProof evaluator alone evaluates a bundle +that passes strict validation. + +These examples prove externally supplied evidence ingestion for one contract +family. They do not prove arbitrary evidence collection or arbitrary promise +support. Article Atlas is a neutral synthetic example product. +`; diff --git a/src/verify/outcome.ts b/src/verify/outcome.ts new file mode 100644 index 0000000..048ec95 --- /dev/null +++ b/src/verify/outcome.ts @@ -0,0 +1,17 @@ +export type ExternalOutcome = + | "PASS" + | "BROKEN_PROMISE" + | "INVALID_EVIDENCE" + | "EXECUTION_ERROR"; + +export const EXIT_CODE = { + PASS: 0, + EXECUTION_ERROR: 1, + BROKEN_PROMISE: 2, + INVALID_EVIDENCE: 3, +} as const satisfies Record; + +export const SUPPORTED_SCHEMA_VERSION = "1" as const; +export const SUPPORTED_CONTRACT_FAMILY = + "activity-personalization/v1" as const; +export const REPORT_SCHEMA_VERSION = "2" as const; diff --git a/src/verify/report.ts b/src/verify/report.ts new file mode 100644 index 0000000..fa1e5f3 --- /dev/null +++ b/src/verify/report.ts @@ -0,0 +1,235 @@ +import type { + PromiseClauseResult, + PromiseEvaluation, + PromiseViolation, +} from "../shared/types.js"; +import { + bindCanonicalJson, + CANONICAL_JSON_ID, + EVALUATOR_SOURCE_SHA256, + SHA256_ALGORITHM, + type InputBinding, +} from "./binding.js"; +import { + REPORT_SCHEMA_VERSION, + SUPPORTED_CONTRACT_FAMILY, +} from "./outcome.js"; +import type { GateResult, VerifyResult } from "./verify.js"; + +const authority = { + evidenceSource: "externally-supplied", + collectionAttested: false, + evaluation: "deterministic-promiseproof-evaluator", + evaluatorSourceSha256: EVALUATOR_SOURCE_SHA256, +} as const; + +export interface EvaluationReport { + readonly scenario: "on" | "off"; + readonly canonicalVerdict: "pass" | "fail"; + readonly clauses: readonly PromiseClauseResult[]; + readonly violations: readonly PromiseViolation[]; +} + +export interface VerifyReport extends EvaluationReport { + readonly schemaVersion: typeof REPORT_SCHEMA_VERSION; + readonly contractFamily: typeof SUPPORTED_CONTRACT_FAMILY; + readonly outcome: "PASS" | "BROKEN_PROMISE"; + readonly inputBinding: InputBinding; + readonly authority: typeof authority; +} + +export interface GateReport { + readonly schemaVersion: typeof REPORT_SCHEMA_VERSION; + readonly contractFamily: typeof SUPPORTED_CONTRACT_FAMILY; + readonly outcome: "PASS" | "BROKEN_PROMISE"; + readonly inputBindings: { + readonly off: InputBinding; + readonly on: InputBinding; + }; + readonly evaluations: { + readonly off: EvaluationReport; + readonly on: EvaluationReport; + }; + readonly authority: typeof authority; +} + +function requireEvaluation(result: VerifyResult): PromiseEvaluation { + if (result.evaluation === null) { + throw new Error("Cannot render a canonical report for invalid evidence."); + } + return result.evaluation; +} + +function requireValidatedCanonicalJson(result: VerifyResult): string { + if (result.validatedCanonicalJson === null) { + throw new Error("Cannot bind a report without validated evidence."); + } + return result.validatedCanonicalJson; +} + +function evaluationReport(result: VerifyResult): EvaluationReport { + const evaluation = requireEvaluation(result); + if (result.scenario === null) { + throw new Error("Evaluated evidence is missing its scenario."); + } + + return { + scenario: result.scenario, + canonicalVerdict: evaluation.verdict, + clauses: evaluation.clauses.map((clause) => ({ ...clause })), + violations: evaluation.violations.map((violation) => ({ ...violation })), + }; +} + +export async function createVerifyReport( + result: VerifyResult, +): Promise { + if ( + result.outcome !== "PASS" && + result.outcome !== "BROKEN_PROMISE" + ) { + throw new Error("Invalid evidence cannot be rendered as a product verdict."); + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + outcome: result.outcome, + inputBinding: await bindCanonicalJson( + requireValidatedCanonicalJson(result), + ), + ...evaluationReport(result), + authority, + }; +} + +export async function createGateReport(result: GateResult): Promise { + if ( + result.outcome !== "PASS" && + result.outcome !== "BROKEN_PROMISE" + ) { + throw new Error("Invalid gate input cannot be rendered as a product verdict."); + } + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + contractFamily: SUPPORTED_CONTRACT_FAMILY, + outcome: result.outcome, + inputBindings: { + off: await bindCanonicalJson(requireValidatedCanonicalJson(result.off)), + on: await bindCanonicalJson(requireValidatedCanonicalJson(result.on)), + }, + evaluations: { + off: evaluationReport(result.off), + on: evaluationReport(result.on), + }, + authority, + }; +} + +export function serializeReportJson( + report: VerifyReport | GateReport, +): string { + return `${JSON.stringify(report, null, 2)}\n`; +} + +function markdownInline(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("`", "`") + .replaceAll("\\", "\\\\") + .replaceAll("|", "\\|") + .replaceAll("\r\n", " ") + .replaceAll("\n", " ") + .replaceAll("\r", " "); +} + +function evaluationMarkdown( + heading: string, + report: EvaluationReport, +): string[] { + const lines = [ + `## ${heading}`, + "", + `Scenario: ${report.scenario.toUpperCase()}`, + `Canonical verdict: ${report.canonicalVerdict}`, + "", + "| Clause | Result | Expected | Observed |", + "| --- | --- | --- | --- |", + ...report.clauses.map( + (clause) => + `| ${clause.id} | ${clause.passed ? "PASS" : "FAIL"} | ${markdownInline(clause.expected)} | ${markdownInline(clause.observed)} |`, + ), + "", + "### Violations", + "", + ]; + + if (report.violations.length === 0) { + lines.push("None."); + } else { + lines.push( + ...report.violations.map( + (violation) => + `- ${violation.code} (${violation.clause}): ${markdownInline(violation.message)}`, + ), + ); + } + + return lines; +} + +function authorityMarkdown(): string[] { + return [ + "## Authority", + "", + "Evidence source: externally supplied", + "Collection integrity: not attested by PromiseProof", + "Evaluation authority: deterministic PromiseProof evaluator", + `Evaluator source SHA-256 (canonical UTF-8/LF): ${EVALUATOR_SOURCE_SHA256}`, + ]; +} + +function inputBindingMarkdown(label: string, binding: InputBinding): string[] { + return [ + `${label} canonicalization: ${CANONICAL_JSON_ID}`, + `${label} digest algorithm: ${SHA256_ALGORITHM}`, + `${label} evidence SHA-256: ${binding.sha256}`, + ]; +} + +export function serializeVerifyReportMarkdown(report: VerifyReport): string { + return [ + "# PromiseProof verification report", + "", + `Contract family: ${report.contractFamily}`, + `Outcome: ${report.outcome}`, + "", + ...inputBindingMarkdown("Input", report.inputBinding), + "", + ...evaluationMarkdown("Evaluation", report), + "", + ...authorityMarkdown(), + "", + ].join("\n"); +} +export function serializeGateReportMarkdown(report: GateReport): string { + return [ + "# PromiseProof OFF + ON gate report", + "", + `Contract family: ${report.contractFamily}`, + `Outcome: ${report.outcome}`, + "", + ...inputBindingMarkdown("OFF input", report.inputBindings.off), + ...inputBindingMarkdown("ON input", report.inputBindings.on), + "", + ...evaluationMarkdown("OFF evaluation", report.evaluations.off), + "", + ...evaluationMarkdown("ON evaluation", report.evaluations.on), + "", + ...authorityMarkdown(), + "", + ].join("\n"); +} diff --git a/src/verify/schema.ts b/src/verify/schema.ts new file mode 100644 index 0000000..7feb5a6 --- /dev/null +++ b/src/verify/schema.ts @@ -0,0 +1,108 @@ +import { z } from "zod"; + +import { + SUPPORTED_CONTRACT_FAMILY, + SUPPORTED_SCHEMA_VERSION, +} from "./outcome.js"; + +export const MAX_INPUT_BYTES = 256 * 1024; +export const MAX_COLLECTION_ITEMS = 100; + +const SAFE_VISIBLE_CHARACTERS = + /^[^\u0000-\u001f\u007f\u202a-\u202e\u2066-\u2069]*$/u; + +const visibleIdentifier = z + .string() + .min(1) + .max(256) + .regex(/\S/, "Identifier must contain a non-whitespace character") + .regex( + SAFE_VISIBLE_CHARACTERS, + "Developer-visible text contains a forbidden control or bidi character", + ); +const isoTimestamp = z.iso.datetime({ offset: true }); +const sequence = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const preference = z.enum(["on", "off"]); +const recommendationSource = z.enum(["contextual", "behavioral"]); + +function boundedArray(schema: T) { + return z.array(schema).max(MAX_COLLECTION_ITEMS); +} + +export const externalActivitySchema = z + .object({ + runId: visibleIdentifier, + subjectId: visibleIdentifier, + eventType: z.literal("page_view"), + itemId: visibleIdentifier, + clientSequence: sequence, + occurredAt: isoTimestamp, + }) + .strict(); + +export const externalRecommendationReceiptSchema = z + .object({ + source: recommendationSource, + subjectId: visibleIdentifier.optional(), + itemIds: boundedArray(visibleIdentifier), + }) + .strict(); + +export const externalEvidenceV1Schema = z + .object({ + scenario: preference, + subjectId: visibleIdentifier, + control: z + .object({ + uiPreference: preference, + toggleChecked: z.boolean(), + storedPreference: preference.nullable(), + backendPreference: preference, + reloadObserved: z.boolean(), + }) + .strict(), + activity: z + .object({ + capturedActivities: boundedArray(externalActivitySchema), + recommendationServiceReceipts: boundedArray(externalActivitySchema), + }) + .strict(), + recommendations: z + .object({ + feedFunctional: z.boolean(), + renderedSource: recommendationSource, + renderedItemIds: boundedArray(visibleIdentifier), + recommendationServiceReceipts: boundedArray( + externalRecommendationReceiptSchema, + ), + }) + .strict(), + }) + .strict(); + +export const externalBundleSchema = z + .object({ + schemaVersion: z.literal(SUPPORTED_SCHEMA_VERSION), + contractFamily: z.literal(SUPPORTED_CONTRACT_FAMILY), + evidence: externalEvidenceV1Schema, + }) + .strict() + .meta({ + id: "https://promiseproof.local/schemas/activity-personalization.v1.schema.json", + title: "PromiseProof activity-personalization/v1 evidence bundle", + description: + "Externally supplied evaluator-relevant facts for the single activity-personalization/v1 contract family.", + }); + +export type ExternalActivity = z.infer; +export type ExternalRecommendationReceipt = z.infer< + typeof externalRecommendationReceiptSchema +>; +export type ExternalEvidenceV1 = z.infer; +export type ExternalBundle = z.infer; + +export function externalBundleJsonSchema(): z.core.JSONSchema.JSONSchema { + return z.toJSONSchema(externalBundleSchema, { + target: "draft-2020-12", + }); +} diff --git a/src/verify/verify.ts b/src/verify/verify.ts new file mode 100644 index 0000000..930bcc3 --- /dev/null +++ b/src/verify/verify.ts @@ -0,0 +1,185 @@ +import { evaluatePromise } from "../shared/evaluator.js"; +import type { + PromiseEvaluation, + PromiseEvidence, +} from "../shared/types.js"; +import { adaptExternalEvidence } from "./adapter.js"; +import { canonicalizeJson } from "./binding.js"; +import { + SUPPORTED_CONTRACT_FAMILY, + type ExternalOutcome, +} from "./outcome.js"; +import { + externalBundleSchema, + type ExternalBundle, +} from "./schema.js"; + +type CanonicalEvaluator = (evidence: PromiseEvidence) => PromiseEvaluation; + +export interface VerifyResult { + readonly outcome: Exclude; + readonly contractFamily: string; + readonly scenario: "on" | "off" | null; + readonly evaluation: PromiseEvaluation | null; + readonly validatedCanonicalJson: string | null; + readonly issues: readonly string[]; +} + +interface ValidatedBundle { + readonly success: true; + readonly bundle: ExternalBundle; +} + +interface RejectedBundle { + readonly success: false; + readonly issues: readonly string[]; +} + +type BundleValidation = ValidatedBundle | RejectedBundle; + +function issuePath(path: PropertyKey[]): string { + if (path.length === 0) { + return ""; + } + + return path.map((part) => String(part)).join("."); +} + +export function validateExternalBundle(raw: unknown): BundleValidation { + const parsed = externalBundleSchema.safeParse(raw); + if (!parsed.success) { + return { + success: false, + issues: parsed.error.issues + .map((issue) => `${issuePath(issue.path)}: ${issue.message}`) + .sort(), + }; + } + + return { + success: true, + bundle: parsed.data, + }; +} + +function rejectedResult(validation: BundleValidation): VerifyResult { + return { + outcome: "INVALID_EVIDENCE", + contractFamily: SUPPORTED_CONTRACT_FAMILY, + scenario: validation.success ? validation.bundle.evidence.scenario : null, + evaluation: null, + validatedCanonicalJson: null, + issues: [...(validation.success ? [] : validation.issues)].sort(), + }; +} + +function evaluateValidatedBundle( + validated: ValidatedBundle, + evaluator: CanonicalEvaluator, +): VerifyResult { + const validatedCanonicalJson = canonicalizeJson(validated.bundle); + const evidence = adaptExternalEvidence(validated.bundle.evidence); + const evaluation = evaluator(evidence); + return { + outcome: evaluation.verdict === "pass" ? "PASS" : "BROKEN_PROMISE", + contractFamily: validated.bundle.contractFamily, + scenario: validated.bundle.evidence.scenario, + evaluation, + validatedCanonicalJson, + issues: [], + }; +} + +function verifyBundleWithEvaluator( + raw: unknown, + evaluator: CanonicalEvaluator, +): VerifyResult { + const validated = validateExternalBundle(raw); + if (!validated.success) { + return rejectedResult(validated); + } + return evaluateValidatedBundle(validated, evaluator); +} + +export function verifyBundle(raw: unknown): VerifyResult { + return verifyBundleWithEvaluator(raw, evaluatePromise); +} + +export interface GateResult { + readonly outcome: Exclude; + readonly contractFamily: string; + readonly off: VerifyResult; + readonly on: VerifyResult; + readonly issues: readonly string[]; +} + +function runGateWithEvaluator( + rawOff: unknown, + rawOn: unknown, + evaluator: CanonicalEvaluator, +): GateResult { + const validatedOff = validateExternalBundle(rawOff); + const validatedOn = validateExternalBundle(rawOn); + const gateIssues: string[] = []; + const offSlotIssues: string[] = []; + const onSlotIssues: string[] = []; + + if ( + validatedOff.success && + validatedOff.bundle.evidence.scenario !== "off" + ) { + offSlotIssues.push("--off bundle must contain OFF-scenario evidence"); + } + if ( + validatedOn.success && + validatedOn.bundle.evidence.scenario !== "on" + ) { + onSlotIssues.push("--on bundle must contain ON-scenario evidence"); + } + + gateIssues.push(...offSlotIssues, ...onSlotIssues); + if ( + !validatedOff.success || + !validatedOn.success || + gateIssues.length > 0 + ) { + const off = rejectedResult(validatedOff); + const on = rejectedResult(validatedOn); + return { + outcome: "INVALID_EVIDENCE", + contractFamily: SUPPORTED_CONTRACT_FAMILY, + off, + on, + issues: [ + ...gateIssues, + ...off.issues.map((issue) => `off.${issue}`), + ...on.issues.map((issue) => `on.${issue}`), + ].sort(), + }; + } + + const off = evaluateValidatedBundle(validatedOff, evaluator); + const on = evaluateValidatedBundle(validatedOn, evaluator); + return { + outcome: + off.outcome === "PASS" && on.outcome === "PASS" + ? "PASS" + : "BROKEN_PROMISE", + contractFamily: SUPPORTED_CONTRACT_FAMILY, + off, + on, + issues: [], + }; +} + +export function runGate(rawOff: unknown, rawOn: unknown): GateResult { + return runGateWithEvaluator(rawOff, rawOn, evaluatePromise); +} + +export const verifierTestHooks = Object.freeze({ + runGateWithEvaluator, +}); + +export function gateValidationIssues(result: GateResult): string[] { + return [...result.issues]; +} diff --git a/tests/verify/canonical-projection.ts b/tests/verify/canonical-projection.ts new file mode 100644 index 0000000..d36624d --- /dev/null +++ b/tests/verify/canonical-projection.ts @@ -0,0 +1,226 @@ +import type { + ActivityPayload, + PromiseEvidence, + RecommendationReceipt, +} from "../../src/shared/types.js"; +import type { ExternalEvidenceV1 } from "../../src/verify/schema.js"; + +const USER_ID = "canonical-reader-001"; +const ITEM_ID = "canonical-item-001"; +const ACTIVITY_TIME = "2026-01-15T12:00:01.000Z"; +const RECEIPT_TIME = "2026-01-15T12:00:02.000Z"; + +function activity(runId: string): ActivityPayload { + return { + runId, + userId: USER_ID, + eventType: "page_view", + itemId: "canonical-origin-001", + clientSequence: 1, + occurredAt: ACTIVITY_TIME, + }; +} + +function recommendationReceipt( + source: "contextual" | "behavioral", + userId: string | undefined, + itemIds: string[], +): RecommendationReceipt { + return { + kind: "recommendation", + receiptId: `canonical-${source}-receipt`, + sequence: 2, + receivedAt: RECEIPT_TIME, + source, + ...(userId === undefined ? {} : { userId }), + items: itemIds.map((id) => ({ + id, + title: "Canonical fixture item", + description: "Canonical fixture description", + eyebrow: "Canonical fixture", + })), + }; +} + +function canonicalOff(runId: string): PromiseEvidence { + return { + scenario: "off", + runId, + userId: USER_ID, + ui: { + preference: "off", + toggleChecked: false, + feedFunctional: true, + }, + storage: { preference: "off" }, + request: { + activityPayloads: [], + preferenceUpdates: [], + }, + response: { preferenceUpdates: [] }, + backend: { + preference: "off", + activityReceipts: [], + recommendationReceipts: [ + recommendationReceipt("contextual", undefined, [ITEM_ID]), + ], + preferenceReceipts: [], + }, + recommendation: { + source: "contextual", + itemIds: [ITEM_ID], + }, + timestamps: { + clientTimeline: [], + activityReceivedAt: [], + preferenceReceivedAt: [], + recommendationReceivedAt: [], + }, + journey: { reloadObserved: true }, + }; +} + +function canonicalOn(runId: string): PromiseEvidence { + const observedActivity = activity(runId); + return { + scenario: "on", + runId, + userId: USER_ID, + ui: { + preference: "on", + toggleChecked: true, + feedFunctional: true, + }, + storage: { preference: "on" }, + request: { + activityPayloads: [observedActivity], + preferenceUpdates: [], + }, + response: { preferenceUpdates: [] }, + backend: { + preference: "on", + activityReceipts: [ + { + kind: "activity", + service: "recommendation", + receiptId: "canonical-activity-receipt", + sequence: 1, + receivedAt: RECEIPT_TIME, + payload: observedActivity, + }, + ], + recommendationReceipts: [ + recommendationReceipt("behavioral", USER_ID, [ITEM_ID]), + ], + preferenceReceipts: [], + }, + recommendation: { + source: "behavioral", + itemIds: [ITEM_ID], + }, + timestamps: { + clientTimeline: [], + activityReceivedAt: [RECEIPT_TIME], + preferenceReceivedAt: [], + recommendationReceivedAt: [RECEIPT_TIME], + }, + journey: { reloadObserved: true }, + }; +} + +const passingOff = canonicalOff("canonical-passing-off"); +const initializationRace = structuredClone(passingOff); +initializationRace.runId = "canonical-initialization-race"; +const leakedActivity = activity(initializationRace.runId); +initializationRace.request.activityPayloads = [leakedActivity]; +initializationRace.backend.activityReceipts = [ + { + kind: "activity", + service: "recommendation", + receiptId: "canonical-leak-receipt", + sequence: 1, + receivedAt: RECEIPT_TIME, + payload: leakedActivity, + }, +]; + +const propagationFailure = structuredClone(passingOff); +propagationFailure.runId = "canonical-propagation-failure"; +propagationFailure.backend.preference = "on"; + +const passingOn = canonicalOn("canonical-passing-on"); +const brokenOnMissingActivity = structuredClone(passingOn); +brokenOnMissingActivity.runId = "canonical-broken-on"; +brokenOnMissingActivity.backend.activityReceipts = []; + +const contextualFeedFailure = structuredClone(passingOff); +contextualFeedFailure.runId = "canonical-contextual-feed-failure"; +contextualFeedFailure.backend.recommendationReceipts = []; + +const behavioralFeedFailure = structuredClone(passingOn); +behavioralFeedFailure.runId = "canonical-behavioral-feed-failure"; +behavioralFeedFailure.backend.recommendationReceipts = []; + +const reloadPersistenceFailure = structuredClone(passingOff); +reloadPersistenceFailure.runId = "canonical-reload-failure"; +reloadPersistenceFailure.journey.reloadObserved = false; + +export const canonicalCases = { + initializationRace, + passingOff, + propagationFailure, + passingOn, + brokenOnMissingActivity, + contextualFeedFailure, + behavioralFeedFailure, + reloadPersistenceFailure, +} as const; + +export function projectCanonicalEvidence( + evidence: PromiseEvidence, +): ExternalEvidenceV1 { + return { + scenario: evidence.scenario, + subjectId: evidence.userId, + control: { + uiPreference: evidence.ui.preference, + toggleChecked: evidence.ui.toggleChecked, + storedPreference: evidence.storage.preference, + backendPreference: evidence.backend.preference, + reloadObserved: evidence.journey.reloadObserved, + }, + activity: { + capturedActivities: evidence.request.activityPayloads.map((payload) => ({ + runId: payload.runId, + subjectId: payload.userId, + eventType: payload.eventType, + itemId: payload.itemId, + clientSequence: payload.clientSequence, + occurredAt: payload.occurredAt, + })), + recommendationServiceReceipts: evidence.backend.activityReceipts.map( + (receipt) => ({ + runId: receipt.payload.runId, + subjectId: receipt.payload.userId, + eventType: receipt.payload.eventType, + itemId: receipt.payload.itemId, + clientSequence: receipt.payload.clientSequence, + occurredAt: receipt.payload.occurredAt, + }), + ), + }, + recommendations: { + feedFunctional: evidence.ui.feedFunctional, + renderedSource: evidence.recommendation.source, + renderedItemIds: [...evidence.recommendation.itemIds], + recommendationServiceReceipts: + evidence.backend.recommendationReceipts.map((receipt) => ({ + source: receipt.source, + ...(receipt.userId === undefined + ? {} + : { subjectId: receipt.userId }), + itemIds: receipt.items.map((item) => item.id), + })), + }, + }; +} diff --git a/tests/verify/check.unit.ts b/tests/verify/check.unit.ts new file mode 100644 index 0000000..f619767 --- /dev/null +++ b/tests/verify/check.unit.ts @@ -0,0 +1,231 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { checkGate, checkSingle } from "../../src/verify/check.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, +} from "../../src/verify/examples.js"; +import { createGateReport, createVerifyReport } from "../../src/verify/report.js"; +import { runGate, verifyBundle } from "../../src/verify/verify.js"; + +// A genuine report is one produced by the real pipeline for the given evidence. +async function genuineSingle(bundle: unknown): Promise { + return createVerifyReport(verifyBundle(bundle)); +} +async function genuineGate(off: unknown, on: unknown): Promise { + return createGateReport(runGate(off, on)); +} +// Deep clone so a mutation never touches the shared example or the source report. +function clone(value: unknown): any { + return JSON.parse(JSON.stringify(value)); +} + +test("a genuine report reproduces from its own evidence", async () => { + const report = await genuineSingle(passingOffExample); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "BOUND_AND_REPRODUCED", + ); +}); + +test("a reformatted/round-tripped report still reproduces", async () => { + const report = clone(await genuineSingle(passingOffExample)); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "BOUND_AND_REPRODUCED", + ); +}); + +test("a report checked against different evidence does not reproduce", async () => { + const report = await genuineSingle(passingOffExample); + assert.equal( + (await checkSingle(report, passingOnExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a fabricated PASS over broken evidence is caught (hashes alone would miss it)", async () => { + // Same correct evidence digest and evaluator fingerprint, but an invented verdict. + const report = clone(await genuineSingle(brokenOffExample)); + report.outcome = "PASS"; + report.canonicalVerdict = "pass"; + report.violations = []; + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a tampered evidence digest is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.inputBinding.sha256 = "0".repeat(64); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a tampered evaluator fingerprint is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.authority.evaluatorSourceSha256 = "0".repeat(64); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("an altered non-attestation disclosure is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.authority.collectionAttested = true; + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a removed violation is caught", async () => { + const report = clone(await genuineSingle(brokenOffExample)); + report.violations = []; + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("an added violation is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.violations = [ + { + code: "PP_IDENTIFIABLE_EVENT_LEAK", + clause: "no_identifiable_activity", + message: "fabricated", + }, + ]; + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a changed clause observed value is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.clauses[0].observed = "tampered observation"; + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a reordered clauses array is caught", async () => { + const report = clone(await genuineSingle(passingOffExample)); + report.clauses.reverse(); + assert.equal( + (await checkSingle(report, passingOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("a document that is not a v2 report is INVALID_REPORT_OR_EVIDENCE", async () => { + assert.equal( + (await checkSingle({ hello: "world" }, passingOffExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); +}); + +test("single and gate checks reject the other report kind", async () => { + const single = await genuineSingle(passingOffExample); + const gate = await genuineGate(passingOffExample, passingOnExample); + + assert.equal( + (await checkSingle(gate, passingOffExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); + assert.equal( + (await checkGate(single, passingOffExample, passingOnExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); +}); + +test("strict report validation rejects missing, malformed, and unknown fields", async () => { + const genuine = clone(await genuineSingle(brokenOffExample)); + const cases: unknown[] = []; + + const missingAuthority = clone(genuine); + delete missingAuthority.authority; + cases.push(missingAuthority); + + const malformedClause = clone(genuine); + malformedClause.clauses[0].passed = "yes"; + cases.push(malformedClause); + + const malformedViolation = clone(genuine); + malformedViolation.violations[0].code = "PP_NOT_A_REAL_VIOLATION"; + cases.push(malformedViolation); + + const unknownField = clone(genuine); + unknownField.unexpected = true; + cases.push(unknownField); + + const wrongOutcome = clone(genuine); + wrongOutcome.outcome = "INCONCLUSIVE"; + cases.push(wrongOutcome); + + const invalidDigest = clone(genuine); + invalidDigest.inputBinding.sha256 = "not-a-sha256"; + cases.push(invalidDigest); + + const wrongVersion = clone(genuine); + wrongVersion.schemaVersion = "1"; + cases.push(wrongVersion); + + for (const report of cases) { + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "INVALID_REPORT_OR_EVIDENCE", + ); + } +}); + +test("valid-shaped semantic forgeries remain STALE_OR_MISMATCH", async () => { + const report = clone(await genuineSingle(brokenOffExample)); + report.outcome = "PASS"; + report.canonicalVerdict = "pass"; + report.authority.evaluatorSourceSha256 = "0".repeat(64); + report.authority.collectionAttested = true; + + assert.equal( + (await checkSingle(report, brokenOffExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("invalid evidence is INVALID_REPORT_OR_EVIDENCE", async () => { + const report = await genuineSingle(passingOffExample); + assert.equal( + (await checkSingle(report, { not: "a bundle" })).status, + "INVALID_REPORT_OR_EVIDENCE", + ); +}); + +test("a genuine gate report reproduces", async () => { + const report = await genuineGate(passingOffExample, passingOnExample); + assert.equal( + (await checkGate(report, passingOffExample, passingOnExample)).status, + "BOUND_AND_REPRODUCED", + ); +}); + +test("a gate report checked against a changed side does not reproduce", async () => { + const report = await genuineGate(passingOffExample, passingOnExample); + assert.equal( + (await checkGate(report, brokenOffExample, passingOnExample)).status, + "STALE_OR_MISMATCH", + ); +}); + +test("gate OFF and ON evidence digests are distinct", async () => { + const report = clone(await genuineGate(passingOffExample, passingOnExample)); + assert.notEqual(report.inputBindings.off.sha256, report.inputBindings.on.sha256); +}); diff --git a/tests/verify/cli.unit.ts b/tests/verify/cli.unit.ts new file mode 100644 index 0000000..384b8f7 --- /dev/null +++ b/tests/verify/cli.unit.ts @@ -0,0 +1,489 @@ +import assert from "node:assert/strict"; +import { + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import test, { after } from "node:test"; + +import { + brokenOffExample, + passingOffExample, + passingOnExample, +} from "../../src/verify/examples.js"; +import { bindExternalBundle } from "../../src/verify/binding.js"; +import { MAX_INPUT_BYTES } from "../../src/verify/schema.js"; + +const projectRoot = process.cwd(); +const temporaryRoots: string[] = []; + +interface CliResult { + readonly status: number | null; + readonly stdout: string; + readonly stderr: string; +} + +function clone(value: T): T { + return structuredClone(value); +} + +async function temporaryRoot(): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), "promiseproof-cli-test-")); + temporaryRoots.push(root); + return root; +} + +async function writeJson(filename: string, value: unknown): Promise { + await writeFile(filename, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function runCli(args: readonly string[]): CliResult { + const env = { ...process.env }; + delete env.OPENAI_API_KEY; + env.OPENAI_BASE_URL = "http://127.0.0.1:9/no-network"; + env.CODEX_HOME = path.join(os.tmpdir(), "promiseproof-no-codex-home"); + + const result = spawnSync( + process.execPath, + [ + path.join(projectRoot, "node_modules", "tsx", "dist", "cli.mjs"), + path.join(projectRoot, "src", "verify", "cli.ts"), + ...args, + ], + { + cwd: projectRoot, + env, + encoding: "utf8", + timeout: 30_000, + windowsHide: true, + }, + ); + + if (result.error) { + throw result.error; + } + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + }; +} + +after(async () => { + await Promise.all( + temporaryRoots.map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +test("CLI verifies broken OFF with exit 2 and deterministic reports", async () => { + const root = await temporaryRoot(); + const evidence = path.join(root, "broken-off.json"); + const output = path.join(root, "report"); + await writeJson(evidence, brokenOffExample); + + const result = runCli([ + "verify", + "--evidence", + evidence, + "--out", + output, + ]); + + assert.equal(result.status, 2); + assert.match(result.stdout, /^BROKEN_PROMISE/); + const report = JSON.parse( + await readFile(path.join(output, "report.json"), "utf8"), + ) as { outcome: string; violations: Array<{ code: string }> }; + assert.equal(report.outcome, "BROKEN_PROMISE"); + assert.deepEqual(report.violations.map((item) => item.code), [ + "PP_IDENTIFIABLE_EVENT_LEAK", + ]); +}); + +test("CLI verifies passing OFF and ON with exit 0 without API key, network, browser, or Codex", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "passing-off.json"); + const on = path.join(root, "passing-on.json"); + await writeJson(off, passingOffExample); + await writeJson(on, passingOnExample); + + for (const [filename, directory] of [ + [off, "off-report"], + [on, "on-report"], + ] as const) { + const result = runCli([ + "verify", + "--evidence", + filename, + "--out", + path.join(root, directory), + ]); + assert.equal(result.status, 0); + assert.match(result.stdout, /^PASS/); + assert.equal(result.stderr, ""); + } +}); + +test("CLI verifies broken ON with exit 2", async () => { + const root = await temporaryRoot(); + const brokenOn = clone(passingOnExample); + brokenOn.evidence.activity.recommendationServiceReceipts = []; + const evidence = path.join(root, "broken-on.json"); + await writeJson(evidence, brokenOn); + + const result = runCli([ + "verify", + "--evidence", + evidence, + "--out", + path.join(root, "report"), + ]); + + assert.equal(result.status, 2); + assert.match(result.stdout, /^BROKEN_PROMISE/); +}); + +test("CLI gate passes only passing OFF plus passing ON", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "passing-off.json"); + const brokenOff = path.join(root, "broken-off.json"); + const on = path.join(root, "passing-on.json"); + await writeJson(off, passingOffExample); + await writeJson(brokenOff, brokenOffExample); + await writeJson(on, passingOnExample); + + const passing = runCli([ + "gate", + "--off", + off, + "--on", + on, + "--out", + path.join(root, "passing-gate"), + ]); + const broken = runCli([ + "gate", + "--off", + brokenOff, + "--on", + on, + "--out", + path.join(root, "broken-gate"), + ]); + + assert.equal(passing.status, 0); + assert.equal(broken.status, 2); + const gateReport = JSON.parse( + await readFile(path.join(root, "passing-gate", "report.json"), "utf8"), + ) as { + evaluations: { + off: { canonicalVerdict: string }; + on: { canonicalVerdict: string }; + }; + }; + assert.equal(gateReport.evaluations.off.canonicalVerdict, "pass"); + assert.equal(gateReport.evaluations.on.canonicalVerdict, "pass"); +}); + +test("CLI returns INVALID_EVIDENCE for malformed JSON and missing fields", async () => { + const root = await temporaryRoot(); + const malformed = path.join(root, "malformed.json"); + const missing = path.join(root, "missing.json"); + await writeFile(malformed, "{not json", "utf8"); + await writeJson(missing, { + schemaVersion: "1", + contractFamily: "activity-personalization/v1", + evidence: {}, + }); + + for (const [filename, reportName] of [ + [malformed, "malformed-report"], + [missing, "missing-report"], + ] as const) { + const result = runCli([ + "verify", + "--evidence", + filename, + "--out", + path.join(root, reportName), + ]); + assert.equal(result.status, 3); + assert.match(result.stderr, /^INVALID_EVIDENCE/m); + } +}); + +test("CLI rejects unknown keys, schema versions, and contract families with exit 3", async () => { + const root = await temporaryRoot(); + const candidates = [ + Object.assign(clone(passingOffExample), { unexpected: true }), + Object.assign(clone(passingOffExample), { schemaVersion: "2" }), + Object.assign(clone(passingOffExample), { + contractFamily: "arbitrary-promise/v1", + }), + ]; + + for (const [index, candidate] of candidates.entries()) { + const evidence = path.join(root, `invalid-${index}.json`); + await writeJson(evidence, candidate); + const result = runCli([ + "verify", + "--evidence", + evidence, + "--out", + path.join(root, `report-${index}`), + ]); + assert.equal(result.status, 3); + assert.match(result.stderr, /^INVALID_EVIDENCE/m); + } +}); + +test("CLI rejects a wrong gate scenario slot with exit 3", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "off.json"); + const on = path.join(root, "on.json"); + await writeJson(off, passingOnExample); + await writeJson(on, passingOffExample); + + const result = runCli([ + "gate", + "--off", + off, + "--on", + on, + "--out", + path.join(root, "gate"), + ]); + + assert.equal(result.status, 3); + assert.match(result.stderr, /--off bundle must contain OFF-scenario evidence/); + assert.match(result.stderr, /--on bundle must contain ON-scenario evidence/); +}); + +test("CLI rejects oversized input and bounded collection overflow with exit 3", async () => { + const root = await temporaryRoot(); + const oversized = path.join(root, "oversized.json"); + const collection = path.join(root, "collection.json"); + await writeFile(oversized, " ".repeat(MAX_INPUT_BYTES + 1), "utf8"); + const tooMany = clone(passingOffExample); + tooMany.evidence.recommendations.renderedItemIds = Array.from( + { length: 101 }, + (_, index) => `article-${index}`, + ); + await writeJson(collection, tooMany); + + for (const [filename, output] of [ + [oversized, "oversized-report"], + [collection, "collection-report"], + ] as const) { + const result = runCli([ + "verify", + "--evidence", + filename, + "--out", + path.join(root, output), + ]); + assert.equal(result.status, 3); + assert.match(result.stderr, /^INVALID_EVIDENCE/m); + } +}); + +test("CLI scaffold is complete and refuses conflicting files", async () => { + const root = await temporaryRoot(); + const output = path.join(root, "scaffold"); + + const first = runCli(["init", "--out", output]); + const second = runCli(["init", "--out", output]); + + assert.equal(first.status, 0); + assert.equal(second.status, 1); + assert.match(second.stderr, /Refusing to overwrite existing scaffold file/); + for (const filename of [ + "activity-personalization.v1.schema.json", + "broken-off.example.json", + "passing-off.example.json", + "passing-on.example.json", + "producer-template.mjs", + "README.md", + ]) { + assert.ok((await readFile(path.join(output, filename))).byteLength > 0); + } +}); + +test("CLI unknown command, missing argument, and output traversal are usage errors", async () => { + const unknown = runCli(["unknown"]); + const missing = runCli(["verify", "--evidence", "bundle.json"]); + const traversal = runCli(["init", "--out", "../escaped"]); + + for (const result of [unknown, missing, traversal]) { + assert.equal(result.status, 1); + assert.match(result.stderr, /USAGE_ERROR/); + } +}); + +test("CLI creates byte-identical JSON and Markdown across repeated runs", async () => { + const root = await temporaryRoot(); + const evidence = path.join(root, "passing-off.json"); + const first = path.join(root, "first"); + const second = path.join(root, "second"); + await writeJson(evidence, passingOffExample); + + const firstResult = runCli([ + "verify", + "--evidence", + evidence, + "--out", + first, + ]); + const secondResult = runCli([ + "verify", + "--evidence", + evidence, + "--out", + second, + ]); + + assert.equal(firstResult.status, 0); + assert.equal(secondResult.status, 0); + assert.deepEqual( + await readFile(path.join(first, "report.json")), + await readFile(path.join(second, "report.json")), + ); + assert.deepEqual( + await readFile(path.join(first, "report.md")), + await readFile(path.join(second, "report.md")), + ); +}); + +test("CLI uses the shared canonical binding and changes reports for semantic input changes", async () => { + const root = await temporaryRoot(); + const reorderedPath = path.join(root, "reordered.json"); + const changedPath = path.join(root, "changed.json"); + const reorderedOutput = path.join(root, "reordered-report"); + const changedOutput = path.join(root, "changed-report"); + const reordered = { + evidence: { + recommendations: passingOffExample.evidence.recommendations, + activity: passingOffExample.evidence.activity, + control: passingOffExample.evidence.control, + subjectId: passingOffExample.evidence.subjectId, + scenario: passingOffExample.evidence.scenario, + }, + contractFamily: passingOffExample.contractFamily, + schemaVersion: passingOffExample.schemaVersion, + }; + const changed = clone(passingOffExample); + changed.evidence.subjectId = "article-atlas-reader-002"; + await writeFile(reorderedPath, JSON.stringify(reordered), "utf8"); + await writeJson(changedPath, changed); + + assert.equal( + runCli([ + "verify", + "--evidence", + reorderedPath, + "--out", + reorderedOutput, + ]).status, + 0, + ); + assert.equal( + runCli([ + "verify", + "--evidence", + changedPath, + "--out", + changedOutput, + ]).status, + 0, + ); + + const reorderedReport = JSON.parse( + await readFile(path.join(reorderedOutput, "report.json"), "utf8"), + ) as { inputBinding: { sha256: string } }; + const changedReport = JSON.parse( + await readFile(path.join(changedOutput, "report.json"), "utf8"), + ) as { inputBinding: { sha256: string } }; + assert.equal( + reorderedReport.inputBinding.sha256, + (await bindExternalBundle(passingOffExample)).sha256, + ); + assert.match( + await readFile(path.join(reorderedOutput, "report.md"), "utf8"), + new RegExp(reorderedReport.inputBinding.sha256, "u"), + ); + assert.notEqual( + reorderedReport.inputBinding.sha256, + changedReport.inputBinding.sha256, + ); + assert.notDeepEqual( + await readFile(path.join(reorderedOutput, "report.json")), + await readFile(path.join(changedOutput, "report.json")), + ); + assert.notDeepEqual( + await readFile(path.join(reorderedOutput, "report.md")), + await readFile(path.join(changedOutput, "report.md")), + ); +}); + +test("CLI help succeeds and documents commands and exit codes", () => { + const result = runCli(["--help"]); + + assert.equal(result.status, 0); + assert.match(result.stdout, /init --out/); + assert.match(result.stdout, /verify --evidence/); + assert.match(result.stdout, /gate --off/); + assert.match(result.stdout, /3 INVALID_EVIDENCE/); +}); + +test("CLI check distinguishes malformed reports from valid semantic mismatches", async () => { + const root = await temporaryRoot(); + const off = path.join(root, "off.json"); + const on = path.join(root, "on.json"); + const output = path.join(root, "output"); + const reportPath = path.join(output, "report.json"); + await writeJson(off, passingOffExample); + await writeJson(on, passingOnExample); + assert.equal( + runCli(["gate", "--off", off, "--on", on, "--out", output]).status, + 0, + ); + + await writeJson(reportPath, { schemaVersion: "2" }); + const malformed = runCli([ + "check", + "--report", + reportPath, + "--off", + off, + "--on", + on, + ]); + assert.equal(malformed.status, 3); + assert.match(malformed.stdout, /INVALID_REPORT_OR_EVIDENCE/); + + const genuineOutput = path.join(root, "genuine"); + assert.equal( + runCli(["gate", "--off", off, "--on", on, "--out", genuineOutput]).status, + 0, + ); + const forged = JSON.parse( + await readFile(path.join(genuineOutput, "report.json"), "utf8"), + ); + forged.outcome = "BROKEN_PROMISE"; + await writeJson(reportPath, forged); + const mismatch = runCli([ + "check", + "--report", + reportPath, + "--off", + off, + "--on", + on, + ]); + assert.equal(mismatch.status, 4); + assert.match(mismatch.stdout, /STALE_OR_MISMATCH/); +}); diff --git a/tests/verify/core.unit.ts b/tests/verify/core.unit.ts new file mode 100644 index 0000000..aeb1ef0 --- /dev/null +++ b/tests/verify/core.unit.ts @@ -0,0 +1,729 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; + +import { evaluatePromise } from "../../src/shared/evaluator.js"; +import { adaptExternalEvidence } from "../../src/verify/adapter.js"; +import { + bindExternalBundle, + canonicalizeJson, + EVALUATOR_SOURCE_SHA256, + evaluatorSourceMatchesFingerprint, +} from "../../src/verify/binding.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, + producerTemplate, + scaffoldReadme, +} from "../../src/verify/examples.js"; +import { + createGateReport, + createVerifyReport, + serializeGateReportMarkdown, + serializeReportJson, + serializeVerifyReportMarkdown, + type VerifyReport, +} from "../../src/verify/report.js"; +import { + externalBundleJsonSchema, + MAX_COLLECTION_ITEMS, + type ExternalBundle, +} from "../../src/verify/schema.js"; +import { + runGate, + verifierTestHooks, + verifyBundle, +} from "../../src/verify/verify.js"; +import { + canonicalCases, + projectCanonicalEvidence, +} from "./canonical-projection.js"; + +const projectRoot = process.cwd(); + +function clone(value: T): T { + return structuredClone(value); +} + +function stable(value: unknown): string { + return JSON.stringify(value); +} + +test("broken OFF, passing OFF, passing ON, and broken ON preserve canonical outcomes", () => { + const brokenOff = verifyBundle(brokenOffExample); + const passingOff = verifyBundle(passingOffExample); + const passingOn = verifyBundle(passingOnExample); + const brokenOnBundle = clone(passingOnExample); + brokenOnBundle.evidence.activity.recommendationServiceReceipts = []; + const brokenOn = verifyBundle(brokenOnBundle); + + assert.equal(brokenOff.outcome, "BROKEN_PROMISE"); + assert.deepEqual( + brokenOff.evaluation?.violations.map((item) => item.code), + ["PP_IDENTIFIABLE_EVENT_LEAK"], + ); + assert.equal(passingOff.outcome, "PASS"); + assert.equal(passingOn.outcome, "PASS"); + assert.equal(brokenOn.outcome, "BROKEN_PROMISE"); + assert.deepEqual( + brokenOn.evaluation?.violations.map((item) => item.code), + ["PP_EXPECTED_ACTIVITY_MISSING"], + ); +}); + +test("gate evaluates exactly twice only after both inputs and slots are valid", () => { + let calls = 0; + const result = verifierTestHooks.runGateWithEvaluator( + passingOffExample, + passingOnExample, + (evidence) => { + calls += 1; + return evaluatePromise(evidence); + }, + ); + + assert.equal(result.outcome, "PASS"); + assert.equal(calls, 2); +}); + +test("invalid gates atomically invoke no canonical evaluation", () => { + const invalidCases: Array<[string, unknown, unknown]> = [ + ["OFF bundle in ON slot", passingOnExample, passingOnExample], + ["ON bundle in OFF slot", passingOffExample, passingOffExample], + ["malformed OFF", { malformed: true }, passingOnExample], + ["malformed ON", passingOffExample, { malformed: true }], + [ + "unsupported version", + { ...clone(passingOffExample), schemaVersion: "2" }, + passingOnExample, + ], + [ + "unsupported family", + passingOffExample, + { + ...clone(passingOnExample), + contractFamily: "generic-consent/v1", + }, + ], + ]; + + for (const [name, off, on] of invalidCases) { + let calls = 0; + const result = verifierTestHooks.runGateWithEvaluator( + off, + on, + (evidence) => { + calls += 1; + return evaluatePromise(evidence); + }, + ); + + assert.equal(result.outcome, "INVALID_EVIDENCE", name); + assert.equal(result.off.evaluation, null, name); + assert.equal(result.on.evaluation, null, name); + assert.equal(calls, 0, name); + assert.ok(result.issues.length > 0, name); + assert.deepEqual([...result.issues], [...result.issues].sort(), name); + assert.deepEqual( + [...result.off.issues], + [...result.off.issues].sort(), + name, + ); + assert.deepEqual( + [...result.on.issues], + [...result.on.issues].sort(), + name, + ); + } +}); + +test("swapped gate slots remain unevaluated", () => { + const result = runGate(passingOnExample, passingOffExample); + + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.off.evaluation, null); + assert.equal(result.on.evaluation, null); + assert.deepEqual(result.issues, [ + "--off bundle must contain OFF-scenario evidence", + "--on bundle must contain ON-scenario evidence", + ]); +}); + +test("strict minimal validation rejects missing, unknown, unsupported, and malformed fields", () => { + const missing = clone(passingOffExample) as Record; + delete (missing.evidence as Record).control; + const unknown = clone(passingOffExample) as Record; + (unknown.evidence as Record).unexpected = true; + const nestedUnknown = clone(passingOffExample); + Object.assign(nestedUnknown.evidence.control, { unexpected: true }); + const version = { ...clone(passingOffExample), schemaVersion: "2" }; + const family = { + ...clone(passingOffExample), + contractFamily: "generic-consent/v1", + }; + const whitespace = clone(passingOffExample); + whitespace.evidence.subjectId = " "; + const timestamp = clone(passingOnExample); + timestamp.evidence.activity.capturedActivities[0]!.occurredAt = + "not-a-timestamp"; + + for (const candidate of [ + missing, + unknown, + nestedUnknown, + version, + family, + whitespace, + timestamp, + ]) { + const result = verifyBundle(candidate); + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.evaluation, null); + assert.ok(result.issues.length > 0); + } +}); + +test("ASCII controls and Unicode bidi override/isolate characters are rejected", () => { + const forbidden = [ + "\u0000", + "\u0008", + "\t", + "\n", + "\r", + "\u007f", + "\u202a", + "\u202b", + "\u202c", + "\u202d", + "\u202e", + "\u2066", + "\u2067", + "\u2068", + "\u2069", + ]; + + for (const character of forbidden) { + const candidate = clone(passingOnExample); + candidate.evidence.subjectId = `reader${character}name`; + assert.equal( + verifyBundle(candidate).outcome, + "INVALID_EVIDENCE", + `U+${character.codePointAt(0)!.toString(16).padStart(4, "0")}`, + ); + } +}); + +test("bounded collections are rejected before canonical evaluation", () => { + const oversized = clone(passingOffExample); + oversized.evidence.recommendations.renderedItemIds = Array.from( + { length: MAX_COLLECTION_ITEMS + 1 }, + (_, index) => `article-${index}`, + ); + + const result = verifyBundle(oversized); + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.evaluation, null); + assert.match(result.issues.join("\n"), /Too big/); +}); + +test("Markdown encodes HTML, ampersands, backticks, pipes, and backslashes deterministically", async () => { + const adversarial = clone(passingOnExample); + const subject = "&`pipe|slash\\"; + adversarial.evidence.subjectId = subject; + adversarial.evidence.activity.capturedActivities[0]!.subjectId = subject; + adversarial.evidence.activity.recommendationServiceReceipts[0]!.subjectId = + subject; + adversarial.evidence.recommendations.recommendationServiceReceipts[0]!.subjectId = + subject; + + const first = await createVerifyReport(verifyBundle(adversarial)); + const second = await createVerifyReport(verifyBundle(clone(adversarial))); + const markdown = serializeVerifyReportMarkdown(first); + + assert.equal(first.outcome, "PASS"); + assert.equal(markdown, serializeVerifyReportMarkdown(second)); + assert.match(markdown, /<reader>&`pipe\\\|slash\\\\/); + assert.doesNotMatch(markdown, //); +}); + +test("Markdown converts CR, LF, and CRLF to inert spaces after encoding", async () => { + const report = clone( + await createVerifyReport(verifyBundle(passingOffExample)), + ) as VerifyReport; + const mutableClause = report.clauses[0]!; + Object.assign(mutableClause, { + expected: "one\r\ntwo\nthree\rfour", + observed: "&`|\\", + }); + + const markdown = serializeVerifyReportMarkdown(report); + assert.match(markdown, /one two three four/); + assert.match(markdown, /<tag>&`\\\|\\\\/); + assert.doesNotMatch(markdown, /\r/); +}); + +test("single and gate reports preserve deterministic canonical ordering", async () => { + const single = await createVerifyReport(verifyBundle(brokenOffExample)); + const gate = await createGateReport( + runGate(passingOffExample, passingOnExample), + ); + + assert.deepEqual( + single.clauses.map((clause) => clause.id), + [ + "no_identifiable_activity", + "contextual_feed_functional", + "preference_survives_reload", + ], + ); + assert.deepEqual( + gate.evaluations.on.clauses.map((clause) => clause.id), + ["expected_activity_received", "behavioral_feed_functional"], + ); + assert.equal( + serializeReportJson(single), + serializeReportJson( + await createVerifyReport(verifyBundle(clone(brokenOffExample))), + ), + ); + assert.equal( + serializeGateReportMarkdown(gate), + serializeGateReportMarkdown( + await createGateReport( + runGate(clone(passingOffExample), clone(passingOnExample)), + ), + ), + ); +}); + +test("adapter placeholders never enter public reports", async () => { + const json = serializeReportJson( + await createVerifyReport(verifyBundle(passingOffExample)), + ); + const markdown = serializeVerifyReportMarkdown( + await createVerifyReport(verifyBundle(passingOffExample)), + ); + + for (const forbidden of [ + "promiseproof-external-adapter", + "2000-01-01T00:00:00.000Z", + "Not part of externally supplied evidence", + ]) { + assert.doesNotMatch(json, new RegExp(forbidden)); + assert.doesNotMatch(markdown, new RegExp(forbidden)); + } +}); + +test("canonical bundle binding is stable across key order and JSON whitespace", async () => { + const reordered: ExternalBundle = { + evidence: { + recommendations: clone(passingOffExample.evidence.recommendations), + activity: clone(passingOffExample.evidence.activity), + control: clone(passingOffExample.evidence.control), + subjectId: passingOffExample.evidence.subjectId, + scenario: passingOffExample.evidence.scenario, + }, + contractFamily: passingOffExample.contractFamily, + schemaVersion: passingOffExample.schemaVersion, + }; + const compact = JSON.parse(JSON.stringify(passingOffExample)) as ExternalBundle; + const formatted = JSON.parse( + JSON.stringify(passingOffExample, null, 4), + ) as ExternalBundle; + const expected = await bindExternalBundle(passingOffExample); + + assert.deepEqual(await bindExternalBundle(reordered), expected); + assert.deepEqual(await bindExternalBundle(compact), expected); + assert.deepEqual(await bindExternalBundle(formatted), expected); + assert.equal(canonicalizeJson(reordered), canonicalizeJson(passingOffExample)); + assert.equal( + canonicalizeJson({ "ä": 1, a: 2, Z: 3, A: 4 }), + '{"A":4,"Z":3,"a":2,"ä":1}', + ); +}); + +test("array order and every evidence category are bound by the digest", async () => { + const baseline = (await bindExternalBundle(passingOnExample)).sha256; + const mutations: Array<[string, (bundle: ExternalBundle) => void]> = [ + [ + "subjectId", + (item) => { + item.evidence.subjectId = "different-subject"; + }, + ], + [ + "control.uiPreference", + (item) => { + item.evidence.control.uiPreference = "off"; + }, + ], + [ + "control.toggleChecked", + (item) => { + item.evidence.control.toggleChecked = false; + }, + ], + [ + "control.storedPreference", + (item) => { + item.evidence.control.storedPreference = "off"; + }, + ], + [ + "control.backendPreference", + (item) => { + item.evidence.control.backendPreference = "off"; + }, + ], + [ + "control.reloadObserved", + (item) => { + item.evidence.control.reloadObserved = false; + }, + ], + [ + "activity", + (item) => { + item.evidence.activity.capturedActivities[0]!.itemId = + "different-origin"; + }, + ], + [ + "recommendation", + (item) => { + item.evidence.recommendations.renderedItemIds = ["different-story"]; + }, + ], + ]; + + for (const [name, mutate] of mutations) { + const candidate = clone(passingOnExample); + mutate(candidate); + assert.notEqual((await bindExternalBundle(candidate)).sha256, baseline, name); + } + + const ordered = clone(passingOnExample); + ordered.evidence.recommendations.renderedItemIds = ["story-a", "story-b"]; + const reversed = clone(ordered); + reversed.evidence.recommendations.renderedItemIds.reverse(); + assert.notEqual( + (await bindExternalBundle(ordered)).sha256, + (await bindExternalBundle(reversed)).sha256, + ); +}); + +test("report schema v2 binds single and gate inputs and discloses evaluator fingerprint", async () => { + const single = await createVerifyReport(verifyBundle(passingOffExample)); + const gate = await createGateReport( + runGate(passingOffExample, passingOnExample), + ); + const expectedOff = await bindExternalBundle(passingOffExample); + const expectedOn = await bindExternalBundle(passingOnExample); + + assert.equal(single.schemaVersion, "2"); + assert.deepEqual(single.inputBinding, expectedOff); + assert.deepEqual(gate.inputBindings.off, expectedOff); + assert.deepEqual(gate.inputBindings.on, expectedOn); + assert.notEqual(gate.inputBindings.off.sha256, gate.inputBindings.on.sha256); + assert.equal(single.authority.evaluatorSourceSha256, EVALUATOR_SOURCE_SHA256); + assert.equal(gate.authority.evaluatorSourceSha256, EVALUATOR_SOURCE_SHA256); + + const singleJson = serializeReportJson(single); + const singleMarkdown = serializeVerifyReportMarkdown(single); + const gateJson = serializeReportJson(gate); + const gateMarkdown = serializeGateReportMarkdown(gate); + for (const digest of [expectedOff.sha256, expectedOn.sha256]) { + assert.match(digest, /^[0-9a-f]{64}$/u); + assert.ok( + digest === expectedOff.sha256 + ? singleJson.includes(digest) && singleMarkdown.includes(digest) + : true, + ); + assert.ok(gateJson.includes(digest)); + assert.ok(gateMarkdown.includes(digest)); + } + for (const output of [singleJson, singleMarkdown, gateJson, gateMarkdown]) { + assert.ok(output.includes(EVALUATOR_SOURCE_SHA256)); + } +}); + +test("unsupported envelope literals cannot reach report binding", async () => { + for (const candidate of [ + { ...clone(passingOffExample), schemaVersion: "2" }, + { ...clone(passingOffExample), contractFamily: "unsupported/v1" }, + ]) { + const result = verifyBundle(candidate); + assert.equal(result.outcome, "INVALID_EVIDENCE"); + assert.equal(result.validatedCanonicalJson, null); + await assert.rejects(createVerifyReport(result), /Invalid evidence/); + } +}); + +test("committed evaluator fingerprint matches canonical-LF source and known Git blob", async () => { + const evaluatorPath = path.join(projectRoot, "src", "shared", "evaluator.ts"); + const source = await readFile(evaluatorPath, "utf8"); + assert.equal(await evaluatorSourceMatchesFingerprint(source), true); + assert.equal( + await evaluatorSourceMatchesFingerprint(source, "0".repeat(64)), + false, + ); + + const git = spawnSync("git", ["hash-object", "src/shared/evaluator.ts"], { + cwd: projectRoot, + encoding: "utf8", + windowsHide: true, + }); + assert.equal(git.status, 0, git.stderr); + assert.equal( + git.stdout.trim(), + "9654cb43f4388c7f399bf0717ceb94bba2718962", + ); +}); + +test("canonical projection and deterministic adapter preserve every evaluation byte-for-byte", () => { + for (const [name, canonical] of Object.entries(canonicalCases)) { + const original = evaluatePromise(canonical); + const roundTripped = evaluatePromise( + adaptExternalEvidence(projectCanonicalEvidence(canonical)), + ); + + assert.equal(stable(roundTripped), stable(original), name); + } +}); + +test("every evaluator-relevant public field has mutation coverage", () => { + const mutationCases: Array<{ + name: string; + bundle: ExternalBundle; + mutate: (bundle: ExternalBundle) => void; + invalid?: boolean; + }> = [ + { + name: "scenario", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.scenario = "on"; + }, + }, + { + name: "subjectId", + bundle: passingOnExample, + mutate: (item) => { + item.evidence.subjectId = "different-subject"; + }, + }, + ...([ + ["uiPreference", "on"], + ["toggleChecked", true], + ["storedPreference", "on"], + ["backendPreference", "on"], + ["reloadObserved", false], + ] as const).map(([field, value]) => ({ + name: `control.${field}`, + bundle: passingOffExample, + mutate: (item: ExternalBundle) => { + Object.assign(item.evidence.control, { [field]: value }); + }, + })), + { + name: "capturedActivities", + bundle: passingOnExample, + mutate: (item) => { + item.evidence.activity.capturedActivities = []; + }, + }, + { + name: "recommendationServiceActivityReceipts", + bundle: passingOnExample, + mutate: (item) => { + item.evidence.activity.recommendationServiceReceipts = []; + }, + }, + ...(["runId", "subjectId", "itemId", "clientSequence", "occurredAt"] as const) + .flatMap((field) => [ + { + name: `capturedActivity.${field}`, + bundle: passingOnExample, + mutate: (item: ExternalBundle) => { + Object.assign(item.evidence.activity.capturedActivities[0]!, { + [field]: + field === "clientSequence" + ? 2 + : field === "occurredAt" + ? "2026-01-15T12:00:02.000Z" + : `different-${field}`, + }); + }, + }, + { + name: `activityReceipt.${field}`, + bundle: passingOnExample, + mutate: (item: ExternalBundle) => { + Object.assign( + item.evidence.activity.recommendationServiceReceipts[0]!, + { + [field]: + field === "clientSequence" + ? 2 + : field === "occurredAt" + ? "2026-01-15T12:00:02.000Z" + : `different-${field}`, + }, + ); + }, + }, + ]), + { + name: "capturedActivity.eventType", + bundle: passingOnExample, + invalid: true, + mutate: (item) => { + Object.assign(item.evidence.activity.capturedActivities[0]!, { + eventType: "click", + }); + }, + }, + { + name: "activityReceipt.eventType", + bundle: passingOnExample, + invalid: true, + mutate: (item) => { + Object.assign( + item.evidence.activity.recommendationServiceReceipts[0]!, + { eventType: "click" }, + ); + }, + }, + { + name: "feedFunctional", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.feedFunctional = false; + }, + }, + { + name: "renderedSource", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.renderedSource = "behavioral"; + }, + }, + { + name: "renderedItemIds", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.renderedItemIds = ["different-item"]; + }, + }, + { + name: "recommendationServiceReceipts", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts = []; + }, + }, + { + name: "recommendationReceipt.source", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts[0]!.source = + "behavioral"; + }, + }, + { + name: "recommendationReceipt.subjectId", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts[0]!.subjectId = + item.evidence.subjectId; + }, + }, + { + name: "recommendationReceipt.itemIds", + bundle: passingOffExample, + mutate: (item) => { + item.evidence.recommendations.recommendationServiceReceipts[0]!.itemIds = + ["different-item"]; + }, + }, + ]; + + for (const mutation of mutationCases) { + const candidate = clone(mutation.bundle); + const baseline = verifyBundle(mutation.bundle); + mutation.mutate(candidate); + const changed = verifyBundle(candidate); + + if (mutation.invalid) { + assert.equal(changed.outcome, "INVALID_EVIDENCE", mutation.name); + } else { + assert.notEqual( + stable(changed.evaluation), + stable(baseline.evaluation), + mutation.name, + ); + assert.equal(changed.outcome, "BROKEN_PROMISE", mutation.name); + } + } +}); + +test("committed schema and scaffold text are generated from runtime definitions", async () => { + const artifactRoot = path.join(projectRoot, "artifacts", "verify"); + const committedSchema = await readFile( + path.join(artifactRoot, "activity-personalization.v1.schema.json"), + "utf8", + ); + + assert.equal( + committedSchema, + `${JSON.stringify(externalBundleJsonSchema(), null, 2)}\n`, + ); + assert.equal( + await readFile(path.join(artifactRoot, "producer-template.mjs"), "utf8"), + producerTemplate, + ); + assert.equal( + await readFile(path.join(artifactRoot, "README.md"), "utf8"), + scaffoldReadme, + ); + assert.ok(committedSchema.split("\n").length < 400); + assert.ok(Buffer.byteLength(committedSchema) < 16_000); +}); + +test("Article Atlas examples remain independent and one-family only", () => { + const serialized = JSON.stringify([ + brokenOffExample, + passingOffExample, + passingOnExample, + ]); + + assert.match(serialized, /article-atlas/i); + assert.doesNotMatch( + serialized, + /signal shelf|initialization-race|propagation-failure|fixture|src\//i, + ); +}); + +test("external verifier modules have no forbidden implementation imports", async () => { + const verifyDirectory = path.join(projectRoot, "src", "verify"); + const filenames = [ + "adapter.ts", + "binding.ts", + "cli.ts", + "examples.ts", + "outcome.ts", + "report.ts", + "schema.ts", + "verify.ts", + ]; + const forbidden = + /from\s+["'][^"']*(client|server|investigation|repair|judge|tests|docs\/evidence|artifacts|@openai|openai|codex|playwright)[^"']*["']/; + + for (const filename of filenames) { + const source = await readFile(path.join(verifyDirectory, filename), "utf8"); + assert.doesNotMatch(source, forbidden, filename); + } +}); diff --git a/tests/verify/judge-mode.playwright.config.ts b/tests/verify/judge-mode.playwright.config.ts new file mode 100644 index 0000000..e2ed3f0 --- /dev/null +++ b/tests/verify/judge-mode.playwright.config.ts @@ -0,0 +1,34 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { defineConfig } from "@playwright/test"; + +// Serves the assembled combined site (dist/site) statically and drives the +// hosted verifier at /verify/. The verifier is a static, self-contained page, +// so it needs no application server — only the built assets. +const PORT = 4188; +const BASE = `http://127.0.0.1:${PORT}`; +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +export default defineConfig({ + testDir: ".", + testMatch: /judge-mode\.spec\.ts/, + fullyParallel: false, + workers: 1, + retries: 0, + reporter: [["list"]], + timeout: 30_000, + use: { + baseURL: BASE, + trace: "retain-on-failure", + }, + webServer: { + command: `npm run build:site && node scripts/serve-site.mjs dist/site ${PORT}`, + cwd: REPO_ROOT, + url: `${BASE}/verify/`, + reuseExistingServer: false, + timeout: 180_000, + stdout: "pipe", + stderr: "pipe", + }, +}); diff --git a/tests/verify/judge-mode.spec.ts b/tests/verify/judge-mode.spec.ts new file mode 100644 index 0000000..16c12b9 --- /dev/null +++ b/tests/verify/judge-mode.spec.ts @@ -0,0 +1,459 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { request as httpRequest } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { expect, test, type Page, type Request } from "@playwright/test"; + +import { EVALUATOR_SOURCE_SHA256 } from "../../src/verify/binding.js"; +import { + brokenOffExample, + passingOffExample, + passingOnExample, +} from "../../src/verify/examples.js"; +import { MAX_INPUT_BYTES, MAX_COLLECTION_ITEMS } from "../../src/verify/schema.js"; + +const ROUTE = "/verify/"; +const REPO = process.cwd(); + +function clone(value: T): T { + return structuredClone(value); +} + +async function downloadBytes(page: Page, selector: string): Promise { + const [download] = await Promise.all([ + page.waitForEvent("download"), + page.locator(selector).click(), + ]); + const filename = await download.path(); + if (filename === null) throw new Error("browser download has no local path"); + return readFileSync(filename); +} + +function cliGateReports( + off: unknown, + on: unknown, + expectedExit: 0 | 2, +): { json: Buffer; markdown: Buffer } { + const kit = mkdtempSync(path.join(tmpdir(), "pp-parity-")); + try { + const offPath = path.join(kit, "off.json"); + const onPath = path.join(kit, "on.json"); + writeFileSync(offPath, `${JSON.stringify(off, null, 2)}\n`); + writeFileSync(onPath, `${JSON.stringify(on, null, 2)}\n`); + const result = spawnSync( + process.execPath, + [ + path.join(REPO, "node_modules", "tsx", "dist", "cli.mjs"), + path.join(REPO, "src", "verify", "cli.ts"), + "gate", + "--off", + offPath, + "--on", + onPath, + "--out", + kit, + ], + { cwd: REPO, encoding: "utf8", windowsHide: true }, + ); + if (result.error !== undefined) throw result.error; + if (result.status !== expectedExit) { + throw new Error( + `CLI gate exited ${String(result.status)} instead of ${expectedExit}: ${result.stderr}`, + ); + } + return { + json: readFileSync(path.join(kit, "report.json")), + markdown: readFileSync(path.join(kit, "report.md")), + }; + } finally { + rmSync(kit, { recursive: true, force: true }); + } +} + +function rawStatus(baseURL: string, pathname: string): Promise { + const url = new URL(baseURL); + return new Promise((resolve, reject) => { + const request = httpRequest( + { + hostname: url.hostname, + port: url.port, + method: "GET", + path: pathname, + }, + (response) => { + response.resume(); + response.on("end", () => resolve(response.statusCode ?? 0)); + }, + ); + request.on("error", reject); + request.end(); + }); +} + +async function open(page: Page): Promise { + await page.goto(ROUTE); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); +} + +test("Judge Mode opens on PASS + BOUND with the pinned evaluator fingerprint", async ({ + page, +}) => { + await open(page); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + + // Five clause rows, all PASS, in the healthy state. + const rows = page.getByTestId("clauses").locator("tr"); + await expect(rows).toHaveCount(5); + await expect(page.getByTestId("clauses").locator(".vf-tag.fail")).toHaveCount(0); + + // The fingerprint is the pinned build value, verbatim, not recomputed copy. + await expect(page.locator("#vf-fp")).toHaveText(EVALUATOR_SOURCE_SHA256); + await expect(page.getByTestId("violations")).toContainText("no violations"); +}); + +test("nothing leaves the page across every Judge and BYO interaction", async ({ + page, + baseURL, +}) => { + await open(page); + const offenders: string[] = []; + const evidenceValues = [ + passingOffExample.evidence.subjectId, + passingOnExample.evidence.activity.capturedActivities[0]!.itemId, + ]; + const record = (request: Request): void => { + const url = request.url(); + const body = request.postData() ?? ""; + const local = url.startsWith("blob:") || url.startsWith("data:") || url.startsWith("about:"); + const networkApi = ["fetch", "xhr", "websocket", "eventsource", "ping"].includes( + request.resourceType(), + ); + const evidenceDerived = evidenceValues.some( + (value) => url.includes(value) || body.includes(value), + ); + if ( + !local && + (networkApi || + request.method() === "POST" || + request.isNavigationRequest() || + evidenceDerived) + ) { + offenders.push(`${request.method()} ${request.resourceType()} ${url}`); + } + }; + page.on("request", record); + page.on("websocket", (socket) => offenders.push(`WEBSOCKET ${socket.url()}`)); + page.on("framenavigated", (frame) => { + if (frame === page.mainFrame() && frame.url() !== `${baseURL}${ROUTE}`) { + offenders.push(`NAVIGATION ${frame.url()}`); + } + }); + + await page.getByTestId("tamper").click(); + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + await page.getByTestId("evaluate").click(); + await page.getByTestId("check").click(); + await page.getByTestId("seal").click(); + await page.getByTestId("reset").click(); + await downloadBytes(page, "#vf-dl-json"); + await downloadBytes(page, "#vf-dl-md"); + + await page.locator("#vf-file-off").setInputFiles({ + name: "passing-off.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOffExample)), + }); + await page.locator("#vf-file-on").setInputFiles({ + name: "passing-on.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOnExample)), + }); + await expect(page.locator("#vf-name-on")).toHaveText("passing-on.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("PASS"); + + await page.locator("#vf-byo-clear").click(); + await page.locator("#vf-file-off").setInputFiles({ + name: "passing-single.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOffExample)), + }); + await expect(page.locator("#vf-name-off")).toHaveText("passing-single.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("PASS"); + + await page.locator("#vf-byo-clear").click(); + await page.locator("#vf-file-off").setInputFiles({ + name: "invalid.json", + mimeType: "application/json", + buffer: Buffer.from("{not-json"), + }); + await expect(page.locator("#vf-name-off")).toHaveText("invalid.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await page.waitForTimeout(200); + + expect(offenders, `unexpected egress:\n${offenders.join("\n")}`).toEqual([]); +}); + +test("tampering the OFF evidence flips the same evaluator to a broken promise", async ({ + page, +}) => { + await open(page); + await page.getByTestId("tamper").click(); + + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + await expect(page.getByTestId("repro-pill")).toContainText("STALE_OR_MISMATCH"); + await expect(page.getByTestId("violations")).toContainText("PP_IDENTIFIABLE_EVENT_LEAK"); + await expect(page.getByTestId("violations").locator("li")).toHaveCount(1); + await expect(page.locator("#vf-off-facts")).toContainText("Identifiable captured activity1"); + await expect(page.locator("#vf-off-facts")).toContainText("Recommendation-service receipts1"); + await expect(page.getByTestId("clauses").locator(".vf-tag.fail")).toHaveCount(1); + await expect(page.getByTestId("clauses").locator(".vf-tag.pass")).toHaveCount(4); + + // The failing clause is the identifiable-activity clause, and the OFF panel is flagged. + const failing = page.getByTestId("clauses").locator("tr", { + has: page.locator(".vf-tag.fail"), + }); + await expect(failing).toContainText("no_identifiable_activity"); + await expect(page.locator("#vf-ev-off")).toHaveClass(/tampered/); +}); + +test("you can only seal the truth: re-sealing tampered evidence binds BROKEN, not PASS", async ({ + page, +}) => { + await open(page); + const originalOn = await page.locator("#vf-on-facts").textContent(); + const originalDigests = await page.locator("#vf-digests").textContent(); + const originalJson = await downloadBytes(page, "#vf-dl-json"); + await page.getByTestId("tamper").click(); + await expect(page.getByTestId("repro-pill")).toContainText("STALE_OR_MISMATCH"); + await expect(page.locator("#vf-on-facts")).toHaveText(originalOn ?? ""); + + await page.getByTestId("seal").click(); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + + // Reset returns to the passing, reproduced baseline. + await page.getByTestId("reset").click(); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + await expect(page.locator("#vf-digests")).toHaveText(originalDigests ?? ""); + await expect(page.locator("#vf-on-facts")).toHaveText(originalOn ?? ""); + expect(await downloadBytes(page, "#vf-dl-json")).toEqual(originalJson); +}); + +test("bring-your-own invalid evidence surfaces validation issues, not a verdict", async ({ + page, +}) => { + await open(page); + const bogus = JSON.stringify({ not: "a bundle" }); + await page + .locator("#vf-file-off") + .setInputFiles({ name: "bad.json", mimeType: "application/json", buffer: Buffer.from(bogus) }); + // Wait for the local FileReader to finish before verifying. + await expect(page.locator("#vf-name-off")).toHaveText("bad.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toBeVisible(); +}); + +test("bring-your-own gate rejects swapped scenario slots atomically", async ({ page }) => { + await open(page); + await page.locator("#vf-file-off").setInputFiles({ + name: "wrong-off.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOnExample)), + }); + await page.locator("#vf-file-on").setInputFiles({ + name: "wrong-on.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(passingOffExample)), + }); + await expect(page.locator("#vf-name-on")).toHaveText("wrong-on.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText( + "--off bundle must contain OFF-scenario evidence", + ); + await expect(page.locator("#vf-byo-issues")).toContainText( + "--on bundle must contain ON-scenario evidence", + ); + await expect(page.locator("#vf-byo-table")).toBeHidden(); + await expect(page.locator("#vf-byo-digests")).toBeEmpty(); +}); + +test("passing and sealed-broken JSON and Markdown are byte-identical to CLI output", async ({ + page, +}) => { + await open(page); + const passingCli = cliGateReports(passingOffExample, passingOnExample, 0); + expect(await downloadBytes(page, "#vf-dl-json")).toEqual(passingCli.json); + expect(await downloadBytes(page, "#vf-dl-md")).toEqual(passingCli.markdown); + + await page.getByTestId("tamper").click(); + await page.getByTestId("seal").click(); + await expect(page.getByTestId("outcome-pill")).toContainText("BROKEN_PROMISE"); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + const brokenCli = cliGateReports(brokenOffExample, passingOnExample, 2); + expect(await downloadBytes(page, "#vf-dl-json")).toEqual(brokenCli.json); + expect(await downloadBytes(page, "#vf-dl-md")).toEqual(brokenCli.markdown); +}); + +test("hostile local files render inertly and invalid input never inherits a verdict", async ({ + page, +}) => { + await open(page); + const hostileKey = ''; + const hostile = clone(passingOffExample) as any; + hostile.evidence.subjectId = "& ` ` | \\ visible text"; + + await page.locator("#vf-file-off").setInputFiles({ + name: `${hostileKey}.json`, + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(hostile)), + }); + await expect(page.locator("#vf-name-off")).toHaveText(`${hostileKey}.json`); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("PASS"); + await expect(page.locator("#pp-injected")).toHaveCount(0); + expect(await page.evaluate(() => (window as any).ppOwned)).toBeUndefined(); + + const unknown = clone(hostile); + unknown.evidence.control[hostileKey] = true; + await page.locator("#vf-file-off").setInputFiles({ + name: "unknown-key.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(unknown)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("Invalid input"); + await expect(page.locator("#pp-injected")).toHaveCount(0); + expect(await page.evaluate(() => (window as any).ppOwned)).toBeUndefined(); + + const bidi = clone(passingOffExample) as any; + bidi.evidence.subjectId = "safe\u202Eunsafe"; + await page.locator("#vf-file-off").setInputFiles({ + name: "bidi.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(bidi)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("forbidden control or bidi"); + + const long = clone(passingOffExample) as any; + long.evidence.subjectId = "x".repeat(257); + await page.locator("#vf-file-off").setInputFiles({ + name: "long.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(long)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + + const collection = clone(passingOffExample) as any; + collection.evidence.recommendations.renderedItemIds = Array.from( + { length: MAX_COLLECTION_ITEMS + 1 }, + (_, index) => `item-${index}`, + ); + await page.locator("#vf-file-off").setInputFiles({ + name: "collection.json", + mimeType: "application/json", + buffer: Buffer.from(JSON.stringify(collection)), + }); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); +}); + +test("input and drag/drop reject oversized or malformed files without a product verdict", async ({ + page, +}) => { + await open(page); + await page.locator("#vf-file-off").setInputFiles({ + name: "oversized.json", + mimeType: "application/json", + buffer: Buffer.alloc(MAX_INPUT_BYTES + 1, 0x20), + }); + await expect(page.locator("#vf-name-off")).toContainText("rejected"); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("input exceeds"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("PASS"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("BROKEN_PROMISE"); + + await page.locator("#vf-byo-clear").click(); + await page.evaluate(() => { + const file = new File(["{not-json"], "drop-bad.json", { + type: "application/json", + }); + const transfer = new DataTransfer(); + transfer.items.add(file); + document.querySelector("#vf-drop-off")!.dispatchEvent( + new DragEvent("drop", { + bubbles: true, + cancelable: true, + dataTransfer: transfer, + }), + ); + }); + await expect(page.locator("#vf-name-off")).toHaveText("drop-bad.json"); + await page.getByTestId("byo-run").click(); + await expect(page.getByTestId("byo-outcome")).toContainText("INVALID_EVIDENCE"); + await expect(page.locator("#vf-byo-issues")).toContainText("malformed JSON"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("PASS"); + await expect(page.getByTestId("byo-outcome")).not.toContainText("BROKEN_PROMISE"); +}); + +test("assembled root, walkthrough, verifier routes, assets, refresh, and traversal boundary", async ({ + page, + request, + baseURL, +}) => { + if (baseURL === undefined) throw new Error("missing test baseURL"); + const root = await request.get("/"); + expect(root.status()).toBe(200); + expect(Buffer.from(await root.body())).toEqual(readFileSync(path.join(REPO, "landing", "index.html"))); + + for (const route of ["/walkthrough/", "/verify/"]) { + const response = await request.get(route); + expect(response.status()).toBe(200); + const html = await response.text(); + const subpath = route.slice(0, -1); + const assets = [...html.matchAll(new RegExp(`${subpath}/assets/[^"'?#]+`, "gu"))].map( + (match) => match[0], + ); + expect(assets.length).toBeGreaterThan(0); + for (const asset of new Set(assets)) { + expect((await request.get(asset)).status()).toBe(200); + } + } + + await page.goto("/verify/"); + await expect(page.getByTestId("outcome-pill")).toContainText("PASS"); + await page.reload(); + await expect(page.getByTestId("repro-pill")).toContainText("BOUND_AND_REPRODUCED"); + + expect(await rawStatus(baseURL, "/%2e%2e%2fpackage.json")).toBe(403); + expect([403, 404]).toContain( + await rawStatus(baseURL, "/%2e%2e%5cpackage.json"), + ); +}); + +test("JavaScript-disabled verifier gives an honest fallback", async ({ browser, baseURL }) => { + const context = await browser.newContext({ javaScriptEnabled: false }); + try { + const page = await context.newPage(); + await page.goto(`${baseURL}${ROUTE}`); + await expect(page.locator(".vf-fallback")).toBeVisible(); + await expect(page.locator(".vf-fallback")).toContainText( + "interactive and needs JavaScript", + ); + await expect(page.locator(".vf-fallback")).toContainText("GitHub repository"); + } finally { + await context.close(); + } +}); diff --git a/tests/verify/pipeline-equivalence.spec.ts b/tests/verify/pipeline-equivalence.spec.ts new file mode 100644 index 0000000..1bdc264 --- /dev/null +++ b/tests/verify/pipeline-equivalence.spec.ts @@ -0,0 +1,37 @@ +import { expect, test } from "@playwright/test"; + +import { evaluatePromise } from "../../src/shared/evaluator.js"; +import { adaptExternalEvidence } from "../../src/verify/adapter.js"; +import { runPromiseScenario } from "../support/scenario.js"; +import { projectCanonicalEvidence } from "./canonical-projection.js"; + +function stable(value: unknown): string { + return JSON.stringify(value); +} + +for (const scenario of ["off", "on"] as const) { + test(`real ${scenario.toUpperCase()} evidence survives projection and adaptation exactly`, async ({ + page, + request, + }, testInfo) => { + const result = await runPromiseScenario(page, request, testInfo, scenario, { + runId: `external-equivalence-${scenario}`, + userId: `external-equivalence-subject-${scenario}`, + }); + const roundTripped = evaluatePromise( + adaptExternalEvidence(projectCanonicalEvidence(result.evidence)), + ); + + expect(result.browserErrors).toEqual([]); + expect( + result.evaluation.violations.map((violation) => violation.code), + ).toEqual( + scenario === "on" + ? [] + : testInfo.project.name === "initialization-race" + ? ["PP_IDENTIFIABLE_EVENT_LEAK"] + : ["PP_PREFERENCE_NOT_PERSISTED"], + ); + expect(stable(roundTripped)).toBe(stable(result.evaluation)); + }); +} diff --git a/tests/verify/pipeline.playwright.config.ts b/tests/verify/pipeline.playwright.config.ts new file mode 100644 index 0000000..246274d --- /dev/null +++ b/tests/verify/pipeline.playwright.config.ts @@ -0,0 +1,46 @@ +import { fileURLToPath } from "node:url"; + +import { defineConfig } from "@playwright/test"; + +const mode = process.env.PROMISEPROOF_EQUIVALENCE_MODE; +if (mode !== "initialization-race" && mode !== "propagation-failure") { + throw new Error( + "PROMISEPROOF_EQUIVALENCE_MODE must select a registered demo mode.", + ); +} + +const baseURL = process.env.PROMISEPROOF_BASE_URL ?? "http://127.0.0.1:4173"; +const projectRoot = fileURLToPath(new URL("../..", import.meta.url)); + +export default defineConfig({ + testDir: ".", + testMatch: "pipeline-equivalence.spec.ts", + fullyParallel: false, + workers: 1, + retries: 0, + forbidOnly: Boolean(process.env.CI), + reporter: [["list"]], + projects: [{ name: mode }], + outputDir: fileURLToPath( + new URL(`../../test-results/external-equivalence-${mode}`, import.meta.url), + ), + expect: { timeout: 5_000 }, + use: { + baseURL, + trace: "retain-on-failure", + screenshot: "only-on-failure", + video: "retain-on-failure", + }, + webServer: { + command: + mode === "initialization-race" + ? "npm run dev:test:race" + : "npm run dev:test:propagation", + cwd: projectRoot, + url: `${baseURL}/api/health`, + reuseExistingServer: false, + timeout: 120_000, + stdout: "pipe", + stderr: "pipe", + }, +}); diff --git a/verify.html b/verify.html new file mode 100644 index 0000000..92ec0b3 --- /dev/null +++ b/verify.html @@ -0,0 +1,103 @@ + + + + + + + + + + PromiseProof Verify: re-derive the verdict yourself + + + + + + + + + + + + + + + + + + + + +
    +
    + + + + PromiseProof + Verify — re-derive the verdict yourself. + + +
    +

    + + Runs in your browser. No network, no upload. +

    + Walkthrough → + ← Overview +
    +
    + +
    +
    +

    PromiseProof Verify · Judge Mode

    +

    Re-derive the verdict yourself.

    +

    + A user turned activity-based personalization off. This page runs the + same frozen, deterministic evaluator the repository ships — right here in your + browser, with no network call — over the recorded evidence. It says + PASS. Now tamper the evidence and watch the same evaluator + return a broken promise, while the report you sealed a moment ago refuses to reproduce. +

    +

    + The AI that repairs the promise never owns PASS. An unchanged evaluator does — + and you are about to run it. +

    +
    + +
    +

    + This verifier is interactive and needs JavaScript. The same deterministic evaluator, + evidence, and reports are in the + GitHub repository, and a + plain-language summary is on the overview page. +

    +
    +
    + +
    +

    + Evidence here is externally supplied and synthetic; PromiseProof does not attest how it + was collected. Only the deterministic evaluator decides a bundle that passes strict + validation. Nothing you load or tamper leaves this page. +

    +
    +
    + + + + diff --git a/vite.config.ts b/vite.config.ts index 20c4c11..86d1602 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -12,9 +12,12 @@ export default defineConfig({ emptyOutDir: true, rollupOptions: { input: { - // The seeded Signal Shelf app and the separate judge walkthrough. + // The seeded Signal Shelf app, the judge walkthrough, and the hosted + // deterministic verifier (Judge Mode). build:site builds each page with + // the base its sub-path needs. main: fileURLToPath(new URL("./index.html", import.meta.url)), judge: fileURLToPath(new URL("./judge.html", import.meta.url)), + verify: fileURLToPath(new URL("./verify.html", import.meta.url)), }, }, },