From d5e4ad19243b17ab63428c1e44a4b9e975cbb7b9 Mon Sep 17 00:00:00 2001 From: Atelier Protocol Experiment Date: Wed, 29 Jul 2026 11:07:54 -0700 Subject: [PATCH 1/3] feat(audit): record explicit operator acceptance Add live delivery auditing across the finite evidence registry, preserve visible feedback dispositions, and fence acceptance to one current mailbox revision, receipt, candidate, policy, ticket, and observation. Reconstruct historical acceptance from Git and report later drift without rewriting it. --- contract_tests/test_audit.py | 294 +++++++++ scripts/validate_repository.py | 14 +- skills/atelier/SKILL.md | 19 +- skills/atelier/references/audit.md | 50 ++ skills/atelier/scripts/audit.py | 943 +++++++++++++++++++++++++++++ 5 files changed, 1307 insertions(+), 13 deletions(-) create mode 100644 contract_tests/test_audit.py create mode 100644 skills/atelier/references/audit.md create mode 100644 skills/atelier/scripts/audit.py diff --git a/contract_tests/test_audit.py b/contract_tests/test_audit.py new file mode 100644 index 00000000..5635471c --- /dev/null +++ b/contract_tests/test_audit.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import json +import unittest +from datetime import timedelta + +from contract_tests import test_claiming +from skills.atelier.scripts.audit import AuditCoordinator +from skills.atelier.scripts.git_mailbox import MailboxTransitionRejected +from skills.atelier.scripts.mailbox import _read_yaml, reconstruct_mailbox + + +class AuditContract(unittest.TestCase): + def setUp(self) -> None: + self.fixture = test_claiming.ClaimingContract(methodName="runTest") + self.fixture.setUp() + self.addCleanup(self.fixture.tearDown) + self.fixture.test_delegation_delivers_one_exact_ready_pull_request() + self.audit = AuditCoordinator(self.fixture.coordinator) + self.observation_not_before = self.fixture.live_at + timedelta(minutes=3) + self.now = self.fixture.live_at + timedelta(minutes=4) + self.read_count = 0 + + def report(self): + return self.audit.audit( + self.fixture.work_id, + policy_target=self.fixture.policy_target(), + host_target=self.fixture.delegation_host_target(), + observation_path=self.fixture.observation_path, + observation_not_before=self.observation_not_before, + now=self.now, + ) + + def accept(self, report, *, confirmed: bool = True): + return self.audit.accept( + self.fixture.work_id, + report.fence, + confirmed=confirmed, + accepted_at=self.now, + policy_target=self.fixture.policy_target(), + host_target=self.fixture.delegation_host_target(), + observation_path=self.fixture.observation_path, + observation_not_before=self.observation_not_before, + now=self.now, + ) + + def mailbox_main(self) -> str: + return test_claiming.git( + None, + "--git-dir", + str(self.fixture.mailbox_remote), + "rev-parse", + "main", + ).stdout.strip() + + def current_work(self) -> dict: + self.read_count += 1 + checkout = self.fixture.mailbox_clone(f"audit-read-{self.read_count}") + work, _ = _read_yaml( + checkout / f"work/{self.fixture.work_id}/work.md", + frontmatter=True, + label="work", + ) + reconstruct_mailbox(checkout) + return work + + def live_observation(self) -> dict: + return json.loads(self.fixture.observation_path.read_text(encoding="utf-8")) + + def write_observation(self, value: dict) -> None: + test_claiming.write_json(self.fixture.observation_path, value) + + def test_live_audit_and_explicit_acceptance_are_exact_and_distinct(self) -> None: + report = self.report() + self.assertEqual(report.overall_verdict, "needs-decision") + self.assertTrue(report.acceptance_possible) + self.assertEqual(report.work_status, "delivered") + self.assertIsNone(report.acceptance_commit) + self.assertEqual( + {item.name: item.verdict for item in report.evidence if item.required}, + { + name: "satisfied" + for name in test_claiming.APPROVED_EVIDENCE + }, + ) + + write = self.accept(report) + work = self.current_work() + self.assertEqual(work["status"], "accepted") + self.assertIsNone(work["claim"]) + self.assertEqual(work["acceptance"]["receipt_id"], report.receipt_id) + self.assertEqual( + work["acceptance"]["candidate_revision"], + report.candidate_revision, + ) + self.assertEqual( + work["acceptance"]["evidence"], + { + name: "satisfied" + for name in test_claiming.APPROVED_EVIDENCE + }, + ) + self.assertEqual(work["delivery_receipt_id"], report.receipt_id) + + accepted = self.report() + self.assertEqual(accepted.work_status, "accepted") + self.assertEqual(accepted.overall_verdict, "satisfied") + self.assertEqual(accepted.acceptance_commit, write.commit) + + def test_acceptance_requires_the_explicit_current_audit_fence(self) -> None: + report = self.report() + before = self.mailbox_main() + with self.assertRaisesRegex( + MailboxTransitionRejected, + "operator acceptance was not explicitly confirmed", + ): + self.accept(report, confirmed=False) + self.assertEqual(self.mailbox_main(), before) + changed = self.live_observation() + changed["pull_request"]["head"]["sha"] = "c" * 40 + for check in changed["checks"]: + check["candidate_sha"] = "c" * 40 + self.write_observation(changed) + before = self.mailbox_main() + + with self.assertRaisesRegex( + MailboxTransitionRejected, + "current audit does not match the explicitly confirmed acceptance fence", + ): + self.accept(report) + self.assertEqual(self.mailbox_main(), before) + + def test_unknown_or_violated_evidence_cannot_be_accepted(self) -> None: + unknown_path = self.fixture.root / "missing-observation.json" + unknown = self.audit.audit( + self.fixture.work_id, + policy_target=self.fixture.policy_target(), + host_target=self.fixture.delegation_host_target(), + observation_path=unknown_path, + observation_not_before=self.observation_not_before, + now=self.now, + ) + self.assertEqual(unknown.overall_verdict, "authority-unreconstructable") + self.assertFalse(unknown.acceptance_possible) + self.assertEqual( + { + item.verdict + for item in unknown.evidence + if item.name + in { + "pull-request-head-current", + "pull-request-open", + "pull-request-mergeable", + "required-checks-pass", + "unresolved-feedback-zero", + } + }, + {"unknown"}, + ) + + live = self.live_observation() + live["pull_request"]["mergeable"] = "CONFLICTING" + live["threads"] = [ + { + "id": "thread-1", + "pull_request_number": live["pull_request"]["number"], + "is_resolved": False, + "is_outdated": False, + "path": "example.py", + "line": 1, + "start_line": 1, + "comments": [ + { + "id": "thread-comment-1", + "author": "reviewer", + "body": "This material concern still needs a decision.", + "created_at": live["observed_at"], + "updated_at": live["observed_at"], + "url": live["pull_request"]["url"] + "#discussion_r1", + } + ], + } + ] + self.write_observation(live) + violated = self.report() + feedback = { + item.name: item.verdict + for item in violated.evidence + } + self.assertEqual(feedback["unresolved-feedback-zero"], "violated") + self.assertFalse(violated.acceptance_possible) + before = self.mailbox_main() + with self.assertRaisesRegex( + MailboxTransitionRejected, + "acceptance is blocked by current audit verdict violated", + ): + self.accept(violated) + self.assertEqual(self.mailbox_main(), before) + + def test_material_ticket_drift_is_stale_and_blocks_acceptance(self) -> None: + live = self.live_observation() + live["issue"]["body"] += "\n\nMaterial contract change." + self.write_observation(live) + + report = self.report() + self.assertEqual(report.ticket_verdict, "stale") + self.assertEqual(report.overall_verdict, "stale") + self.assertFalse(report.acceptance_possible) + before = self.mailbox_main() + with self.assertRaisesRegex( + MailboxTransitionRejected, + "acceptance is blocked by current audit verdict stale", + ): + self.accept(report) + self.assertEqual(self.mailbox_main(), before) + + def test_review_dispositions_remain_visible_without_becoming_failure(self) -> None: + live = self.live_observation() + live["pull_request_comments"] = [ + { + "id": "comment-1", + "author": "reviewer", + "body": "P3 deferred to a focused follow-up.", + "created_at": live["observed_at"], + "updated_at": live["observed_at"], + "url": live["pull_request"]["url"] + "#issuecomment-1", + } + ] + live["threads"] = [ + { + "id": "thread-1", + "pull_request_number": live["pull_request"]["number"], + "is_resolved": True, + "is_outdated": False, + "path": "example.py", + "line": 1, + "start_line": 1, + "comments": [ + { + "id": "thread-comment-1", + "author": "reviewer", + "body": "Deliberately deferred; the delivery remains clean.", + "created_at": live["observed_at"], + "updated_at": live["observed_at"], + "url": live["pull_request"]["url"] + "#discussion_r1", + } + ], + } + ] + self.write_observation(live) + + report = self.report() + dispositions = { + (item.kind, item.identifier): (item.disposition, item.body) + for item in report.feedback + } + self.assertEqual( + dispositions[("pull-request-comment", "comment-1")], + ("recorded", "P3 deferred to a focused follow-up."), + ) + self.assertEqual( + dispositions[("review-thread", "thread-1")], + ("resolved", "Deliberately deferred; the delivery remains clean."), + ) + self.assertEqual( + next( + item.verdict + for item in report.evidence + if item.name == "unresolved-feedback-zero" + ), + "satisfied", + ) + self.assertTrue(report.acceptance_possible) + + def test_later_head_drift_changes_audit_without_rewriting_acceptance(self) -> None: + accepted = self.accept(self.report()) + original = self.current_work()["acceptance"] + live = self.live_observation() + live["pull_request"]["head"]["sha"] = "c" * 40 + for check in live["checks"]: + check["candidate_sha"] = "c" * 40 + self.write_observation(live) + + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(report.acceptance_commit, accepted.commit) + self.assertEqual(report.overall_verdict, "stale") + self.assertEqual(verdicts["pull-request-head-current"], "stale") + self.assertEqual(verdicts["pull-request-mergeable"], "stale") + self.assertEqual(self.current_work()["acceptance"], original) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index c5848036..e40a6067 100755 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -82,11 +82,17 @@ def validate_skill(errors: list[str]) -> None: errors.append("Atelier skill frontmatter must include a description") if "[TODO:" in text: errors.append("Atelier skill contains a TODO placeholder") - if "Production `audit` behavior is not implemented yet" not in text: - errors.append("skill must state that audit is unavailable") - for relative in ("references/delegation.md", "scripts/delegation.py"): + if "Production `audit` behavior is implemented" not in text: + errors.append("skill must publish the audit boundary") + required = ( + ("delegated work", "references/delegation.md"), + ("delegated work", "scripts/delegation.py"), + ("audit", "references/audit.md"), + ("audit", "scripts/audit.py"), + ) + for boundary, relative in required: if not (path.parent / relative).is_file(): - errors.append(f"delegated work boundary missing: {relative}") + errors.append(f"{boundary} boundary missing: {relative}") def validate_reset_boundary(errors: list[str]) -> None: diff --git a/skills/atelier/SKILL.md b/skills/atelier/SKILL.md index 3ad76cd3..20892a68 100644 --- a/skills/atelier/SKILL.md +++ b/skills/atelier/SKILL.md @@ -61,15 +61,16 @@ result. Launch one fresh worker through the host with the installed substitute CLI process, cache provider observations, or widen Atelier's v0 authority ceiling. -Production `audit` behavior is not implemented yet. When it is requested: - -1. Report that audit is unavailable in the reset scaffold. -1. Link the owning issue: shaug/atelier#780. -1. Make no mailbox, repository, ticket, pull-request, or acceptance mutation. - -An explicit host-readiness request may complete the read-only host preflight and -return its exact compatibility or failure result. That does not make audit -available. +Production `audit` behavior is implemented for one delivered or accepted +assignment. Before audit, read `references/audit.md`, repeat the fail-closed host +preflight, obtain one fresh complete GitHub observation, and use +`scripts/audit.py audit`. Preserve every predicate verdict and visible feedback +disposition in the report; do not infer success from unavailable or stale state. + +Audit itself is read-only. Only after the operator explicitly confirms the +report's exact acceptance fence may `scripts/audit.py accept` publish the one +acceptance commit. Acceptance never implies merge, deployment, native-ticket +mutation, or issue closure. ## Mailbox boundary diff --git a/skills/atelier/references/audit.md b/skills/atelier/references/audit.md new file mode 100644 index 00000000..7ffc5f50 --- /dev/null +++ b/skills/atelier/references/audit.md @@ -0,0 +1,50 @@ +# Audit and acceptance + +Use this boundary only for one validated mailbox assignment whose current state +is `delivered` or `accepted`. + +## Live audit + +1. Complete the host preflight in `host-boundary.md`. +2. Read the canonical mailbox through `GitMailboxWriter.observe`; do not reuse a + prior clone or write an audit projection. +3. Generate one complete `atelier.github-observation/v1` snapshot at the live + read boundary. It must include the exact issue relationships, pull request, + comments, reviews, checks, and review threads. +4. Invoke `scripts/audit.py audit` with the mailbox remote and branch, work ID, + policy target, host target, observation path, and observation lower bound. +5. Present the complete report. Every registry predicate is classified as + `satisfied`, `violated`, `unknown`, or `stale`; the report also preserves + receipt obligations and the visible disposition and body of every review, + pull-request comment, and thread. + +The report reconstructs the exact delivered receipt, remote candidate, pull +request, head and comparison base, approved and current policy, native ticket +observation, required validation, independent `review-code-change` result, and +current feedback. An accepted assignment additionally derives the Git commit +that introduced acceptance and the preceding delivery observation. Later drift +changes the current audit verdict; it never rewrites historical acceptance. + +Audit is read-only. Missing host capability or project-policy identity reports +`authority-unreconstructable`. Missing live evidence reports `unknown`. +Contradictory state reports `violated`; changed candidate-bound state reports +`stale`. + +## Explicit acceptance + +Acceptance is a second operation. Do not infer it from a clean audit. + +1. Show the operator the report and its `acceptance_fence`. +2. Require explicit confirmation of that exact fence. +3. Invoke `scripts/audit.py accept` with the same fresh-read inputs, the complete + fence, `--accepted-at`, and `--confirm`. +4. The transition rereads the canonical mailbox, host boundary, policy, ticket, + candidate, pull request, checks, reviews, comments, and threads. It rejects + any changed report digest, mailbox revision, receipt, candidate, policy, + ticket, or evidence. +5. Only when every currently required predicate is `satisfied` does one verified + mailbox commit move `delivered` to `accepted`, clear the claim, and bind the + operator record to the exact receipt and candidate. + +Acceptance does not merge a pull request, deploy, mutate or close a native +ticket, delete a branch, or accept a parent initiative. diff --git a/skills/atelier/scripts/audit.py b/skills/atelier/scripts/audit.py new file mode 100644 index 00000000..7e63da49 --- /dev/null +++ b/skills/atelier/scripts/audit.py @@ -0,0 +1,943 @@ +#!/usr/bin/env python3 +"""Live delivery audit and explicit operator acceptance for Atelier v0.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import sys +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Mapping, Sequence + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from skills.atelier.scripts.claiming import ( + ClaimCoordinator, + HostTarget, + _effective_policy, + _read_policy_at_commit, + _read_work, + _read_work_at, + _require_policy_identity, + _timestamp, + _work_path, +) +from skills.atelier.scripts.git_mailbox import ( + FileChange, + MailboxTransitionRejected, + TransitionContext, + TransitionPlan, + WriteResult, + run_git, +) +from skills.atelier.scripts.mailbox import _read_yaml +from skills.atelier.scripts.planning import ( + PlanningError, + PolicyTarget, + _read_current_policy, + _read_project, + _render_document, + _ticket_material_digest, + _validated_observation, +) + +EVIDENCE_NAMES = ( + "candidate-remote-reachable", + "pull-request-head-current", + "pull-request-open", + "pull-request-mergeable", + "required-checks-pass", + "required-validation-reported", + "independent-review-current", + "unresolved-feedback-zero", +) + + +class AuditError(RuntimeError): + """The delivery promise cannot be audited without guessing.""" + + +@dataclass(frozen=True) +class EvidenceResult: + name: str + verdict: str + required: bool + detail: str + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "verdict": self.verdict, + "required": self.required, + "detail": self.detail, + } + + +@dataclass(frozen=True) +class FeedbackItem: + kind: str + identifier: str + disposition: str + body: str + url: str | None + + def as_dict(self) -> dict[str, Any]: + return { + "kind": self.kind, + "identifier": self.identifier, + "disposition": self.disposition, + "body": self.body, + "url": self.url, + } + + +@dataclass(frozen=True) +class AcceptanceFence: + report_digest: str + mailbox_revision: str + receipt_id: str + candidate_revision: str + + def as_dict(self) -> dict[str, str]: + return { + "report_digest": self.report_digest, + "mailbox_revision": self.mailbox_revision, + "receipt_id": self.receipt_id, + "candidate_revision": self.candidate_revision, + } + + +@dataclass(frozen=True) +class AuditReport: + work_id: str + mailbox_revision: str + work_status: str + receipt_id: str + candidate_revision: str + comparison_base_revision: str + approved_policy_commit: str + current_policy_commit: str | None + observed_at: str | None + ticket_verdict: str + ticket_detail: str + acceptance_commit: str | None + evidence: tuple[EvidenceResult, ...] + feedback: tuple[FeedbackItem, ...] + authority_errors: tuple[str, ...] + + @property + def overall_verdict(self) -> str: + if self.authority_errors: + return "authority-unreconstructable" + if self.ticket_verdict != "satisfied": + return self.ticket_verdict + required = [item.verdict for item in self.evidence if item.required] + for verdict in ("violated", "stale", "unknown"): + if verdict in required: + return verdict + if self.work_status == "delivered": + return "needs-decision" + return "satisfied" + + @property + def acceptance_possible(self) -> bool: + return ( + self.work_status == "delivered" + and not self.authority_errors + and self.ticket_verdict == "satisfied" + and all(item.verdict == "satisfied" for item in self.evidence if item.required) + ) + + def as_dict(self, *, include_digest: bool = True) -> dict[str, Any]: + value = { + "schema": "atelier.audit-report/v1", + "work_id": self.work_id, + "mailbox_revision": self.mailbox_revision, + "work_status": self.work_status, + "receipt_id": self.receipt_id, + "candidate_revision": self.candidate_revision, + "comparison_base_revision": self.comparison_base_revision, + "approved_policy_commit": self.approved_policy_commit, + "current_policy_commit": self.current_policy_commit, + "observed_at": self.observed_at, + "ticket": { + "verdict": self.ticket_verdict, + "detail": self.ticket_detail, + }, + "acceptance_commit": self.acceptance_commit, + "evidence": [item.as_dict() for item in self.evidence], + "feedback": [item.as_dict() for item in self.feedback], + "authority_errors": list(self.authority_errors), + "overall_verdict": self.overall_verdict, + "acceptance_possible": self.acceptance_possible, + } + if include_digest: + value["report_digest"] = self.digest + value["acceptance_fence"] = self.fence.as_dict() + return value + + @property + def digest(self) -> str: + encoded = json.dumps( + self.as_dict(include_digest=False), + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(encoded).hexdigest() + + @property + def fence(self) -> AcceptanceFence: + return AcceptanceFence( + report_digest=self.digest, + mailbox_revision=self.mailbox_revision, + receipt_id=self.receipt_id, + candidate_revision=self.candidate_revision, + ) + + +class AuditCoordinator: + """Reconstruct one delivery promise and record a separately confirmed acceptance.""" + + def __init__(self, claims: ClaimCoordinator): + self.claims = claims + + def audit( + self, + work_id: str, + *, + policy_target: PolicyTarget, + host_target: HostTarget, + observation_path: Path, + observation_not_before: datetime, + now: datetime | None = None, + ) -> AuditReport: + return self.claims.writer.observe( + "audit delivery", + lambda context: self._report( + context, + work_id, + policy_target=policy_target, + host_target=host_target, + observation_path=observation_path, + observation_not_before=observation_not_before, + now=now, + ), + ) + + def accept( + self, + work_id: str, + fence: AcceptanceFence, + *, + confirmed: bool, + accepted_at: datetime, + policy_target: PolicyTarget, + host_target: HostTarget, + observation_path: Path, + observation_not_before: datetime, + now: datetime | None = None, + ) -> WriteResult: + if not confirmed: + raise MailboxTransitionRejected("operator acceptance was not explicitly confirmed") + if accepted_at.utcoffset() is None: + raise AuditError("accepted_at must include a UTC offset") + if observation_not_before.utcoffset() is None: + raise AuditError("observation_not_before must include a UTC offset") + if accepted_at < observation_not_before: + raise AuditError("accepted_at cannot precede the live observation boundary") + if now is not None and accepted_at > now: + raise AuditError("accepted_at cannot be in the future") + planned: dict[str, Any] = {} + + def revalidate(context: TransitionContext) -> None: + report = self._report( + context, + work_id, + policy_target=policy_target, + host_target=host_target, + observation_path=observation_path, + observation_not_before=observation_not_before, + now=now, + ) + if report.fence != fence: + raise MailboxTransitionRejected( + "current audit does not match the explicitly confirmed acceptance fence" + ) + if report.observed_at is not None and accepted_at < _parse_timestamp( + report.observed_at + ): + raise MailboxTransitionRejected( + "operator acceptance cannot precede the current live observation" + ) + if not report.acceptance_possible: + raise MailboxTransitionRejected( + f"acceptance is blocked by current audit verdict {report.overall_verdict}" + ) + work, body = _read_work(context.checkout, work_id) + if work["status"] != "delivered" or work["acceptance"] is not None: + raise MailboxTransitionRejected(f"{work_id}: work is not awaiting acceptance") + current_policy_commit = report.current_policy_commit + if current_policy_commit is None: + raise MailboxTransitionRejected("current policy identity is unavailable") + accepted = copy.deepcopy(work) + accepted["status"] = "accepted" + accepted["claim"] = None + accepted["acceptance"] = { + "receipt_id": report.receipt_id, + "accepted_by": "operator", + "accepted_at": _timestamp(accepted_at), + "policy_commit": current_policy_commit, + "candidate_revision": report.candidate_revision, + "evidence": { + item.name: "satisfied" + for item in report.evidence + if item.required + }, + } + planned.clear() + planned.update(work=accepted, body=body) + + def plan(context: TransitionContext) -> TransitionPlan: + del context + return TransitionPlan( + commit_message=f"accept delivered work {work_id}", + changes=( + FileChange( + _work_path(work_id), + _render_document(planned["work"], planned["body"]), + ), + ), + ) + + return self.claims.writer.publish( + "accept delivery", + revalidate=revalidate, + plan=plan, + ) + + def _report( + self, + context: TransitionContext, + work_id: str, + *, + policy_target: PolicyTarget, + host_target: HostTarget, + observation_path: Path, + observation_not_before: datetime, + now: datetime | None, + ) -> AuditReport: + work, body = _read_work(context.checkout, work_id) + if work["status"] not in {"delivered", "accepted"}: + raise MailboxTransitionRejected( + f"{work_id}: audit requires delivered or accepted work" + ) + receipt_id = work["delivery_receipt_id"] + if receipt_id is None: + raise MailboxTransitionRejected(f"{work_id}: delivery receipt is missing") + receipt, _ = _read_yaml( + context.checkout / f"work/{work_id}/receipts/{receipt_id}.md", + frontmatter=True, + label=f"work/{work_id}/receipts/{receipt_id}.md", + ) + candidate = receipt["candidate"] + if candidate is None: + raise MailboxTransitionRejected(f"{work_id}: delivery candidate is missing") + approval = work["approval"] + if approval is None: + raise MailboxTransitionRejected(f"{work_id}: approved contract is missing") + project = _read_project(context.checkout, work["project_id"]) + authority_errors: list[str] = [] + observation: dict[str, Any] | None = None + effective_policy: dict[str, Any] | None = None + current_policy_commit: str | None = None + + try: + self.claims._verify_capability(host_target) + except Exception as error: + authority_errors.append(f"host capability: {error}") + try: + observation = _validated_observation( + observation_path, + not_before=observation_not_before, + now=now, + ) + except Exception as error: + authority_errors.append(f"live GitHub observation: {error}") + try: + self.claims._require_approved_commit( + context, + work_id, + work, + body, + receipt["approved_commit"], + ) + if not self.claims.policy_remote_verifier( + policy_target, + project["repository"], + ): + raise AuditError( + "policy remote is foreign or unverifiable for the managed project" + ) + approved_policy = _read_policy_at_commit( + policy_target, + approval["policy"]["commit"], + ) + current_policy = _read_current_policy(policy_target) + effective_policy = _effective_policy(approved_policy, current_policy.value) + _require_policy_identity( + effective_policy, + current_policy, + project=project, + work=work, + mailbox_remote=self.claims.remote, + mailbox_branch=self.claims.branch, + mailbox_realm=context.snapshot["realm_id"], + approval=approval, + ) + current_policy_commit = current_policy.commit + except Exception as error: + authority_errors.append(f"project policy: {error}") + + acceptance_commit, baseline_ticket_digest = _acceptance_history( + context, + work_id, + work, + ) + if baseline_ticket_digest is None: + authority_errors.append("delivery ticket observation is unreconstructable") + ticket_verdict, ticket_detail = _ticket_verdict( + work, + project, + observation, + effective_policy, + baseline_ticket_digest, + ) + required_evidence = tuple( + ( + effective_policy["acceptance"]["evidence"] + if effective_policy is not None + else approval["acceptance"]["required_evidence"] + ) + ) + verdicts = _evaluate_evidence( + candidate, + receipt, + observation, + effective_policy, + self.claims, + ) + feedback = _feedback_items(receipt, observation) + return AuditReport( + work_id=work_id, + mailbox_revision=context.base_revision, + work_status=work["status"], + receipt_id=receipt_id, + candidate_revision=candidate["head_revision"], + comparison_base_revision=candidate["base_revision"], + approved_policy_commit=approval["policy"]["commit"], + current_policy_commit=current_policy_commit, + observed_at=observation["observed_at"] if observation is not None else None, + ticket_verdict=ticket_verdict, + ticket_detail=ticket_detail, + acceptance_commit=acceptance_commit, + evidence=tuple( + EvidenceResult( + name=name, + verdict=verdicts[name][0], + required=name in required_evidence, + detail=verdicts[name][1], + ) + for name in EVIDENCE_NAMES + ), + feedback=feedback, + authority_errors=tuple(authority_errors), + ) + + +def _acceptance_history( + context: TransitionContext, + work_id: str, + work: Mapping[str, Any], +) -> tuple[str | None, str | None]: + if work["status"] == "delivered": + claim = work["claim"] + return ( + None, + claim["ticket_observation_digest"] if claim is not None else None, + ) + acceptance = work["acceptance"] + if acceptance is None: + return None, None + path = _work_path(work_id) + history = run_git( + context.checkout, + ("log", "--format=%H", "--", path), + ) + if history.returncode != 0: + return None, None + for commit in history.stdout.splitlines(): + try: + current, _ = _read_work_at(context.checkout, commit, path) + except Exception: + continue + if current["acceptance"] != acceptance: + continue + parent = run_git(context.checkout, ("rev-parse", f"{commit}^")) + if parent.returncode != 0: + continue + try: + previous, _ = _read_work_at( + context.checkout, + parent.stdout.strip(), + path, + ) + except Exception: + continue + if previous["acceptance"] is None: + claim = previous["claim"] + return ( + commit, + claim["ticket_observation_digest"] if claim is not None else None, + ) + return None, None + + +def _ticket_verdict( + work: Mapping[str, Any], + project: Mapping[str, Any], + observation: Mapping[str, Any] | None, + policy: Mapping[str, Any] | None, + baseline_digest: str | None, +) -> tuple[str, str]: + if observation is None: + return "unknown", "current native ticket state is unavailable" + repository = observation["repository"]["name_with_owner"] + if repository != project["repository"].removeprefix("github:"): + return "violated", "live repository does not identify the managed project" + ticket = work["native_ticket"] + issue = observation["issue"] + if ( + ticket is None + or ticket["provider"] != "github" + or ticket["id"] != str(issue["number"]) + or ticket["url"] != issue["url"] + ): + return "violated", "live native ticket does not identify the approved assignment" + if policy is None: + return "unknown", "effective project policy is unavailable" + if issue["state"].lower() not in policy["ticket"]["allowed_states"]: + return "violated", f"ticket state {issue['state']} is not allowed" + if policy["ticket"]["require_no_blockers"]: + blockers = [item for item in issue["blocked_by"] if item["state"] != "CLOSED"] + if blockers: + numbers = ", ".join(f"#{item['number']}" for item in blockers) + return "violated", f"native ticket has unresolved blockers: {numbers}" + current_digest = _ticket_material_digest(issue, policy) + if baseline_digest is None: + return "unknown", "delivery ticket observation cannot be reconstructed" + if current_digest != baseline_digest: + return "stale", "material native ticket state changed after the delivery observation" + return "satisfied", "native ticket identity, eligibility, and material state are current" + + +def _evaluate_evidence( + candidate: Mapping[str, Any], + receipt: Mapping[str, Any], + observation: Mapping[str, Any] | None, + policy: Mapping[str, Any] | None, + claims: ClaimCoordinator, +) -> dict[str, tuple[str, str]]: + values: dict[str, tuple[str, str]] = {} + try: + reachable = claims.candidate_verifier(candidate) + except Exception as error: + values["candidate-remote-reachable"] = ( + "unknown", + f"candidate remote cannot be read: {error}", + ) + else: + values["candidate-remote-reachable"] = ( + "satisfied" if reachable else "violated", + ( + "delivered SHA is reachable from the declared remote ref" + if reachable + else "declared remote ref does not contain the delivered SHA" + ), + ) + + pull_request = observation["pull_request"] if observation is not None else None + if observation is None: + for name in ( + "pull-request-head-current", + "pull-request-open", + "pull-request-mergeable", + "required-checks-pass", + "unresolved-feedback-zero", + ): + values[name] = ("unknown", "current GitHub state is unavailable") + elif pull_request is None: + values["pull-request-head-current"] = ( + "violated", + "delivered pull request no longer exists in the live observation", + ) + values["pull-request-open"] = ("violated", "delivered pull request is absent") + values["pull-request-mergeable"] = ( + "unknown", + "mergeability is unavailable without a live pull request", + ) + values["required-checks-pass"] = ( + "unknown", + "required check results are unavailable without a live pull request", + ) + values["unresolved-feedback-zero"] = ( + "unknown", + "thread-aware feedback is unavailable without a live pull request", + ) + else: + expected_repository = candidate["repository"].removeprefix("github:") + live_head = pull_request["head"] + same_pr = pull_request["url"] == candidate["pull_request"] + same_repository_ref = ( + live_head["repository"] == expected_repository + and live_head["ref"] == candidate["remote_ref"] + ) + same_head = live_head["sha"] == candidate["head_revision"] + if not same_pr or not same_repository_ref: + values["pull-request-head-current"] = ( + "violated", + "live pull request repository, ref, or URL differs from the delivery", + ) + elif not same_head: + values["pull-request-head-current"] = ( + "stale", + "live pull request head changed after delivery", + ) + else: + values["pull-request-head-current"] = ( + "satisfied", + "live pull request repository, ref, and head match the delivery", + ) + values["pull-request-open"] = ( + ( + "satisfied", + "live pull request is open", + ) + if pull_request["state"] == "OPEN" + else ( + "violated", + f"live pull request state is {pull_request['state']}", + ) + ) + if not same_head: + values["pull-request-mergeable"] = ( + "stale", + "mergeability belongs to a different pull request head", + ) + elif pull_request["mergeable"] == "MERGEABLE": + values["pull-request-mergeable"] = ( + "satisfied", + "GitHub reports the exact delivered head mergeable", + ) + elif pull_request["mergeable"] == "CONFLICTING": + values["pull-request-mergeable"] = ( + "violated", + "GitHub reports a merge conflict", + ) + else: + values["pull-request-mergeable"] = ( + "unknown", + "GitHub mergeability is unavailable", + ) + values["required-checks-pass"] = _checks_verdict( + observation["checks"], + candidate["head_revision"], + ) + values["unresolved-feedback-zero"] = _feedback_verdict( + receipt, + observation, + ) + + values["required-validation-reported"] = _validation_verdict( + receipt, + candidate["head_revision"], + ( + policy["validation"]["required_commands"] + if policy is not None + else () + ), + ) + values["independent-review-current"] = _review_verdict( + receipt, + candidate["head_revision"], + candidate["base_revision"], + ) + return values + + +def _checks_verdict( + checks: Sequence[Mapping[str, Any]], + candidate_revision: str, +) -> tuple[str, str]: + if any(check["candidate_sha"] != candidate_revision for check in checks): + return "stale", "one or more required checks belong to another candidate" + incomplete = [ + check + for check in checks + if check["status"].upper() != "COMPLETED" or check["conclusion"] is None + ] + if incomplete: + return "unknown", "one or more required check results are incomplete" + failing = [ + check + for check in checks + if check["conclusion"].upper() not in {"SUCCESS", "NEUTRAL", "SKIPPED"} + ] + if failing: + return "violated", "one or more required checks completed unsuccessfully" + return "satisfied", "all observed required checks passed on the delivered head" + + +def _validation_verdict( + receipt: Mapping[str, Any], + candidate_revision: str, + required_commands: Sequence[str], +) -> tuple[str, str]: + observations = { + item["command"]: item + for item in receipt["validation"] + if item["command"] in required_commands + } + missing = [command for command in required_commands if command not in observations] + if missing: + return "unknown", f"required validation is missing: {', '.join(missing)}" + selected = [observations[command] for command in required_commands] + if any(item["candidate_revision"] != candidate_revision for item in selected): + return "stale", "required validation belongs to another candidate" + if any(item["outcome"] != "passed" for item in selected): + return "violated", "one or more required validation commands failed" + return "satisfied", "every effective required command passed on the delivered head" + + +def _review_verdict( + receipt: Mapping[str, Any], + candidate_revision: str, + comparison_base_revision: str, +) -> tuple[str, str]: + reviews = [ + item + for item in receipt["reviews"] + if item["mechanism"] == "review-code-change" + ] + if not reviews: + return "unknown", "independent review-code-change evidence is missing" + dated = [(_parse_timestamp(item["observed_at"]), item) for item in reviews] + latest_time = max(observed_at for observed_at, _ in dated) + latest = [item for observed_at, item in dated if observed_at == latest_time] + if any( + item["candidate_revision"] != candidate_revision + or item["comparison_base_revision"] != comparison_base_revision + for item in latest + ): + return "stale", "latest independent review belongs to another head or base" + verdicts = {item["verdict"] for item in latest} + if verdicts == {"clean"}: + return "satisfied", "latest independent review is clean on the exact head and base" + if "changes_required" in verdicts: + return "violated", "latest independent review requires changes" + return "unknown", "latest independent review is blocked or contradictory" + + +def _feedback_verdict( + receipt: Mapping[str, Any], + observation: Mapping[str, Any], +) -> tuple[str, str]: + unresolved_threads = [ + item + for item in observation["threads"] + if not item["is_resolved"] and not item["is_outdated"] + ] + if unresolved_threads: + return "violated", "one or more live review threads remain unresolved" + pull_request = observation["pull_request"] + if pull_request is not None and pull_request["review_decision"] == "CHANGES_REQUESTED": + return "violated", "the live pull request review decision requires changes" + if receipt["unresolved_obligations"]: + return "violated", "the delivered receipt retains unresolved obligations" + return ( + "satisfied", + "no unresolved material receipt obligation, review decision, or live thread remains", + ) + + +def _feedback_items( + receipt: Mapping[str, Any], + observation: Mapping[str, Any] | None, +) -> tuple[FeedbackItem, ...]: + items = [ + FeedbackItem( + kind="receipt-obligation", + identifier=f"obligation-{index}", + disposition="unresolved", + body=value, + url=None, + ) + for index, value in enumerate(receipt["unresolved_obligations"], start=1) + ] + if observation is None: + return tuple(items) + items.extend( + FeedbackItem( + kind="review", + identifier=review["id"], + disposition=review["state"], + body=review["body"], + url=review["url"], + ) + for review in observation["reviews"] + ) + items.extend( + FeedbackItem( + kind="pull-request-comment", + identifier=comment["id"], + disposition="recorded", + body=comment["body"], + url=comment["url"], + ) + for comment in observation["pull_request_comments"] + ) + items.extend( + FeedbackItem( + kind="review-thread", + identifier=thread["id"], + disposition=( + "resolved" + if thread["is_resolved"] + else "outdated" + if thread["is_outdated"] + else "unresolved" + ), + body="\n\n".join(comment["body"] for comment in thread["comments"]), + url=thread["comments"][-1]["url"], + ) + for thread in observation["threads"] + ) + return tuple(items) + + +def _parse_timestamp(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.utcoffset() is None: + raise AuditError("timestamps must include a UTC offset") + return parsed + + +def _json_object(value: str, label: str) -> dict[str, Any]: + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise AuditError(f"{label} must be a JSON object") + return parsed + + +def _policy_target(value: Mapping[str, Any]) -> PolicyTarget: + return PolicyTarget( + checkout=Path(value["checkout"]), + remote=value["remote"], + canonical_ref=value["canonical_ref"], + path=value["path"], + ) + + +def _host_target(value: Mapping[str, Any]) -> HostTarget: + return HostTarget( + descriptor_path=Path(value["descriptor_path"]), + skill_name=value["skill_name"], + skill_root=Path(value["skill_root"]), + connector=value["connector"], + operations=tuple(value["operations"]), + ) + + +def _fence(value: Mapping[str, Any]) -> AcceptanceFence: + return AcceptanceFence( + report_digest=value["report_digest"], + mailbox_revision=value["mailbox_revision"], + receipt_id=value["receipt_id"], + candidate_revision=value["candidate_revision"], + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("audit", "accept"): + subparser = subparsers.add_parser(command) + subparser.add_argument("--mailbox-remote", required=True) + subparser.add_argument("--mailbox-branch", required=True) + subparser.add_argument("--work-id", required=True) + subparser.add_argument("--policy-target", required=True) + subparser.add_argument("--host-target", required=True) + subparser.add_argument("--observation", required=True, type=Path) + subparser.add_argument("--observation-not-before", required=True) + subparser.add_argument("--now") + accept = subparsers.choices["accept"] + accept.add_argument("--fence", required=True) + accept.add_argument("--accepted-at", required=True) + accept.add_argument("--confirm", action="store_true") + return parser + + +def main(arguments: Sequence[str] | None = None) -> int: + parsed = _parser().parse_args(arguments) + claims = ClaimCoordinator(parsed.mailbox_remote, parsed.mailbox_branch) + coordinator = AuditCoordinator(claims) + common = { + "policy_target": _policy_target( + _json_object(parsed.policy_target, "policy target") + ), + "host_target": _host_target( + _json_object(parsed.host_target, "host target") + ), + "observation_path": parsed.observation, + "observation_not_before": _parse_timestamp( + parsed.observation_not_before + ), + "now": _parse_timestamp(parsed.now) if parsed.now else None, + } + try: + if parsed.command == "audit": + value = coordinator.audit(parsed.work_id, **common).as_dict() + else: + result = coordinator.accept( + parsed.work_id, + _fence(_json_object(parsed.fence, "acceptance fence")), + confirmed=parsed.confirm, + accepted_at=_parse_timestamp(parsed.accepted_at), + **common, + ) + value = { + "operation": result.operation, + "branch": result.branch, + "commit": result.commit, + "base_revision": result.base_revision, + "attempts": result.attempts, + "recovered": result.recovered, + } + except ( + AuditError, + MailboxTransitionRejected, + PlanningError, + ValueError, + ) as error: + print(str(error), file=sys.stderr) + return 1 + print(json.dumps(value, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From d185b827515c3525b6a5e44a0ab8f175223e1085 Mon Sep 17 00:00:00 2001 From: Atelier Protocol Experiment Date: Wed, 29 Jul 2026 12:31:38 -0700 Subject: [PATCH 2/3] fix(audit): enforce live acceptance evidence Bind delivery audit to the exact live pull-request base and effective required-check identities. Preserve structured review findings and body-bound feedback dispositions in acceptance, and require a strictly newer provider snapshot before the operator transition. --- contract_tests/test_audit.py | 215 ++++++++++- contract_tests/test_claiming.py | 1 + contract_tests/test_host_boundary.py | 6 + contract_tests/test_mailbox.py | 12 + contract_tests/test_planning.py | 2 + docs/git-mailbox-contract.md | 19 +- docs/project-policy-contract.md | 44 ++- skills/atelier/SKILL.md | 17 +- skills/atelier/references/audit.md | 52 ++- .../references/github-observation.schema.json | 45 +++ skills/atelier/references/host-boundary.md | 10 +- .../atelier/references/host-capability.json | 1 + .../atelier/references/mailbox-v1.schema.json | 148 +++++++- skills/atelier/scripts/audit.py | 346 +++++++++++++++--- 14 files changed, 820 insertions(+), 98 deletions(-) diff --git a/contract_tests/test_audit.py b/contract_tests/test_audit.py index 5635471c..0c0e0c00 100644 --- a/contract_tests/test_audit.py +++ b/contract_tests/test_audit.py @@ -1,11 +1,12 @@ from __future__ import annotations +import hashlib import json import unittest -from datetime import timedelta +from datetime import UTC, datetime, timedelta from contract_tests import test_claiming -from skills.atelier.scripts.audit import AuditCoordinator +from skills.atelier.scripts.audit import AuditCoordinator, AuditError from skills.atelier.scripts.git_mailbox import MailboxTransitionRejected from skills.atelier.scripts.mailbox import _read_yaml, reconstruct_mailbox @@ -20,18 +21,53 @@ def setUp(self) -> None: self.observation_not_before = self.fixture.live_at + timedelta(minutes=3) self.now = self.fixture.live_at + timedelta(minutes=4) self.read_count = 0 + self.audit_evidence = self.evidence() - def report(self): + def evidence( + self, + *, + findings: list[dict] | None = None, + feedback_dispositions: list[dict] | None = None, + ) -> dict: + live = self.live_observation() + return { + "schema": "atelier.audit-evidence/v1", + "review": { + "mechanism": "review-code-change", + "verdict": "clean", + "candidate_revision": live["pull_request"]["head"]["sha"], + "comparison_base_revision": live["pull_request"]["base"]["sha"], + "observed_at": live["observed_at"], + "findings": findings or [], + }, + "feedback_dispositions": feedback_dispositions or [], + } + + def report(self, *, evidence: dict | None = None): return self.audit.audit( self.fixture.work_id, policy_target=self.fixture.policy_target(), host_target=self.fixture.delegation_host_target(), observation_path=self.fixture.observation_path, observation_not_before=self.observation_not_before, + audit_evidence=evidence or self.audit_evidence, now=self.now, ) - def accept(self, report, *, confirmed: bool = True): + def accept( + self, + report, + *, + confirmed: bool = True, + evidence: dict | None = None, + refresh_observation: bool = True, + ): + boundary = datetime.fromisoformat(report.observed_at.replace("Z", "+00:00")) + boundary += timedelta(seconds=1) + if refresh_observation: + live = self.live_observation() + live["observed_at"] = boundary.isoformat().replace("+00:00", "Z") + self.write_observation(live) return self.audit.accept( self.fixture.work_id, report.fence, @@ -40,7 +76,8 @@ def accept(self, report, *, confirmed: bool = True): policy_target=self.fixture.policy_target(), host_target=self.fixture.delegation_host_target(), observation_path=self.fixture.observation_path, - observation_not_before=self.observation_not_before, + observation_not_before=boundary, + audit_evidence=evidence or self.audit_evidence, now=self.now, ) @@ -80,7 +117,7 @@ def test_live_audit_and_explicit_acceptance_are_exact_and_distinct(self) -> None {item.name: item.verdict for item in report.evidence if item.required}, { name: "satisfied" - for name in test_claiming.APPROVED_EVIDENCE + for name in test_claiming.EVIDENCE }, ) @@ -97,9 +134,10 @@ def test_live_audit_and_explicit_acceptance_are_exact_and_distinct(self) -> None work["acceptance"]["evidence"], { name: "satisfied" - for name in test_claiming.APPROVED_EVIDENCE + for name in test_claiming.EVIDENCE }, ) + self.assertEqual(work["acceptance"]["audit_evidence"], report.audit_evidence) self.assertEqual(work["delivery_receipt_id"], report.receipt_id) accepted = self.report() @@ -138,6 +176,7 @@ def test_unknown_or_violated_evidence_cannot_be_accepted(self) -> None: host_target=self.fixture.delegation_host_target(), observation_path=unknown_path, observation_not_before=self.observation_not_before, + audit_evidence=self.audit_evidence, now=self.now, ) self.assertEqual(unknown.overall_verdict, "authority-unreconstructable") @@ -248,15 +287,39 @@ def test_review_dispositions_remain_visible_without_becoming_failure(self) -> No } ] self.write_observation(live) + evidence = self.evidence( + findings=[ + { + "id": "finding-1", + "summary": "A nonblocking improvement was deliberately deferred.", + "disposition": "deferred", + "rationale": "It is outside this ticket's acceptance boundary.", + "follow_up": "#999", + } + ], + feedback_dispositions=[ + { + "kind": "pull-request-comment", + "id": "comment-1", + "body_digest": "sha256:" + + hashlib.sha256( + b"P3 deferred to a focused follow-up." + ).hexdigest(), + "disposition": "deferred", + "rationale": "Tracked by a focused follow-up.", + "follow_up": "#999", + } + ], + ) - report = self.report() + report = self.report(evidence=evidence) dispositions = { (item.kind, item.identifier): (item.disposition, item.body) for item in report.feedback } self.assertEqual( dispositions[("pull-request-comment", "comment-1")], - ("recorded", "P3 deferred to a focused follow-up."), + ("deferred", "P3 deferred to a focused follow-up."), ) self.assertEqual( dispositions[("review-thread", "thread-1")], @@ -270,8 +333,142 @@ def test_review_dispositions_remain_visible_without_becoming_failure(self) -> No ), "satisfied", ) + self.assertEqual(report.audit_evidence, evidence) self.assertTrue(report.acceptance_possible) + edited = self.live_observation() + edited["pull_request_comments"][0]["body"] += " Edited." + self.write_observation(edited) + stale = self.report(evidence=evidence) + stale_verdicts = {item.name: item.verdict for item in stale.evidence} + self.assertEqual(stale_verdicts["unresolved-feedback-zero"], "stale") + self.assertFalse(stale.acceptance_possible) + + def test_live_base_ref_or_sha_drift_is_stale_and_cannot_be_accepted(self) -> None: + original = self.live_observation() + for field, value in (("ref", "refs/heads/other"), ("sha", "d" * 40)): + with self.subTest(field=field): + live = json.loads(json.dumps(original)) + live["pull_request"]["base"][field] = value + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["independent-review-current"], "stale") + self.assertEqual(verdicts["pull-request-mergeable"], "stale") + self.assertFalse(report.acceptance_possible) + before = self.mailbox_main() + with self.assertRaisesRegex( + MailboxTransitionRejected, + "acceptance is blocked by current audit verdict stale", + ): + self.accept(report) + self.assertEqual(self.mailbox_main(), before) + + def test_required_check_configuration_is_identity_aware_and_fail_closed(self) -> None: + original = self.live_observation() + live = json.loads(json.dumps(original)) + live["required_checks"]["configuration_read"] = False + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["required-checks-pass"], "unknown") + + live = json.loads(json.dumps(original)) + live["checks"] = [ + { + "id": "optional-check", + "pull_request_number": live["pull_request"]["number"], + "kind": "CHECK_RUN", + "name": "optional", + "status": "COMPLETED", + "conclusion": "FAILURE", + "candidate_sha": live["pull_request"]["head"]["sha"], + "details_url": None, + } + ] + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["required-checks-pass"], "satisfied") + + live["required_checks"]["contexts"] = [ + {"kind": "CHECK_RUN", "name": "required"} + ] + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["required-checks-pass"], "unknown") + + live["checks"].append( + { + "id": "required-check", + "pull_request_number": live["pull_request"]["number"], + "kind": "CHECK_RUN", + "name": "required", + "status": "COMPLETED", + "conclusion": "SUCCESS", + "candidate_sha": live["pull_request"]["head"]["sha"], + "details_url": None, + } + ) + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["required-checks-pass"], "satisfied") + + def test_undispositioned_live_comment_blocks_acceptance(self) -> None: + live = self.live_observation() + live["pull_request_comments"] = [ + { + "id": "comment-blocking", + "author": "reviewer", + "body": "This concern has no recorded disposition.", + "created_at": live["observed_at"], + "updated_at": live["observed_at"], + "url": live["pull_request"]["url"] + "#issuecomment-blocking", + } + ] + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["unresolved-feedback-zero"], "unknown") + self.assertFalse(report.acceptance_possible) + before = self.mailbox_main() + with self.assertRaisesRegex( + MailboxTransitionRejected, + "acceptance is blocked by current audit verdict unknown", + ): + self.accept(report) + self.assertEqual(self.mailbox_main(), before) + + def test_acceptance_requires_a_second_provider_snapshot(self) -> None: + report = self.report() + before = self.mailbox_main() + with self.assertRaisesRegex( + MailboxTransitionRejected, + "current audit does not match the explicitly confirmed acceptance fence", + ): + self.accept(report, refresh_observation=False) + self.assertEqual(self.mailbox_main(), before) + + def test_acceptance_without_test_clock_rejects_future_timestamp(self) -> None: + report = self.report() + boundary = datetime.fromisoformat(report.observed_at.replace("Z", "+00:00")) + boundary += timedelta(seconds=1) + with self.assertRaisesRegex(AuditError, "accepted_at cannot be in the future"): + self.audit.accept( + self.fixture.work_id, + report.fence, + confirmed=True, + accepted_at=datetime.now(UTC) + timedelta(minutes=1), + policy_target=self.fixture.policy_target(), + host_target=self.fixture.delegation_host_target(), + observation_path=self.fixture.observation_path, + observation_not_before=boundary, + audit_evidence=self.audit_evidence, + now=None, + ) + def test_later_head_drift_changes_audit_without_rewriting_acceptance(self) -> None: accepted = self.accept(self.report()) original = self.current_work()["acceptance"] diff --git a/contract_tests/test_claiming.py b/contract_tests/test_claiming.py index 62693642..8e4b5d26 100644 --- a/contract_tests/test_claiming.py +++ b/contract_tests/test_claiming.py @@ -73,6 +73,7 @@ "github.pull-request.comments.read", "github.pull-request.reviews.read", "github.pull-request.checks.read", + "github.repository.required-checks.read", "github.pull-request.threads.read", ) diff --git a/contract_tests/test_host_boundary.py b/contract_tests/test_host_boundary.py index 33dc8c73..7d20944c 100644 --- a/contract_tests/test_host_boundary.py +++ b/contract_tests/test_host_boundary.py @@ -22,6 +22,7 @@ "github.pull-request.comments.read", "github.pull-request.reviews.read", "github.pull-request.checks.read", + "github.repository.required-checks.read", "github.pull-request.threads.read", ] HEAD_SHA = "a" * 40 @@ -155,6 +156,10 @@ def complete_observation() -> dict[str, object]: "details_url": "https://github.com/shaug/atelier/actions/runs/1", } ], + "required_checks": { + "configuration_read": True, + "contexts": [{"kind": "CHECK_RUN", "name": "test"}], + }, "threads": [ { "id": "thread-1", @@ -175,6 +180,7 @@ def complete_observation() -> dict[str, object]: "pull_request_comments": True, "reviews": True, "checks": True, + "required_checks": True, "threads": True, }, } diff --git a/contract_tests/test_mailbox.py b/contract_tests/test_mailbox.py index 6f7c1b46..852b8617 100644 --- a/contract_tests/test_mailbox.py +++ b/contract_tests/test_mailbox.py @@ -413,6 +413,18 @@ def add_work( "policy_commit": SHA_A, "candidate_revision": SHA_B, "evidence": {name: "satisfied" for name in EVIDENCE}, + "audit_evidence": { + "schema": "atelier.audit-evidence/v1", + "review": { + "mechanism": "review-code-change", + "verdict": "clean", + "candidate_revision": SHA_B, + "comparison_base_revision": SHA_A, + "observed_at": TIMESTAMP, + "findings": [], + }, + "feedback_dispositions": [], + }, } self.works[work_id] = work self.write_work(work_id) diff --git a/contract_tests/test_planning.py b/contract_tests/test_planning.py index f4e81419..eae14e84 100644 --- a/contract_tests/test_planning.py +++ b/contract_tests/test_planning.py @@ -126,6 +126,7 @@ def observation(*, blocked: bool = False, with_pull_request: bool = False) -> di "pull_request_comments": [], "reviews": [], "checks": [], + "required_checks": {"configuration_read": True, "contexts": []}, "threads": [], "completeness": { "issue": True, @@ -135,6 +136,7 @@ def observation(*, blocked: bool = False, with_pull_request: bool = False) -> di "pull_request_comments": True, "reviews": True, "checks": True, + "required_checks": True, "threads": True, }, } diff --git a/docs/git-mailbox-contract.md b/docs/git-mailbox-contract.md index fc08fec4..85f5402d 100644 --- a/docs/git-mailbox-contract.md +++ b/docs/git-mailbox-contract.md @@ -696,12 +696,23 @@ acceptance: required-validation-reported: satisfied independent-review-current: satisfied unresolved-feedback-zero: satisfied + audit_evidence: + schema: atelier.audit-evidence/v1 + review: + mechanism: review-code-change + verdict: clean + candidate_revision: 2222222222222222222222222222222222222222 + comparison_base_revision: 1111111111111111111111111111111111111111 + observed_at: 2026-07-25T13:14:30Z + findings: [] + feedback_dispositions: [] ``` -Acceptance records what was verified at that commit. If evidence later becomes -unavailable, stale, or contradictory, history is not rewritten. Audit reports -the current promise as `unknown`, `stale`, or `violated` and cites the earlier -acceptance. +Acceptance records what was verified at that commit, including the exact structured +review findings and explicit dispositions for top-level review and comment bodies. +If evidence later becomes unavailable, stale, or contradictory, history is not +rewritten. Audit reports the current promise as `unknown`, `stale`, or `violated` +and cites the earlier acceptance. ## Minimal planner-worker interactions diff --git a/docs/project-policy-contract.md b/docs/project-policy-contract.md index 26427628..38410c30 100644 --- a/docs/project-policy-contract.md +++ b/docs/project-policy-contract.md @@ -255,15 +255,19 @@ Each predicate has one authoritative source and exact evaluation. - **Satisfied:** GitHub reports the PR mergeable. - **Violated:** GitHub reports a conflict. - **Unknown:** mergeability is unavailable or still being calculated. -- **Stale:** the PR head changed after the observation. +- **Stale:** the PR head or exact comparison base changed after the observation. ### `required-checks-pass` -- **Source:** live required GitHub checks for the exact PR head. -- **Satisfied:** every required check completed successfully. -- **Violated:** a required check completed unsuccessfully. -- **Unknown:** required-check configuration or a required result cannot be read. -- **Stale:** the PR head changed after the check result. +- **Source:** live required-check configuration plus check results for the exact PR + head. Required identities are the exact check kind and name obtained from + effective branch protection and repository rulesets. +- **Satisfied:** the required configuration was read and every named required + check completed successfully; an empty configured set is satisfied. +- **Violated:** a named required check completed unsuccessfully. +- **Unknown:** required-check configuration or a named required result cannot be + read, is missing, or is ambiguous. +- **Stale:** a named required result belongs to another PR head. ### `required-validation-reported` @@ -276,20 +280,26 @@ Each predicate has one authoritative source and exact evaluation. ### `independent-review-current` -- **Source:** a structured `review-code-change` result recorded in the receipt. -- **Satisfied:** the result is clean and bound to the delivered SHA and - comparison base. -- **Violated:** the result requires changes. -- **Unknown:** the result is missing, malformed, or blocked. -- **Stale:** the delivered head or effective comparison base changed. +- **Source:** the delivered receipt plus normalized aggregate + `review-code-change` audit evidence with structured finding dispositions. +- **Satisfied:** both records are clean, every finding has a non-unresolved + disposition, and both are bound to the delivered SHA and comparison base. +- **Violated:** the result requires changes or retains an unresolved finding. +- **Unknown:** either record is missing, malformed, blocked, or contradictory. +- **Stale:** the delivered head, live PR base, or effective comparison base changed. ### `unresolved-feedback-zero` -- **Source:** live GitHub review, comment, and thread state. -- **Satisfied:** no unresolved material review, comment, or thread remains. -- **Violated:** an unresolved material item remains. -- **Unknown:** thread-aware feedback cannot be read. -- **Stale:** the PR head changed after a previously clean observation. +- **Source:** live GitHub review, comment, and thread state plus the normalized + audit-evidence dispositions bound to provider IDs and exact body digests. +- **Satisfied:** every nonempty top-level review or comment body has an explicit + non-unresolved disposition and no unresolved material thread or receipt + obligation remains. This predicate is an unconditional v0 acceptance guard, + even when an older approval omitted it from `required_evidence`. +- **Violated:** an item is explicitly unresolved, the live review decision requires + changes, or a material thread or receipt obligation remains unresolved. +- **Unknown:** thread-aware feedback cannot be read or a live body lacks a disposition. +- **Stale:** a disposition identifies missing feedback or an earlier body revision. ## Policy drift diff --git a/skills/atelier/SKILL.md b/skills/atelier/SKILL.md index 20892a68..09db7700 100644 --- a/skills/atelier/SKILL.md +++ b/skills/atelier/SKILL.md @@ -29,7 +29,7 @@ The preflight must prove: the host capability descriptor; - the installed and authorized `github@openai-curated` connector; - every required read-only issue, relationship, pull-request, comment, review, - check, and thread operation; and + check, effective required-check configuration, and thread operation; and - one complete observation conforming to `references/github-observation.schema.json` when live state is requested. @@ -63,14 +63,17 @@ authority ceiling. Production `audit` behavior is implemented for one delivered or accepted assignment. Before audit, read `references/audit.md`, repeat the fail-closed host -preflight, obtain one fresh complete GitHub observation, and use -`scripts/audit.py audit`. Preserve every predicate verdict and visible feedback -disposition in the report; do not infer success from unavailable or stale state. +preflight, obtain one fresh complete GitHub observation, normalize the exact +aggregate review and live feedback dispositions as `atelier.audit-evidence/v1`, +and use `scripts/audit.py audit`. Preserve every predicate verdict, structured +finding, and visible feedback disposition in the report; do not infer success +from unavailable, undispositioned, or stale state. Audit itself is read-only. Only after the operator explicitly confirms the -report's exact acceptance fence may `scripts/audit.py accept` publish the one -acceptance commit. Acceptance never implies merge, deployment, native-ticket -mutation, or issue closure. +report's exact acceptance fence may a strictly newer complete provider snapshot +be supplied to `scripts/audit.py accept` for the one acceptance commit. Reusing +report evidence is rejected. Acceptance never implies merge, deployment, +native-ticket mutation, or issue closure. ## Mailbox boundary diff --git a/skills/atelier/references/audit.md b/skills/atelier/references/audit.md index 7ffc5f50..96481ca0 100644 --- a/skills/atelier/references/audit.md +++ b/skills/atelier/references/audit.md @@ -10,13 +10,24 @@ is `delivered` or `accepted`. prior clone or write an audit projection. 3. Generate one complete `atelier.github-observation/v1` snapshot at the live read boundary. It must include the exact issue relationships, pull request, - comments, reviews, checks, and review threads. -4. Invoke `scripts/audit.py audit` with the mailbox remote and branch, work ID, - policy target, host target, observation path, and observation lower bound. -5. Present the complete report. Every registry predicate is classified as - `satisfied`, `violated`, `unknown`, or `stale`; the report also preserves - receipt obligations and the visible disposition and body of every review, - pull-request comment, and thread. + comments, reviews, checks, effective required-check configuration, and review + threads. A failed ruleset or branch-protection read is represented by + `required_checks.configuration_read: false`; it is never inferred from the + visible check list. +4. Normalize the exact aggregate `review-code-change` result and every deliberate + disposition for a nonempty top-level review or pull-request comment into an + `atelier.audit-evidence/v1` JSON object. Each review finding records an ID, + summary, disposition, rationale, and optional follow-up. Each live review or + comment disposition records its kind, provider ID, exact `sha256:` body digest, + disposition, rationale, and optional follow-up. Omitted, duplicate, foreign, + or body-stale identities fail closed. +5. Invoke `scripts/audit.py audit` with the mailbox remote and branch, work ID, + policy target, host target, observation path and lower bound, and + `--audit-evidence` path. +6. Present the complete report. Every registry predicate is classified as + `satisfied`, `violated`, `unknown`, or `stale`; the report also preserves the + structured review findings, receipt obligations, and explicit disposition and + body of every review, pull-request comment, and thread. The report reconstructs the exact delivered receipt, remote candidate, pull request, head and comparison base, approved and current policy, native ticket @@ -36,15 +47,24 @@ Acceptance is a second operation. Do not infer it from a clean audit. 1. Show the operator the report and its `acceptance_fence`. 2. Require explicit confirmation of that exact fence. -3. Invoke `scripts/audit.py accept` with the same fresh-read inputs, the complete - fence, `--accepted-at`, and `--confirm`. -4. The transition rereads the canonical mailbox, host boundary, policy, ticket, - candidate, pull request, checks, reviews, comments, and threads. It rejects - any changed report digest, mailbox revision, receipt, candidate, policy, - ticket, or evidence. -5. Only when every currently required predicate is `satisfied` does one verified - mailbox commit move `delivered` to `accepted`, clear the claim, and bind the - operator record to the exact receipt and candidate. +3. Start a new provider read only after the confirmed report's `observed_at`. + Produce a second complete observation whose live-read lower bound and + `observed_at` are both strictly later. Reusing the report snapshot is rejected. +4. Invoke `scripts/audit.py accept` with that second observation and lower bound, + the same normalized audit-evidence path, the complete fence, `--accepted-at`, + and `--confirm`. +5. The transition rereads the canonical mailbox, host boundary, policy, ticket, + candidate, pull request base and head, required-check configuration and + results, reviews, comments, and threads. It requires the fence's mailbox, + receipt, candidate, and semantic evidence digest to remain exact while the + observation timestamp advances. Changed facts are rejected as stale or + violated; only the observation timestamp may differ. +6. Only when every currently required predicate and the unconditional + no-unresolved-feedback guard are `satisfied` does one verified mailbox commit + move `delivered` to `accepted`, clear the claim, and bind the operator record + to the exact receipt, candidate, audit evidence, and satisfied verdicts. The + accepted timestamp is always checked against the current UTC clock, even when + no test clock is supplied. Acceptance does not merge a pull request, deploy, mutate or close a native ticket, delete a branch, or accept a parent initiative. diff --git a/skills/atelier/references/github-observation.schema.json b/skills/atelier/references/github-observation.schema.json index 4c334d23..45fb440f 100644 --- a/skills/atelier/references/github-observation.schema.json +++ b/skills/atelier/references/github-observation.schema.json @@ -14,6 +14,7 @@ "pull_request_comments", "reviews", "checks", + "required_checks", "threads", "completeness" ], @@ -65,6 +66,26 @@ "$ref": "#/$defs/check" } }, + "required_checks": { + "type": "object", + "additionalProperties": false, + "required": [ + "configuration_read", + "contexts" + ], + "properties": { + "configuration_read": { + "type": "boolean" + }, + "contexts": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/check_context" + } + } + } + }, "threads": { "type": "array", "items": { @@ -82,6 +103,7 @@ "pull_request_comments", "reviews", "checks", + "required_checks", "threads" ], "properties": { @@ -106,6 +128,9 @@ "checks": { "const": true }, + "required_checks": { + "const": true + }, "threads": { "const": true } @@ -495,6 +520,26 @@ } } }, + "check_context": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "name" + ], + "properties": { + "kind": { + "enum": [ + "CHECK_RUN", + "STATUS_CONTEXT" + ] + }, + "name": { + "type": "string", + "minLength": 1 + } + } + }, "thread_comment": { "type": "object", "additionalProperties": false, diff --git a/skills/atelier/references/host-boundary.md b/skills/atelier/references/host-boundary.md index da0f1908..aaedd65b 100644 --- a/skills/atelier/references/host-boundary.md +++ b/skills/atelier/references/host-boundary.md @@ -37,6 +37,7 @@ Before native-state inspection or future work delegation: --operation github.pull-request.comments.read \ --operation github.pull-request.reviews.read \ --operation github.pull-request.checks.read \ + --operation github.repository.required-checks.read \ --operation github.pull-request.threads.read ``` @@ -44,9 +45,12 @@ Before native-state inspection or future work delegation: provider read. Read live GitHub state and normalize it to `github-observation.schema.json`. Preserve GitHub node IDs, exact candidate SHAs, timestamps, native `parent`, `subIssues`, `blockedBy`, and `blocking` - relationships, and complete pagination. Mark every completeness field true - only after the corresponding collection is fully read. Set `observed_at` - after the last required read completes. + relationships, and complete pagination. Read the effective required-check + configuration from branch protection and repository rulesets, recording the + exact check kind/name identities and whether that configuration read + succeeded. Mark every completeness field true only after the corresponding + collection is fully read. Set `observed_at` after the last required read + completes. 6. Validate the observation: ```text diff --git a/skills/atelier/references/host-capability.json b/skills/atelier/references/host-capability.json index b0c5a103..66b3e61c 100644 --- a/skills/atelier/references/host-capability.json +++ b/skills/atelier/references/host-capability.json @@ -46,6 +46,7 @@ "github.pull-request.comments.read", "github.pull-request.reviews.read", "github.pull-request.checks.read", + "github.repository.required-checks.read", "github.pull-request.threads.read" ] }, diff --git a/skills/atelier/references/mailbox-v1.schema.json b/skills/atelier/references/mailbox-v1.schema.json index af8f7c53..34d47027 100644 --- a/skills/atelier/references/mailbox-v1.schema.json +++ b/skills/atelier/references/mailbox-v1.schema.json @@ -12,7 +12,8 @@ "accepted_at", "policy_commit", "candidate_revision", - "evidence" + "evidence", + "audit_evidence" ], "properties": { "receipt_id": { @@ -32,6 +33,9 @@ }, "evidence": { "$ref": "#/$defs/evidence_results" + }, + "audit_evidence": { + "$ref": "#/$defs/audit_evidence" } } }, @@ -930,6 +934,148 @@ "type": "string", "pattern": "^github:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" }, + "audit_evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "review", + "feedback_dispositions" + ], + "properties": { + "schema": { + "const": "atelier.audit-evidence/v1" + }, + "review": { + "$ref": "#/$defs/audit_review" + }, + "feedback_dispositions": { + "type": "array", + "items": { + "$ref": "#/$defs/feedback_disposition" + } + } + } + }, + "audit_review": { + "type": "object", + "additionalProperties": false, + "required": [ + "mechanism", + "verdict", + "candidate_revision", + "comparison_base_revision", + "observed_at", + "findings" + ], + "properties": { + "mechanism": { + "const": "review-code-change" + }, + "verdict": { + "enum": [ + "clean", + "changes_required", + "blocked" + ] + }, + "candidate_revision": { + "$ref": "#/$defs/sha" + }, + "comparison_base_revision": { + "$ref": "#/$defs/sha" + }, + "observed_at": { + "$ref": "#/$defs/timestamp" + }, + "findings": { + "type": "array", + "items": { + "$ref": "#/$defs/review_finding" + } + } + } + }, + "feedback_disposition": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "id", + "body_digest", + "disposition", + "rationale", + "follow_up" + ], + "properties": { + "kind": { + "enum": [ + "review", + "pull-request-comment" + ] + }, + "id": { + "$ref": "#/$defs/nonempty_string" + }, + "body_digest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "disposition": { + "enum": [ + "resolved", + "deferred", + "not-actionable", + "unresolved" + ] + }, + "rationale": { + "$ref": "#/$defs/nonempty_string" + }, + "follow_up": { + "type": [ + "string", + "null" + ] + } + } + }, + "review_finding": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "summary", + "disposition", + "rationale", + "follow_up" + ], + "properties": { + "id": { + "$ref": "#/$defs/nonempty_string" + }, + "summary": { + "$ref": "#/$defs/nonempty_string" + }, + "disposition": { + "enum": [ + "fixed", + "deferred", + "not-actionable", + "unresolved" + ] + }, + "rationale": { + "$ref": "#/$defs/nonempty_string" + }, + "follow_up": { + "type": [ + "string", + "null" + ] + } + } + }, "review_observation": { "type": "object", "additionalProperties": false, diff --git a/skills/atelier/scripts/audit.py b/skills/atelier/scripts/audit.py index 7e63da49..31a7fc98 100644 --- a/skills/atelier/scripts/audit.py +++ b/skills/atelier/scripts/audit.py @@ -9,7 +9,7 @@ import json import sys from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any, Mapping, Sequence @@ -85,6 +85,8 @@ class FeedbackItem: disposition: str body: str url: str | None + rationale: str | None = None + follow_up: str | None = None def as_dict(self) -> dict[str, Any]: return { @@ -93,22 +95,28 @@ def as_dict(self) -> dict[str, Any]: "disposition": self.disposition, "body": self.body, "url": self.url, + "rationale": self.rationale, + "follow_up": self.follow_up, } @dataclass(frozen=True) class AcceptanceFence: report_digest: str + semantic_digest: str mailbox_revision: str receipt_id: str candidate_revision: str + observed_at: str | None - def as_dict(self) -> dict[str, str]: + def as_dict(self) -> dict[str, Any]: return { "report_digest": self.report_digest, + "semantic_digest": self.semantic_digest, "mailbox_revision": self.mailbox_revision, "receipt_id": self.receipt_id, "candidate_revision": self.candidate_revision, + "observed_at": self.observed_at, } @@ -128,6 +136,7 @@ class AuditReport: acceptance_commit: str | None evidence: tuple[EvidenceResult, ...] feedback: tuple[FeedbackItem, ...] + audit_evidence: Mapping[str, Any] | None authority_errors: tuple[str, ...] @property @@ -172,6 +181,7 @@ def as_dict(self, *, include_digest: bool = True) -> dict[str, Any]: "acceptance_commit": self.acceptance_commit, "evidence": [item.as_dict() for item in self.evidence], "feedback": [item.as_dict() for item in self.feedback], + "audit_evidence": copy.deepcopy(self.audit_evidence), "authority_errors": list(self.authority_errors), "overall_verdict": self.overall_verdict, "acceptance_possible": self.acceptance_possible, @@ -190,13 +200,22 @@ def digest(self) -> str: ).encode() return hashlib.sha256(encoded).hexdigest() + @property + def semantic_digest(self) -> str: + value = self.as_dict(include_digest=False) + value["observed_at"] = None + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + @property def fence(self) -> AcceptanceFence: return AcceptanceFence( report_digest=self.digest, + semantic_digest=self.semantic_digest, mailbox_revision=self.mailbox_revision, receipt_id=self.receipt_id, candidate_revision=self.candidate_revision, + observed_at=self.observed_at, ) @@ -214,6 +233,7 @@ def audit( host_target: HostTarget, observation_path: Path, observation_not_before: datetime, + audit_evidence: Mapping[str, Any] | None, now: datetime | None = None, ) -> AuditReport: return self.claims.writer.observe( @@ -225,6 +245,7 @@ def audit( host_target=host_target, observation_path=observation_path, observation_not_before=observation_not_before, + audit_evidence=audit_evidence, now=now, ), ) @@ -240,6 +261,7 @@ def accept( host_target: HostTarget, observation_path: Path, observation_not_before: datetime, + audit_evidence: Mapping[str, Any], now: datetime | None = None, ) -> WriteResult: if not confirmed: @@ -248,9 +270,16 @@ def accept( raise AuditError("accepted_at must include a UTC offset") if observation_not_before.utcoffset() is None: raise AuditError("observation_not_before must include a UTC offset") + if fence.observed_at is None: + raise MailboxTransitionRejected("confirmed audit has no live observation") + if observation_not_before <= _parse_timestamp(fence.observed_at): + raise AuditError("acceptance requires a new read boundary after the confirmed audit") if accepted_at < observation_not_before: raise AuditError("accepted_at cannot precede the live observation boundary") - if now is not None and accepted_at > now: + current_time = now or datetime.now(UTC) + if current_time.utcoffset() is None: + raise AuditError("now must include a UTC offset") + if accepted_at > current_time: raise AuditError("accepted_at cannot be in the future") planned: dict[str, Any] = {} @@ -262,12 +291,34 @@ def revalidate(context: TransitionContext) -> None: host_target=host_target, observation_path=observation_path, observation_not_before=observation_not_before, + audit_evidence=audit_evidence, now=now, ) - if report.fence != fence: + confirmed_value = report.as_dict(include_digest=False) + confirmed_value["observed_at"] = fence.observed_at + confirmed_digest = hashlib.sha256( + json.dumps( + confirmed_value, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + if ( + confirmed_digest != fence.report_digest + or report.mailbox_revision != fence.mailbox_revision + or report.receipt_id != fence.receipt_id + or report.candidate_revision != fence.candidate_revision + or report.semantic_digest != fence.semantic_digest + ): raise MailboxTransitionRejected( "current audit does not match the explicitly confirmed acceptance fence" ) + if report.observed_at is None or _parse_timestamp( + report.observed_at + ) <= _parse_timestamp(fence.observed_at): + raise MailboxTransitionRejected( + "acceptance observation is not newer than the confirmed audit" + ) if report.observed_at is not None and accepted_at < _parse_timestamp( report.observed_at ): @@ -298,6 +349,7 @@ def revalidate(context: TransitionContext) -> None: for item in report.evidence if item.required }, + "audit_evidence": copy.deepcopy(report.audit_evidence), } planned.clear() planned.update(work=accepted, body=body) @@ -329,6 +381,7 @@ def _report( host_target: HostTarget, observation_path: Path, observation_not_before: datetime, + audit_evidence: Mapping[str, Any] | None, now: datetime | None, ) -> AuditReport: work, body = _read_work(context.checkout, work_id) @@ -355,6 +408,7 @@ def _report( observation: dict[str, Any] | None = None effective_policy: dict[str, Any] | None = None current_policy_commit: str | None = None + effective_audit_evidence: dict[str, Any] | None = None try: self.claims._verify_capability(host_target) @@ -403,6 +457,15 @@ def _report( except Exception as error: authority_errors.append(f"project policy: {error}") + supplied_audit_evidence = ( + work["acceptance"]["audit_evidence"] + if work["status"] == "accepted" and work["acceptance"] is not None + else audit_evidence + ) + try: + effective_audit_evidence = _validated_audit_evidence(supplied_audit_evidence) + except Exception as error: + authority_errors.append(f"audit evidence: {error}") acceptance_commit, baseline_ticket_digest = _acceptance_history( context, work_id, @@ -429,9 +492,10 @@ def _report( receipt, observation, effective_policy, + effective_audit_evidence, self.claims, ) - feedback = _feedback_items(receipt, observation) + feedback = _feedback_items(receipt, observation, effective_audit_evidence) return AuditReport( work_id=work_id, mailbox_revision=context.base_revision, @@ -449,12 +513,15 @@ def _report( EvidenceResult( name=name, verdict=verdicts[name][0], - required=name in required_evidence, + required=( + name in required_evidence or name == "unresolved-feedback-zero" + ), detail=verdicts[name][1], ) for name in EVIDENCE_NAMES ), feedback=feedback, + audit_evidence=effective_audit_evidence, authority_errors=tuple(authority_errors), ) @@ -545,11 +612,123 @@ def _ticket_verdict( return "satisfied", "native ticket identity, eligibility, and material state are current" +def _validated_audit_evidence(value: Mapping[str, Any] | None) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise AuditError("a normalized audit-evidence record is required") + _require_exact_keys(value, {"schema", "review", "feedback_dispositions"}, "record") + if value["schema"] != "atelier.audit-evidence/v1": + raise AuditError("unsupported audit-evidence schema") + review = value["review"] + if not isinstance(review, Mapping): + raise AuditError("audit review must be an object") + _require_exact_keys( + review, + { + "mechanism", + "verdict", + "candidate_revision", + "comparison_base_revision", + "observed_at", + "findings", + }, + "review", + ) + if review["mechanism"] != "review-code-change": + raise AuditError("audit review must use review-code-change") + if review["verdict"] not in {"clean", "changes_required", "blocked"}: + raise AuditError("audit review verdict is invalid") + for name in ("candidate_revision", "comparison_base_revision"): + revision = review[name] + if ( + not isinstance(revision, str) + or len(revision) != 40 + or any(character not in "0123456789abcdef" for character in revision) + ): + raise AuditError(f"audit review {name} must be a lowercase Git SHA") + if not isinstance(review["observed_at"], str): + raise AuditError("audit review observed_at must be a timestamp") + _parse_timestamp(review["observed_at"]) + if not isinstance(review["findings"], list): + raise AuditError("audit review findings must be a list") + finding_ids: set[str] = set() + for finding in review["findings"]: + _validate_disposition( + finding, + kind="review finding", + required={"id", "summary", "disposition", "rationale", "follow_up"}, + dispositions={"fixed", "deferred", "not-actionable", "unresolved"}, + ) + if finding["id"] in finding_ids: + raise AuditError(f"duplicate review finding {finding['id']}") + finding_ids.add(finding["id"]) + dispositions = value["feedback_dispositions"] + if not isinstance(dispositions, list): + raise AuditError("feedback dispositions must be a list") + disposition_ids: set[tuple[str, str]] = set() + for disposition in dispositions: + _validate_disposition( + disposition, + kind="feedback disposition", + required={ + "kind", + "id", + "body_digest", + "disposition", + "rationale", + "follow_up", + }, + dispositions={"resolved", "deferred", "not-actionable", "unresolved"}, + ) + if disposition["kind"] not in {"review", "pull-request-comment"}: + raise AuditError("feedback disposition kind is invalid") + body_digest = disposition["body_digest"] + if ( + not body_digest.startswith("sha256:") + or len(body_digest) != 71 + or any(character not in "0123456789abcdef" for character in body_digest[7:]) + ): + raise AuditError("feedback disposition body_digest is invalid") + identity = (disposition["kind"], disposition["id"]) + if identity in disposition_ids: + raise AuditError(f"duplicate feedback disposition {identity[0]} {identity[1]}") + disposition_ids.add(identity) + return copy.deepcopy(dict(value)) + + +def _require_exact_keys(value: Mapping[str, Any], expected: set[str], label: str) -> None: + if set(value) != expected: + raise AuditError(f"{label} fields do not match the v1 contract") + + +def _validate_disposition( + value: Any, + *, + kind: str, + required: set[str], + dispositions: set[str], +) -> None: + if not isinstance(value, Mapping): + raise AuditError(f"{kind} must be an object") + _require_exact_keys(value, required, kind) + for name in required - {"follow_up", "disposition"}: + if not isinstance(value[name], str) or not value[name].strip(): + raise AuditError(f"{kind} {name} must be nonempty") + if value["disposition"] not in dispositions: + raise AuditError(f"{kind} disposition is invalid") + if value["follow_up"] is not None and not isinstance(value["follow_up"], str): + raise AuditError(f"{kind} follow_up must be text or null") + + + +def _body_digest(body: str) -> str: + return "sha256:" + hashlib.sha256(body.encode()).hexdigest() + def _evaluate_evidence( candidate: Mapping[str, Any], receipt: Mapping[str, Any], observation: Mapping[str, Any] | None, policy: Mapping[str, Any] | None, + audit_evidence: Mapping[str, Any] | None, claims: ClaimCoordinator, ) -> dict[str, tuple[str, str]]: values: dict[str, tuple[str, str]] = {} @@ -571,6 +750,7 @@ def _evaluate_evidence( ) pull_request = observation["pull_request"] if observation is not None else None + live_base_current = False if observation is None: for name in ( "pull-request-head-current", @@ -601,12 +781,19 @@ def _evaluate_evidence( else: expected_repository = candidate["repository"].removeprefix("github:") live_head = pull_request["head"] + live_base = pull_request["base"] same_pr = pull_request["url"] == candidate["pull_request"] same_repository_ref = ( live_head["repository"] == expected_repository and live_head["ref"] == candidate["remote_ref"] ) same_head = live_head["sha"] == candidate["head_revision"] + live_base_current = bool( + policy is not None + and live_base["repository"] == expected_repository + and live_base["ref"] == policy["repository"]["canonical_ref"] + and live_base["sha"] == candidate["base_revision"] + ) if not same_pr or not same_repository_ref: values["pull-request-head-current"] = ( "violated", @@ -633,10 +820,10 @@ def _evaluate_evidence( f"live pull request state is {pull_request['state']}", ) ) - if not same_head: + if not same_head or not live_base_current: values["pull-request-mergeable"] = ( "stale", - "mergeability belongs to a different pull request head", + "mergeability belongs to a different pull request head or base", ) elif pull_request["mergeable"] == "MERGEABLE": values["pull-request-mergeable"] = ( @@ -655,11 +842,13 @@ def _evaluate_evidence( ) values["required-checks-pass"] = _checks_verdict( observation["checks"], + observation["required_checks"], candidate["head_revision"], ) values["unresolved-feedback-zero"] = _feedback_verdict( receipt, observation, + audit_evidence, ) values["required-validation-reported"] = _validation_verdict( @@ -673,33 +862,53 @@ def _evaluate_evidence( ) values["independent-review-current"] = _review_verdict( receipt, + audit_evidence, candidate["head_revision"], candidate["base_revision"], ) + if pull_request is not None and not live_base_current: + values["independent-review-current"] = ( + "stale", + "the live pull request base differs from the reviewed delivery base", + ) return values def _checks_verdict( checks: Sequence[Mapping[str, Any]], + required: Mapping[str, Any], candidate_revision: str, ) -> tuple[str, str]: - if any(check["candidate_sha"] != candidate_revision for check in checks): + if not required["configuration_read"]: + return "unknown", "effective required-check configuration could not be read" + observed_by_identity: dict[tuple[str, str], list[Mapping[str, Any]]] = {} + for check in checks: + observed_by_identity.setdefault((check["kind"], check["name"]), []).append(check) + selected: list[Mapping[str, Any]] = [] + for context in required["contexts"]: + matches = observed_by_identity.get((context["kind"], context["name"]), []) + if len(matches) != 1: + return "unknown", "a required check context is missing or ambiguous" + selected.append(matches[0]) + if any(check["candidate_sha"] != candidate_revision for check in selected): return "stale", "one or more required checks belong to another candidate" incomplete = [ check - for check in checks + for check in selected if check["status"].upper() != "COMPLETED" or check["conclusion"] is None ] if incomplete: return "unknown", "one or more required check results are incomplete" failing = [ check - for check in checks + for check in selected if check["conclusion"].upper() not in {"SUCCESS", "NEUTRAL", "SKIPPED"} ] if failing: return "violated", "one or more required checks completed unsuccessfully" - return "satisfied", "all observed required checks passed on the delivered head" + if not selected: + return "satisfied", "required-check configuration was read and names no contexts" + return "satisfied", "every configured required check passed on the delivered head" def _validation_verdict( @@ -725,6 +934,7 @@ def _validation_verdict( def _review_verdict( receipt: Mapping[str, Any], + audit_evidence: Mapping[str, Any] | None, candidate_revision: str, comparison_base_revision: str, ) -> tuple[str, str]: @@ -733,28 +943,38 @@ def _review_verdict( for item in receipt["reviews"] if item["mechanism"] == "review-code-change" ] - if not reviews: - return "unknown", "independent review-code-change evidence is missing" + if not reviews or audit_evidence is None: + return "unknown", "structured independent review evidence is missing" dated = [(_parse_timestamp(item["observed_at"]), item) for item in reviews] latest_time = max(observed_at for observed_at, _ in dated) latest = [item for observed_at, item in dated if observed_at == latest_time] + review = audit_evidence["review"] if any( item["candidate_revision"] != candidate_revision or item["comparison_base_revision"] != comparison_base_revision for item in latest + ) or ( + review["candidate_revision"] != candidate_revision + or review["comparison_base_revision"] != comparison_base_revision ): - return "stale", "latest independent review belongs to another head or base" - verdicts = {item["verdict"] for item in latest} - if verdicts == {"clean"}: - return "satisfied", "latest independent review is clean on the exact head and base" - if "changes_required" in verdicts: - return "violated", "latest independent review requires changes" - return "unknown", "latest independent review is blocked or contradictory" + return "stale", "independent review belongs to another head or base" + receipt_verdicts = {item["verdict"] for item in latest} + if "changes_required" in receipt_verdicts or review["verdict"] == "changes_required": + return "violated", "independent review requires changes" + if review["verdict"] == "blocked" or receipt_verdicts != {"clean"}: + return "unknown", "independent review is blocked or contradictory" + if any(item["disposition"] == "unresolved" for item in review["findings"]): + return "violated", "independent review retains an unresolved finding" + return ( + "satisfied", + "structured independent review is clean on the exact head and base", + ) def _feedback_verdict( receipt: Mapping[str, Any], observation: Mapping[str, Any], + audit_evidence: Mapping[str, Any] | None, ) -> tuple[str, str]: unresolved_threads = [ item @@ -768,15 +988,43 @@ def _feedback_verdict( return "violated", "the live pull request review decision requires changes" if receipt["unresolved_obligations"]: return "violated", "the delivered receipt retains unresolved obligations" + live_feedback = { + (kind, item["id"]): _body_digest(item["body"]) + for kind, collection in ( + ("review", observation["reviews"]), + ("pull-request-comment", observation["pull_request_comments"]), + ) + for item in collection + if item["body"].strip() + } + dispositions = { + (item["kind"], item["id"]): item + for item in ( + audit_evidence["feedback_dispositions"] if audit_evidence is not None else () + ) + } + if set(dispositions) - set(live_feedback): + return "stale", "a feedback disposition does not identify current live feedback" + if any( + item["body_digest"] != live_feedback[identity] + for identity, item in dispositions.items() + ): + return "stale", "a feedback disposition belongs to an earlier body revision" + missing = set(live_feedback) - set(dispositions) + if missing: + return "unknown", "one or more live review or comment bodies lack a disposition" + if any(item["disposition"] == "unresolved" for item in dispositions.values()): + return "violated", "one or more live review or comment bodies remain unresolved" return ( "satisfied", - "no unresolved material receipt obligation, review decision, or live thread remains", + "every material obligation, review, comment, and thread has a durable disposition", ) def _feedback_items( receipt: Mapping[str, Any], observation: Mapping[str, Any] | None, + audit_evidence: Mapping[str, Any] | None, ) -> tuple[FeedbackItem, ...]: items = [ FeedbackItem( @@ -790,26 +1038,35 @@ def _feedback_items( ] if observation is None: return tuple(items) - items.extend( - FeedbackItem( - kind="review", - identifier=review["id"], - disposition=review["state"], - body=review["body"], - url=review["url"], + dispositions = { + (item["kind"], item["id"]): item + for item in ( + audit_evidence["feedback_dispositions"] if audit_evidence is not None else () ) - for review in observation["reviews"] - ) - items.extend( - FeedbackItem( - kind="pull-request-comment", - identifier=comment["id"], - disposition="recorded", - body=comment["body"], - url=comment["url"], - ) - for comment in observation["pull_request_comments"] - ) + } + for kind, collection in ( + ("review", observation["reviews"]), + ("pull-request-comment", observation["pull_request_comments"]), + ): + for value in collection: + disposition = dispositions.get((kind, value["id"])) + items.append( + FeedbackItem( + kind=kind, + identifier=value["id"], + disposition=( + disposition["disposition"] + if disposition is not None + else "undispositioned" + if value["body"].strip() + else "no-material-body" + ), + body=value["body"], + url=value["url"], + rationale=(disposition["rationale"] if disposition is not None else None), + follow_up=(disposition["follow_up"] if disposition is not None else None), + ) + ) items.extend( FeedbackItem( kind="review-thread", @@ -865,9 +1122,11 @@ def _host_target(value: Mapping[str, Any]) -> HostTarget: def _fence(value: Mapping[str, Any]) -> AcceptanceFence: return AcceptanceFence( report_digest=value["report_digest"], + semantic_digest=value["semantic_digest"], mailbox_revision=value["mailbox_revision"], receipt_id=value["receipt_id"], candidate_revision=value["candidate_revision"], + observed_at=value["observed_at"], ) @@ -883,6 +1142,7 @@ def _parser() -> argparse.ArgumentParser: subparser.add_argument("--host-target", required=True) subparser.add_argument("--observation", required=True, type=Path) subparser.add_argument("--observation-not-before", required=True) + subparser.add_argument("--audit-evidence", required=True, type=Path) subparser.add_argument("--now") accept = subparsers.choices["accept"] accept.add_argument("--fence", required=True) @@ -906,6 +1166,10 @@ def main(arguments: Sequence[str] | None = None) -> int: "observation_not_before": _parse_timestamp( parsed.observation_not_before ), + "audit_evidence": _json_object( + parsed.audit_evidence.read_text(encoding="utf-8"), + "audit evidence", + ), "now": _parse_timestamp(parsed.now) if parsed.now else None, } try: From 52defbda09689d951ed715aeb836494010a51251 Mon Sep 17 00:00:00 2001 From: Atelier Protocol Experiment Date: Wed, 29 Jul 2026 12:55:52 -0700 Subject: [PATCH 3/3] fix(audit): bind live readiness identity Match configured required checks by context and provider provenance, preserve optional-check semantics, and reject draft, review-blocked, or policy-blocked pull requests before acceptance. --- contract_tests/test_audit.py | 29 +++++++- contract_tests/test_host_boundary.py | 3 +- docs/project-policy-contract.md | 28 +++++--- .../references/github-observation.schema.json | 25 ++++--- skills/atelier/references/host-boundary.md | 8 ++- skills/atelier/scripts/audit.py | 72 ++++++++++++++----- 6 files changed, 122 insertions(+), 43 deletions(-) diff --git a/contract_tests/test_audit.py b/contract_tests/test_audit.py index 0c0e0c00..de84a335 100644 --- a/contract_tests/test_audit.py +++ b/contract_tests/test_audit.py @@ -380,19 +380,21 @@ def test_required_check_configuration_is_identity_aware_and_fail_closed(self) -> "pull_request_number": live["pull_request"]["number"], "kind": "CHECK_RUN", "name": "optional", + "integration_id": 7, "status": "COMPLETED", "conclusion": "FAILURE", "candidate_sha": live["pull_request"]["head"]["sha"], "details_url": None, } ] + live["pull_request"]["merge_state_status"] = "UNSTABLE" self.write_observation(live) report = self.report() verdicts = {item.name: item.verdict for item in report.evidence} self.assertEqual(verdicts["required-checks-pass"], "satisfied") live["required_checks"]["contexts"] = [ - {"kind": "CHECK_RUN", "name": "required"} + {"name": "required", "integration_id": 42} ] self.write_observation(live) report = self.report() @@ -405,6 +407,7 @@ def test_required_check_configuration_is_identity_aware_and_fail_closed(self) -> "pull_request_number": live["pull_request"]["number"], "kind": "CHECK_RUN", "name": "required", + "integration_id": 7, "status": "COMPLETED", "conclusion": "SUCCESS", "candidate_sha": live["pull_request"]["head"]["sha"], @@ -414,8 +417,32 @@ def test_required_check_configuration_is_identity_aware_and_fail_closed(self) -> self.write_observation(live) report = self.report() verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts["required-checks-pass"], "unknown") + + live["checks"][-1]["integration_id"] = 42 + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} self.assertEqual(verdicts["required-checks-pass"], "satisfied") + def test_draft_review_and_policy_merge_blocks_prevent_acceptance(self) -> None: + original = self.live_observation() + cases = ( + ("is_draft", True, "pull-request-open", "violated"), + ("review_decision", "REVIEW_REQUIRED", "unresolved-feedback-zero", "violated"), + ("merge_state_status", "BLOCKED", "pull-request-mergeable", "violated"), + ("merge_state_status", "UNKNOWN", "pull-request-mergeable", "unknown"), + ) + for field, value, predicate, expected in cases: + with self.subTest(field=field, value=value): + live = json.loads(json.dumps(original)) + live["pull_request"][field] = value + self.write_observation(live) + report = self.report() + verdicts = {item.name: item.verdict for item in report.evidence} + self.assertEqual(verdicts[predicate], expected) + self.assertFalse(report.acceptance_possible) + def test_undispositioned_live_comment_blocks_acceptance(self) -> None: live = self.live_observation() live["pull_request_comments"] = [ diff --git a/contract_tests/test_host_boundary.py b/contract_tests/test_host_boundary.py index 7d20944c..0596bbc0 100644 --- a/contract_tests/test_host_boundary.py +++ b/contract_tests/test_host_boundary.py @@ -150,6 +150,7 @@ def complete_observation() -> dict[str, object]: "pull_request_number": 785, "kind": "CHECK_RUN", "name": "test", + "integration_id": 42, "status": "COMPLETED", "conclusion": "SUCCESS", "candidate_sha": HEAD_SHA, @@ -158,7 +159,7 @@ def complete_observation() -> dict[str, object]: ], "required_checks": { "configuration_read": True, - "contexts": [{"kind": "CHECK_RUN", "name": "test"}], + "contexts": [{"name": "test", "integration_id": 42}], }, "threads": [ { diff --git a/docs/project-policy-contract.md b/docs/project-policy-contract.md index 38410c30..0a5490bd 100644 --- a/docs/project-policy-contract.md +++ b/docs/project-policy-contract.md @@ -244,29 +244,35 @@ Each predicate has one authoritative source and exact evaluation. ### `pull-request-open` - **Source:** live GitHub pull-request state. -- **Satisfied:** the PR is open. -- **Violated:** the PR is closed or merged. +- **Satisfied:** the PR is open and not a draft. +- **Violated:** the PR is closed, merged, or still a draft. - **Unknown:** GitHub cannot be read. - **Stale:** not applicable. ### `pull-request-mergeable` -- **Source:** live GitHub mergeability. -- **Satisfied:** GitHub reports the PR mergeable. -- **Violated:** GitHub reports a conflict. -- **Unknown:** mergeability is unavailable or still being calculated. +- **Source:** live GitHub conflict and merge-state status. +- **Satisfied:** GitHub reports the exact head mergeable and its current merge state + does not block readiness. `UNSTABLE` does not fail this predicate by itself; + configured required checks are evaluated separately. +- **Violated:** GitHub reports a conflict or a policy-blocked merge state. +- **Unknown:** conflict or merge-state status is unavailable or still being + calculated. - **Stale:** the PR head or exact comparison base changed after the observation. ### `required-checks-pass` - **Source:** live required-check configuration plus check results for the exact PR - head. Required identities are the exact check kind and name obtained from - effective branch protection and repository rulesets. + head. Required identities are the exact context name plus optional GitHub App or + ruleset integration identity obtained from effective branch protection and + repository rulesets; observed results retain their check/status kind and + integration identity. - **Satisfied:** the required configuration was read and every named required - check completed successfully; an empty configured set is satisfied. + check from its configured provider completed successfully; an empty configured + set is satisfied. - **Violated:** a named required check completed unsuccessfully. -- **Unknown:** required-check configuration or a named required result cannot be - read, is missing, or is ambiguous. +- **Unknown:** required-check configuration or a named required result from its + configured provider cannot be read, is missing, or is ambiguous. - **Stale:** a named required result belongs to another PR head. ### `required-validation-reported` diff --git a/skills/atelier/references/github-observation.schema.json b/skills/atelier/references/github-observation.schema.json index 45fb440f..b8e07797 100644 --- a/skills/atelier/references/github-observation.schema.json +++ b/skills/atelier/references/github-observation.schema.json @@ -474,6 +474,7 @@ "pull_request_number", "kind", "name", + "integration_id", "status", "conclusion", "candidate_sha", @@ -498,6 +499,13 @@ "type": "string", "minLength": 1 }, + "integration_id": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, "status": { "type": "string", "minLength": 1 @@ -524,19 +532,20 @@ "type": "object", "additionalProperties": false, "required": [ - "kind", - "name" + "name", + "integration_id" ], "properties": { - "kind": { - "enum": [ - "CHECK_RUN", - "STATUS_CONTEXT" - ] - }, "name": { "type": "string", "minLength": 1 + }, + "integration_id": { + "type": [ + "integer", + "null" + ], + "minimum": 1 } } }, diff --git a/skills/atelier/references/host-boundary.md b/skills/atelier/references/host-boundary.md index aaedd65b..1d4ad26c 100644 --- a/skills/atelier/references/host-boundary.md +++ b/skills/atelier/references/host-boundary.md @@ -46,9 +46,11 @@ Before native-state inspection or future work delegation: `github-observation.schema.json`. Preserve GitHub node IDs, exact candidate SHAs, timestamps, native `parent`, `subIssues`, `blockedBy`, and `blocking` relationships, and complete pagination. Read the effective required-check - configuration from branch protection and repository rulesets, recording the - exact check kind/name identities and whether that configuration read - succeeded. Mark every completeness field true only after the corresponding + configuration from branch protection and repository rulesets, normalizing each + configured context name plus optional GitHub App or ruleset integration ID. + Preserve the observed check/status kind and integration ID on every result, and + record whether the configuration read succeeded. Mark every completeness field + true only after the corresponding collection is fully read. Set `observed_at` after the last required read completes. 6. Validate the observation: diff --git a/skills/atelier/scripts/audit.py b/skills/atelier/scripts/audit.py index 31a7fc98..0b198036 100644 --- a/skills/atelier/scripts/audit.py +++ b/skills/atelier/scripts/audit.py @@ -809,37 +809,60 @@ def _evaluate_evidence( "satisfied", "live pull request repository, ref, and head match the delivery", ) - values["pull-request-open"] = ( - ( - "satisfied", - "live pull request is open", - ) - if pull_request["state"] == "OPEN" - else ( + if pull_request["state"] != "OPEN": + values["pull-request-open"] = ( "violated", f"live pull request state is {pull_request['state']}", ) - ) + elif pull_request["is_draft"]: + values["pull-request-open"] = ( + "violated", + "live pull request is a draft rather than ready for review", + ) + else: + values["pull-request-open"] = ( + "satisfied", + "live pull request is open and ready for review", + ) if not same_head or not live_base_current: values["pull-request-mergeable"] = ( "stale", "mergeability belongs to a different pull request head or base", ) - elif pull_request["mergeable"] == "MERGEABLE": + elif pull_request["is_draft"]: values["pull-request-mergeable"] = ( - "satisfied", - "GitHub reports the exact delivered head mergeable", + "violated", + "the live pull request is still a draft", ) elif pull_request["mergeable"] == "CONFLICTING": values["pull-request-mergeable"] = ( "violated", "GitHub reports a merge conflict", ) - else: + elif pull_request["mergeable"] != "MERGEABLE": values["pull-request-mergeable"] = ( "unknown", "GitHub mergeability is unavailable", ) + elif pull_request["merge_state_status"].upper() == "UNKNOWN": + values["pull-request-mergeable"] = ( + "unknown", + "GitHub merge readiness is unavailable", + ) + elif pull_request["merge_state_status"].upper() not in { + "CLEAN", + "HAS_HOOKS", + "UNSTABLE", + }: + values["pull-request-mergeable"] = ( + "violated", + "GitHub reports the pull request blocked from its current merge state", + ) + else: + values["pull-request-mergeable"] = ( + "satisfied", + "GitHub reports the exact delivered head mergeable under current policy", + ) values["required-checks-pass"] = _checks_verdict( observation["checks"], observation["required_checks"], @@ -881,14 +904,22 @@ def _checks_verdict( ) -> tuple[str, str]: if not required["configuration_read"]: return "unknown", "effective required-check configuration could not be read" - observed_by_identity: dict[tuple[str, str], list[Mapping[str, Any]]] = {} - for check in checks: - observed_by_identity.setdefault((check["kind"], check["name"]), []).append(check) selected: list[Mapping[str, Any]] = [] for context in required["contexts"]: - matches = observed_by_identity.get((context["kind"], context["name"]), []) + matches = [ + check + for check in checks + if check["name"] == context["name"] + and ( + context["integration_id"] is None + or check["integration_id"] == context["integration_id"] + ) + ] if len(matches) != 1: - return "unknown", "a required check context is missing or ambiguous" + return ( + "unknown", + "a required check context or its configured provider is missing or ambiguous", + ) selected.append(matches[0]) if any(check["candidate_sha"] != candidate_revision for check in selected): return "stale", "one or more required checks belong to another candidate" @@ -984,8 +1015,11 @@ def _feedback_verdict( if unresolved_threads: return "violated", "one or more live review threads remain unresolved" pull_request = observation["pull_request"] - if pull_request is not None and pull_request["review_decision"] == "CHANGES_REQUESTED": - return "violated", "the live pull request review decision requires changes" + if pull_request is not None and pull_request["review_decision"] in { + "CHANGES_REQUESTED", + "REVIEW_REQUIRED", + }: + return "violated", "the live pull request review decision is not ready" if receipt["unresolved_obligations"]: return "violated", "the delivered receipt retains unresolved obligations" live_feedback = {