From 60aa3c419988ed7b1c353261f48460f887dc0a02 Mon Sep 17 00:00:00 2001 From: Scott Haug Date: Mon, 27 Jul 2026 12:23:49 -0700 Subject: [PATCH] feat: require acceptance evidence for workflow closeout ## Summary - Gate ticket and epic completion on criterion-specific acceptance evidence - Preserve delivery state while missing post-merge evidence blocks tracker closeout - Add delegated execution v2 and result-blind ticket and epic evaluation cases ## Why - Prevent merged PRs, green CI, or closed tracker state from being mistaken for acceptance - Keep acceptance bound to the exact candidate, deployment, environment, and evidence source --- CHANGELOG.md | 4 +- README.md | 20 +- justfile | 4 + skills/implement-epic/SKILL.md | 114 ++-- skills/implement-epic/evals/cases.json | 14 +- skills/implement-epic/evals/expectations.json | 12 +- skills/implement-epic/references/closeout.md | 55 +- skills/implement-epic/references/github.md | 39 +- skills/implement-epic/references/linear.md | 38 +- .../tests/test_orchestration_contract.py | 57 ++ skills/implement-ticket/SKILL.md | 189 +++++-- skills/implement-ticket/evals/cases.json | 39 ++ .../implement-ticket/evals/expectations.json | 30 + .../implement-ticket/evals/forward_cases.json | 350 ++++++++++-- .../evals/forward_expectations.json | 128 +++++ .../references/babysit-pr-handoff.md | 45 +- .../references/cleanup-and-result.md | 52 +- .../delegated-execution/CONTRACT.md | 84 ++- .../delegated-execution/capability.json | 7 +- .../capability.schema.json | 7 +- .../checkpoint-request.schema.json | 30 +- .../checkpoint-response.schema.json | 27 +- .../invocation.schema.json | 79 ++- .../delegated-execution/result.schema.json | 89 ++- .../delegated-execution/validate.py | 307 ++++++++++- skills/implement-ticket/references/github.md | 46 +- skills/implement-ticket/references/linear.md | 23 +- .../references/review-and-merge-gates.md | 29 +- .../scripts/evals/claude_executor.py | 37 +- .../scripts/evals/fixture_executor.py | 229 +++++++- .../scripts/evals/run_forward.py | 47 +- .../test_delegated_execution_contract.py | 516 +++++++++++++++++- .../scripts/tests/test_forward_evals.py | 193 ++++++- .../tests/test_implement_ticket_contract.py | 63 +++ 34 files changed, 2691 insertions(+), 312 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b969c8c..2e69dba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ summary: Chronological history of repository and skill changes. # Changelog -## 2026-07-27 — Populated the correctness stratum, and recovered carved suffixes +## 2026-07-27 — Enforced acceptance-gated closeout, populated the correctness stratum, and recovered carved suffixes +- feat: require acceptance evidence for workflow closeout - fix: make every packet diff a valid patch, and gate the adjudication record + (`c4e01d3c6fc94f1bd174d1ad5bd2374511c85471`) - fix: stop a grader formulation being quotable from its own packet (`fa772a7d770bd3d07f3fdd9bdc45a0c237b1d14e`) - docs: record the batch-2 delivery, the clean-control standard, and its limits diff --git a/README.md b/README.md index 7e76ad3..f60e532 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ just test-babysit-pr just test-implement-ticket just test-implement-epic just eval-implement-ticket +just eval-implement-epic ``` Validate a review packet and result together: @@ -149,6 +150,7 @@ Run deterministic local evaluation harnesses without an agent runtime: ```bash just eval-carve-changesets just eval-implement-ticket +just eval-implement-epic ``` The carve-changesets command first runs its objective integration self-test, @@ -163,14 +165,16 @@ just eval-carve-changesets-executor "python3 /path/to/adapter.py" ``` The ticket-composition evaluator starts a fresh process for each case, with -fixture identity and grader expectations withheld. Case artifacts carry -pre-classified scenario flags (for example, a CI failure already labeled -branch-caused or infrastructure), so the harness grades obligation mapping and -terminal-state selection — not evidence classification itself. Its bundled -reference executor is a deterministic simulation of a compliant runtime, not a -model. To forward-evaluate a real agent runtime, pass its stdin/stdout JSON -adapter through `scripts/evals/run_forward.py --executor` and retain captured -observations with `--output-dir`. A Claude Code headless adapter is bundled: +fixture identity and grader expectations withheld. Its shared corpus targets +both `implement-ticket` and `implement-epic`; `just eval-implement-epic` filters +the same validated corpus to epic packets. Case artifacts carry raw, live-shaped +workflow state, including criterion-specific acceptance evidence bound to the +candidate or deployed SHA. The harness grades obligation mapping and +terminal-state selection. Its bundled reference executor is a deterministic +simulation of a compliant runtime, not a model. To forward-evaluate a real agent +runtime, pass its stdin/stdout JSON adapter through +`scripts/evals/run_forward.py --executor` and retain captured observations with +`--output-dir`. A Claude Code headless adapter is bundled: ```bash just eval-implement-ticket-claude diff --git a/justfile b/justfile index e808892..8c7dfb0 100644 --- a/justfile +++ b/justfile @@ -71,6 +71,10 @@ test-implement-ticket: eval-implement-ticket: python3 {{skills_dir}}/implement-ticket/scripts/evals/run_forward.py +# Run only implement-epic packets from the shared result-blind corpus. +eval-implement-epic: + python3 {{skills_dir}}/implement-ticket/scripts/evals/run_forward.py \ + --target-skill implement-epic # Real-runtime forward evaluation; requires the `claude` CLI on PATH. eval-implement-ticket-claude: python3 {{skills_dir}}/implement-ticket/scripts/evals/run_forward.py \ diff --git a/skills/implement-epic/SKILL.md b/skills/implement-epic/SKILL.md index 18b53c4..22e2bac 100644 --- a/skills/implement-epic/SKILL.md +++ b/skills/implement-epic/SKILL.md @@ -75,20 +75,23 @@ Before selecting work, discover or receive and verify: - named architecture, design, contract, migration, and rollout documents; - completion policy: ready PRs only, merge children after gates, or merge plus separately authorized epic closeout; +- each child and parent acceptance criterion, required verification item, + pre/post-merge stage, and criterion-specific evidence ledger; - serial execution by default, with parallel execution only when explicitly authorized and proven non-overlapping; and -- authority for child execution, merge, manual transitions, graph edits, - follow-up creation, decomposition of an oversized coherent candidate into a - stacked chain, branch deletion, parent closeout, deployment, production - mutation, and destructive operations. +- authority for child execution, merge, post-merge verification, manual + transitions, graph edits, follow-up creation, decomposition of an oversized + coherent candidate into a stacked chain, branch deletion, parent closeout, + deployment, production mutation, and destructive operations. Pass authority into `implement-ticket` without expansion. The `decompose oversized candidates into stacked changesets` grant is off by default and must be passed through verbatim; this skill gains no decomposition mechanics. Ready-PR authority does not imply merge. Child merge authority does -not imply parent closeout. Words such as `finish`, `complete`, or `end to end` -do not independently grant decomposition, merge, graph mutation, deployment, or -closeout authority. +not imply deployment, post-merge verification, child tracker transition, or +parent closeout. Words such as `finish`, `complete`, or `end to end` do not +independently grant decomposition, merge, graph mutation, deployment, +verification, or closeout authority. Use this source order: @@ -108,9 +111,10 @@ blocker requires user input. ### 1. Refresh live state -- Read every in-scope epic and its current children. +- Read every in-scope epic and its current children, including closed children. - Read native parent, sub-issue, `blockedBy`, and `blocking` relationships. -- Read dispositions and delivered outcomes of closed blockers. +- Read dispositions, delivered outcomes, and criterion-specific acceptance + ledgers for every required child and closed blocker. - Inspect existing branches and open or merged PRs before selecting a child. - Separate the serial critical-path recommendation from other parallel-ready work. @@ -120,8 +124,13 @@ list, label, or previous loop iteration when native relationships are available. ### 2. Select one child -Select an open, in-scope, PR-sized child only when native graph state shows no -open blocker. At the graph boundary, verify that every required closed-blocker +Select an in-scope, PR-sized child only when native graph state shows no open +blocker and the child is either open or was auto-closed while required +acceptance remains missing. Route that auto-closed child through +`implement-ticket`, passing its closeout state and granted or withheld reopen +authority so the ticket workflow can reopen it when authorized or report the +authority blocker. Do not select an accepted, superseded, or otherwise terminal +closed child. At the graph boundary, verify that every required closed-blocker outcome exists in its authoritative repository, artifact registry, tracker, or environment. Treat canceled or not-planned blockers with missing required outcomes as unresolved. @@ -161,34 +170,42 @@ material overlap. Do not trust a reported result until ticket identity, repository, base, branch/worktree, candidate, PR, validation, review, remote-gate, merge, -transition, and cleanup evidence are internally consistent and match live state. +delivery, criterion-specific acceptance, transition, and cleanup evidence are +internally consistent and match live state. -- `ready_pr`: verify the candidate is open, mergeable, and at the complete - current-candidate non-merge gate with only merge withheld. Do not count the - child complete or unblock dependents that require merge. Continue only with - another independently ready child when the requested scope permits it. +- `ready_pr`: verify the candidate is open, mergeable, at the complete + current-candidate non-merge gate, and has every required pre-merge acceptance + entry passing. Do not count the child complete or unblock dependents that + require merge or acceptance. - `ready_prs`: verify the reported PR count, ordered predecessor-base topology, - final-only closing syntax, per-PR candidate and non-merge gate evidence, and - whole-chain equivalence with the ticket candidate. Do not count the child - complete or unblock dependents that require merge. -- `merged`: verify mainline and tracker evidence, then refresh the complete live - graph before any selection or completion claim. For a stacked child, also - verify `all_merged`, every PR merge and propagation step, and full-chain + correct closing/non-closing syntax, per-PR candidate and non-merge gate + evidence, passing required pre-merge entries, and whole-chain equivalence. Do + not count the child complete or unblock dependents that require merge or + acceptance. +- `merged`: verify mainline, the child's complete current acceptance ledger, and + tracker evidence before refreshing the graph. For a stacked child, also verify + `all_merged`, every PR merge and propagation step, and full-chain representation on the base. Do not reproduce decomposition or propagation mechanics while verifying the result. -- `blocked`: preserve the exact reason and partial artifacts. Never count it as - complete. Select another independently ready child only when the requested - scope permits; otherwise stop for the missing decision, outcome, or - capability. +- `blocked`: preserve the exact reason and partial artifacts, including a merged + delivery whose post-merge acceptance is pending. Never count it as complete. A + verified merge, delivery, or tracker transition still requires a complete + graph refresh before any next selection; acceptance separately determines + which dependency edges are satisfied. Select another independently ready child + only when the refreshed graph and requested scope permit; otherwise stop for + the missing decision, outcome, evidence, authority, or capability. - `requires_epic`: treat it as an invalid child selection or malformed handoff. Stop or refresh and resolve scope; never recursively invoke this skill, bounce back to `implement-ticket`, or flatten the returned epic into the child. ### 5. Refresh or stop at the requested boundary -After every verified merge or tracker transition, reread the native graph. Do -not reuse an earlier ready set. Report newly unblocked work even when the -requested boundary has been reached. +After every verified merge, delivery, or tracker transition, reread the complete +native graph regardless of the ticket terminal state. Then separately determine +which edges are satisfied by delivery and which require complete acceptance. A +merged delivery with pending acceptance remains incomplete, but its graph-state +change must still inform the next ready set. Do not reuse an earlier ready set. +Report newly unblocked work even when the requested boundary has been reached. For one named child, stop after that child's completion policy. For a named subset, process only that subset in dependency order. Do not implement unnamed @@ -200,15 +217,31 @@ eligible or a genuine blocker remains. Follow [epic closeout](references/closeout.md). Close a parent only with explicit parent-close authority and current evidence that: -- every required child and blocker outcome is satisfied; -- every required PR result is represented on the base; -- parent acceptance criteria hold against resulting behavior; +- every required child's criterion-specific acceptance ledger is complete and + current, regardless of its native state; +- every required blocker outcome is satisfied; +- every required PR result is represented on the current remote base; +- the parent's own acceptance ledger passes against resulting behavior; +- current-main representation and the exact deployed SHA are verified whenever + deployment is required; +- explicit visual-layout requirements have screenshot or + geometry/computed-layout evidence rather than functional browser evidence + alone; - required clean-main, documentation, migration, compatibility, rollout, and cleanup checks pass; and - the epic-wide late-feedback sweep has no undispositioned material finding. -Validate and close each epic separately before an umbrella parent. A nearly -complete graph is not complete. +Validate and close each epic separately before an umbrella parent. "All +implementation PRs merged" and "all native children closed" are delivery and +administrative milestones, not epic acceptance. + +## Reclose escaped epics proportionally + +When an epic was reopened for an escaped acceptance defect, require a focused +corrective child, a regression test at the escaped boundary, and renewed +evidence for the full affected customer journey before reclosure. A merged +corrective PR alone is insufficient. Revalidate only the affected journey and +requirements; do not impose unrelated full-system testing. ## Stop conditions @@ -219,6 +252,8 @@ Stop and report `blocked` when: - a product, architecture, migration, destructive, or authorization decision is unresolved; - delegated mutation ownership or returned evidence is ambiguous; +- a child or parent has missing, failed, unavailable, stale, wrong-environment, + or category-mismatched required acceptance evidence; - epic-wide validation or late feedback shows a required unresolved gap; or - parent closeout lacks authority. @@ -226,8 +261,9 @@ Difficulty, ordinary CI wait time, or unrelated ready children are not blockers. ## Report the epic result -Report the requested scope, each invoked ticket and its terminal state, merged -and ready PRs or stacks, refreshed graph state, serial critical-path and -parallel-ready work, parent acceptance and closeout evidence, intentionally -deferred work, and one concrete next action. Never report a child or parent -complete from stale or unverified evidence. +Report the requested scope, each invoked ticket and its delivery, acceptance, +and terminal state, merged and ready PRs or stacks, refreshed graph state, +serial critical-path and parallel-ready work, child and parent acceptance +ledgers, closeout evidence, intentionally deferred work, and one concrete next +action. Never report a child or parent complete from tracker state, stale +verification, or delivery evidence alone. diff --git a/skills/implement-epic/evals/cases.json b/skills/implement-epic/evals/cases.json index dd7d492..9c90b2c 100644 --- a/skills/implement-epic/evals/cases.json +++ b/skills/implement-epic/evals/cases.json @@ -57,7 +57,7 @@ "id": "authorized-full-epic-closeout", "request": "Implement and close epic G-190.", "authority": "child merge and parent closeout explicitly granted", - "closeout": "All children merged and closed; graph, base representation, acceptance, clean-main, rollout, and late-feedback gates pass." + "closeout": "Every child and parent criterion ledger passes, every required PR is represented on current main, exact deployed SHA evidence is current, clean-main and late-feedback gates pass." }, { "id": "umbrella-closeout-separately", @@ -113,5 +113,17 @@ "request": "Continue epic G-280 after authorized child G-281 returns a three-PR carved result.", "graph": "G-282 is blocked until G-281 is fully merged.", "ticket_results": ["G-281 reports all_merged with three ordered PRs, per-PR gate evidence, propagation evidence, and full-chain representation on main"] + }, + { + "id": "closed-children-missing-manual-browser", + "request": "Close epic G-290 after every child PR merged and every child ticket closed.", + "graph": "Every required child is administratively closed and represented on main.", + "closeout": "A required child acceptance ledger has missing authenticated manual-browser evidence for the exact deployment." + }, + { + "id": "reopened-correction-missing-journey-revalidation", + "request": "Reclose escaped epic G-300 after corrective child G-301 merged.", + "graph": "G-301 is merged and closed with a focused regression test.", + "closeout": "The escaped boundary regression passes, but the full affected customer journey has not been revalidated." } ] diff --git a/skills/implement-epic/evals/expectations.json b/skills/implement-epic/evals/expectations.json index 8f34db1..65ee8e3 100644 --- a/skills/implement-epic/evals/expectations.json +++ b/skills/implement-epic/evals/expectations.json @@ -47,7 +47,7 @@ { "case_id": "authorized-full-epic-closeout", "workflow_state": "epic_closed", - "required_actions": ["verify every closeout gate", "close G-190", "report final graph and evidence"] + "required_actions": ["verify every child and parent acceptance ledger", "verify current-main and exact deployment identities", "close G-190", "report final graph and evidence"] }, { "case_id": "umbrella-closeout-separately", @@ -93,5 +93,15 @@ "case_id": "verify-stacked-child-result", "workflow_state": "stack_child_verified", "required_actions": ["verify stack topology and every PR gate", "verify all_merged and full-chain representation on base", "do not invoke carve-changesets directly", "refresh graph before considering G-282"] + }, + { + "case_id": "closed-children-missing-manual-browser", + "workflow_state": "closeout_blocked", + "required_actions": ["read the closed child ledger", "record manual browser evidence as missing", "keep G-290 open", "report delivery separately from acceptance"] + }, + { + "case_id": "reopened-correction-missing-journey-revalidation", + "workflow_state": "closeout_blocked", + "required_actions": ["preserve the corrective child result", "require full affected journey revalidation", "keep G-300 open", "do not impose unrelated full-system testing"] } ] diff --git a/skills/implement-epic/references/closeout.md b/skills/implement-epic/references/closeout.md index c5dc91f..3bb1737 100644 --- a/skills/implement-epic/references/closeout.md +++ b/skills/implement-epic/references/closeout.md @@ -1,23 +1,30 @@ # Epic closeout -Use current tracker, repository, and PR evidence. Do not close a parent from -merged-PR count or stale graph state. +Use current tracker, repository, PR, deployment, and criterion-specific +acceptance evidence. Closed children and merged PRs are delivery/administrative +state, not parent acceptance. ## Verify every required outcome -- Reread the complete native child and blocker graph. +- Reread the complete native child and blocker graph, including closed children. - Confirm every required child disposition and every required outcome from a closed, canceled, duplicate, or superseded blocker. +- Read every required child's criterion-specific acceptance ledger and reject + missing, failed, unavailable, stale, wrong-environment, or category-mismatched + evidence regardless of native state. - Confirm every required PR result is represented on the current remote base. -- Verify parent acceptance criteria against resulting behavior, not ticket +- Build and verify the parent's own ledger against resulting behavior, not labels or administrative state. +- Verify current-main representation and the exact deployed SHA whenever a + criterion requires deployment. +- Require screenshot or geometry/computed-layout evidence for explicit visual + requirements; functional browser checks alone are insufficient. - Run required clean-main validation and any parent-level smoke check. - Verify required documentation, migration, compatibility, rollout, deployment, and cleanup outcomes. Deployment and production mutation still require their own authority. -Keep the parent open when a not-planned or canceled child leaves an unsatisfied -outcome. +Keep the parent open when any required outcome or evidence item is unsatisfied. ## Sweep late feedback @@ -28,28 +35,32 @@ for every late actionable item. When late feedback reveals a required correctness, security, acceptance, architecture, or validation gap: -- keep the epic open; -- use or propose the correctly owned ticket; +- keep or reopen the epic when authorized; +- require a focused corrective ticket and a regression test at the escaped + boundary; - invoke `implement-ticket` from a fresh branch based on current remote state - when implementation is authorized; and -- require the normal one-ticket validation, review, merge, transition, and - cleanup result before retrying closeout. + when implementation is authorized; +- revalidate the full affected customer journey proportionally; and +- require renewed acceptance evidence plus the normal one-ticket validation, + review, merge, transition, and cleanup result before retrying closeout. -Never reopen or reuse a merged feature branch as the fix path. Do not reproduce -single-ticket review or merge mechanics here. +Never reopen or reuse a merged feature branch as the fix path. A merged +corrective child does not justify reclosure until affected-journey evidence +passes. Do not reproduce single-ticket review or merge mechanics here. ## Apply close authority -Parent-close authority is separate from child merge and manual child-transition -authority. Without explicit close authority, report that every closeout gate -passes and leave the parent open. +Parent-close authority is separate from child merge, deployment, protected +verification, and manual child-transition authority. Without explicit close +authority, report that every closeout gate passes and leave the parent open. -Close each epic separately. For a series or umbrella, verify all component epics -and umbrella acceptance criteria before the umbrella transition. +Close each epic separately. For a series or umbrella, verify all component child +and parent ledgers before the umbrella transition. ## Report closeout -Record child dispositions, merged PRs and base representation, clean-main and -acceptance evidence, late-feedback dispositions, remaining deferred or blocked -work, the parent transition performed or withheld, and the exact reason any -parent remains open. +Record child dispositions, merged PRs and base representation, +criterion-specific child and parent acceptance ledgers, exact deployment +identities, clean-main evidence, late-feedback dispositions, remaining deferred +or blocked work, the parent transition performed or withheld, and the exact +reason any parent remains open. diff --git a/skills/implement-epic/references/github.md b/skills/implement-epic/references/github.md index e756bd4..40b8700 100644 --- a/skills/implement-epic/references/github.md +++ b/skills/implement-epic/references/github.md @@ -7,12 +7,14 @@ single-ticket mechanics. ## Read native graph state - Resolve the repository and live epic identity. -- Read epic and child titles, bodies, issue types, states, and scope-affecting - comments. +- Read epic and child titles, bodies, issue types, states, scope-affecting + comments, acceptance criteria, and required verification items. - Read native `parent`, `subIssues`, `blockedBy`, and `blocking` relationships through GraphQL or an equivalent structured API. - Read relevant closed-blocker dispositions and verify their delivered outcomes in the authoritative source. +- Read criterion-specific acceptance ledgers for every required child regardless + of native state. - Inspect existing open and merged PR identities only to prevent duplicate child selection; delegate detailed PR state to `implement-ticket`. @@ -21,16 +23,22 @@ label, Markdown task list, or prose when native relationships exist. ## Select and refresh -Choose only an open in-scope child whose native `blockedBy` set has no open -issue and whose required closed-blocker outcomes exist. Treat canceled or -not-planned blockers with missing outcomes as unresolved. - -After a returned `merged` result, verify the GitHub issue transition and reread -the complete parent/sub-issue/blocker graph before selecting another child or -claiming completion. A `ready_pr` or `ready_prs` result does not satisfy a -dependency that requires merge. For a stacked `merged` result, verify the -reported topology, every PR merge, and full-chain representation on the base -without taking ownership of decomposition mechanics. +Choose an in-scope child whose native `blockedBy` set has no open issue, whose +required closed-blocker outcomes exist, and that is either open or auto-closed +with required acceptance still missing. Route the latter through +`implement-ticket` with its closeout observation and granted or withheld reopen +authority. Do not select an accepted, superseded, or otherwise terminal closed +child. Treat canceled or not-planned blockers with missing outcomes as +unresolved. + +After every caller-verified merge, delivery, or GitHub issue transition, reread +the complete graph regardless of the returned terminal state. Verify the live +transition first, then separately decide which edges require delivery and which +require the child's complete current acceptance ledger. A `ready_pr`, +`ready_prs`, or merged delivery with acceptance pending remains incomplete but +still changes graph state. For a stacked delivery, also verify reported +topology, every completed PR transition, and full-chain representation on the +base without taking ownership of decomposition mechanics. When duplicate implementation paths exist, do not choose a competing path; pass the canonical ownership evidence into `implement-ticket` or return `blocked` @@ -46,6 +54,7 @@ Linear adapter and let `implement-ticket` use its GitHub PR-host adapter. ## Close GitHub epics With explicit parent-close authority, transition the GitHub epic only after the -shared closeout reference passes and live native relationships show no required -open child or blocker. Record evidence in the issue when authorized and useful. -Never infer parent completion solely from merged PR count. +shared closeout reference passes, every required child ledger passes, and live +native relationships show no required open child or blocker. Record the ledger +evidence in the issue when authorized and useful. Never infer parent completion +from `CLOSED` children or merged PR count. diff --git a/skills/implement-epic/references/linear.md b/skills/implement-epic/references/linear.md index c16b733..aced913 100644 --- a/skills/implement-epic/references/linear.md +++ b/skills/implement-epic/references/linear.md @@ -5,11 +5,13 @@ status state. Let `implement-ticket` resolve repository and PR-host mechanics. ## Read native graph state -- Read the live parent or epic, project context, children, comments, and linked - specifications. +- Read the live parent or epic, project context, children, comments, linked + specifications, acceptance criteria, and required verification items. - Read explicit blocking and duplicate relationships plus current dispositions. - Verify completed, canceled, duplicate, or superseding blocker outcomes in their authoritative repository, artifact registry, tracker, or environment. +- Read criterion-specific acceptance ledgers for every required child regardless + of Linear state. - Resolve repository and PR identities only far enough to avoid duplicate child selection and build the `implement-ticket` handoff. @@ -20,15 +22,20 @@ graph state. ## Select and refresh -Select only an open in-scope child with no unresolved blocker and verified -prerequisite outcomes. Treat canceled or not-planned blockers with missing -outcomes as unresolved. - -After `implement-ticket` returns `merged`, verify the expected Linear transition -and reread the complete epic relationship state. For a stacked result, also -verify every PR merge and full-chain representation on the base. Do not treat -`ready_pr` or `ready_prs` as a completed child or unblock a dependent that -requires merge. +Select an in-scope child with no unresolved blocker and verified prerequisite +outcomes when it is either open or auto-closed with required acceptance still +missing. Route the latter through `implement-ticket` with its closeout +observation and granted or withheld reopen authority. Do not select an accepted, +superseded, or otherwise terminal closed child. Treat canceled or not-planned +blockers with missing outcomes as unresolved. + +After every caller-verified merge, delivery, or Linear transition, reread the +complete graph regardless of the returned terminal state. Verify the live +transition first, then separately decide which edges require delivery and which +require the child's complete current acceptance ledger. For a stacked delivery, +also verify every completed PR transition and full-chain representation on the +base. A `ready_pr`, `ready_prs`, or merged delivery with acceptance pending +remains incomplete but must still inform the refreshed ready set. ## Separate tracker and PR host @@ -39,7 +46,8 @@ a same-numbered GitHub issue as a substitute for Linear state. ## Close Linear epics Apply the shared closeout reference and require explicit parent-close authority. -Update the epic only when every required child and blocker outcome is satisfied, -acceptance holds on the base, and late feedback is dispositioned. Preserve -deferred or canceled scope in the final report; never count an unmet canceled -outcome as complete. +Update the epic only when every required child ledger and blocker outcome is +satisfied, the parent's own acceptance ledger holds on the current base and +required deployment, and late feedback is dispositioned. Preserve deferred or +canceled scope in the final report; never count an unmet outcome or completed +Linear state as acceptance. diff --git a/skills/implement-epic/scripts/tests/test_orchestration_contract.py b/skills/implement-epic/scripts/tests/test_orchestration_contract.py index a86295d..75098c9 100644 --- a/skills/implement-epic/scripts/tests/test_orchestration_contract.py +++ b/skills/implement-epic/scripts/tests/test_orchestration_contract.py @@ -118,6 +118,63 @@ def test_eval_expectations_preserve_critical_boundaries(self): "stack_child_verified", self.expectations["verify-stacked-child-result"]["workflow_state"], ) + for case_id in ( + "closed-children-missing-manual-browser", + "reopened-correction-missing-journey-revalidation", + ): + self.assertEqual( + "closeout_blocked", self.expectations[case_id]["workflow_state"] + ) + self.assertEqual( + "epic_closed", + self.expectations["authorized-full-epic-closeout"]["workflow_state"], + ) + + def test_verified_delivery_refreshes_graph_even_when_acceptance_blocks(self): + self.assertIn( + "After every verified merge, delivery, or tracker transition", + self.skill, + ) + for adapter in (self.github, self.linear): + self.assertIn("regardless of the returned terminal state", adapter) + self.assertIn("complete current acceptance ledger", adapter) + self.assertIn("merged delivery with acceptance pending", self.github) + self.assertIn("merged delivery with acceptance pending", self.linear) + + def test_closeout_requires_child_and_parent_acceptance_ledgers(self): + self.assertIn("complete native child and blocker graph", self.contract) + self.assertIn("criterion-specific acceptance ledger", self.contract) + self.assertIn("parent's own ledger", self.contract) + self.assertIn("current-main representation", self.contract) + self.assertIn("exact deployed SHA", self.contract) + self.assertIn("functional browser checks alone are insufficient", self.contract) + + def test_delivery_and_acceptance_are_reported_separately(self): + self.assertIn( + '"all native children closed" are delivery and administrative milestones, not epic acceptance', + self.contract, + ) + self.assertIn("Keep the parent open", self.contract) + self.assertIn("Parent-close authority is separate", self.contract) + + def test_auto_closed_incomplete_child_routes_through_ticket_recovery(self): + self.assertIn( + "auto-closed while required acceptance remains missing", self.contract + ) + self.assertIn( + "Route that auto-closed child through `implement-ticket`", self.contract + ) + self.assertIn("granted or withheld reopen authority", self.contract) + self.assertIn( + "Do not select an accepted, superseded, or otherwise terminal closed child", + self.contract, + ) + + def test_reopened_epic_requires_affected_journey_revalidation(self): + self.assertIn("focused corrective child", self.contract) + self.assertIn("regression test at the escaped boundary", self.contract) + self.assertIn("full affected customer journey", self.contract) + self.assertIn("do not impose unrelated full-system testing", self.contract) if __name__ == "__main__": diff --git a/skills/implement-ticket/SKILL.md b/skills/implement-ticket/SKILL.md index 6147ea0..c83c5a7 100644 --- a/skills/implement-ticket/SKILL.md +++ b/skills/implement-ticket/SKILL.md @@ -92,16 +92,19 @@ Use this default authority matrix unless the user or repository is stricter: branch push, PR creation or update, evidence-based review replies, and resolution of fully addressed threads; - `merge after gates` additionally permits merging this ticket's ordinary PR or - carved stack and safely deleting its verified merged feature branches; -- `merge plus manual transition` additionally permits the explicitly requested - status or close transition for this ticket only; + carved stack and safely deleting its verified merged feature branches, but it + does not permit deployment, post-merge verification, or tracker transition; +- `merge plus manual transition` additionally permits only the explicitly + requested status, reopen, or close transition for this ticket after its + acceptance evidence passes; - `decompose oversized candidates into stacked changesets` permits an oversized but coherent ticket candidate to be transferred to `carve-changesets`; it is off by default and is independent of every completion policy; - ticket-body edits, dependency mutations, and follow-up creation require explicit ticket-management authority; and -- deployment, production mutation, destructive data operations, and parent - closure always require separate explicit authority. +- deployment, production mutation, destructive data operations, parent closeout, + and post-merge verification in a protected environment always require their + own explicit authority. Do not infer decomposition, merge, issue-close, parent-close, deployment, or production authority from words such as `implement`, `finish`, `complete`, or @@ -109,11 +112,47 @@ production authority from words such as `implement`, `finish`, `complete`, or stack. When decomposition authority is absent, never silently publish an oversized monolith or silently carve it. -Treat an automatic ticket transition caused by the selected closing syntax as a -disclosed consequence of authorized merge, not as an independently requested -manual transition. State that consequence before publication or merge. Do not -use automatic closing syntax when its effect conflicts with the resolved -completion policy. +Treat an automatic ticket transition caused by closing syntax as a disclosed +consequence only when explicit tracker-transition authority exists and every +required acceptance item can pass before merge. Without tracker-transition +authority, or when any required item is post-merge, use a non-closing reference +and leave the tracker open. Transition it manually only after the evidence +passes and close authority is available. A closed state caused by `Fixes`, a +merged PR, or another automation is delivery state, not acceptance proof. + +## Build the acceptance evidence ledger + +Before readiness, merge, tracker transition, or a completion claim, inventory +every ticket-authored acceptance criterion and required verification item. Do +not add browser, deployment, authenticated, integration, manual, visual, or +full-system gates that the ticket does not require. + +Record one evidence entry per criterion or required verification item in the +existing packet/result style. Each entry includes: + +- the criterion text or stable identity and whether it is required; +- its required evidence category and whether it applies pre-merge or post-merge; +- the exact candidate SHA or deployed SHA it evaluates; +- the environment and URL when relevant; +- the concrete source, such as command output, CI job, deployment run, browser + observation, screenshot, geometry/computed-layout observation, or tracker + comment; and +- `pass`, `fail`, or `missing`. + +Evidence satisfies an item only when its category matches the authored +requirement and its candidate, deployment, and environment are current. +Functional browser behavior, URL stability, DOM content, and clean console +output do not satisfy an explicit visual-layout requirement without a screenshot +or geometry/computed-layout observation. Reject stale candidate or deployment +SHAs and wrong-environment evidence. + +All required pre-merge entries must pass before `ready_pr`, `ready_prs`, or a +merge. Required post-merge entries may remain `missing` through an authorized +merge only when the PR used non-closing syntax; afterward report delivery as +merged but return `blocked` with acceptance pending until every required entry +passes. A required unavailable manual or protected-environment check is a +blocker, not a caveat on success. `merged` is reserved for merged delivery, +complete acceptance evidence, and the authorized tracker transition. ## Honor delegated execution when supplied @@ -122,8 +161,11 @@ Delegated execution is optional. Standalone behavior remains the default. When a caller supplies a delegated invocation: 1. Validate it with the bundled delegated-execution validator before mutation. - Reject unsupported versions, unknown fields, excluded terminal states, or an - unusable checkpoint command. + Reject unsupported versions, unknown fields, excluded terminal states, an + unusable checkpoint command, duplicate or unstructured acceptance + requirements, or a missing explicit starting-deployment snapshot (`null` when + none applies). A starting deployment must identify both its candidate and + deployed SHA. 2. Treat its authority as an additional ceiling. It may narrow the ordinary operating contract but never widen repository, user, host, or provider authority. @@ -136,12 +178,18 @@ When a caller supplies a delegated invocation: 4. Immediately after every candidate push or advancement, verify the full remote ref and exact SHA, then run `candidate_published`. Do not continue until the caller acknowledges that exact SHA. -5. If a checkpoint is denied, unavailable, malformed, ambiguous, or mismatched, +5. When deployment-based acceptance is required, after merge and after any + authorized deployment, reread the authoritative deployment and run + `deployment_observed` with the exact candidate, deployed SHA, environment, + and URL. Continue only when the caller acknowledges a live deployment + observation bound to that candidate. +6. If a checkpoint is denied, unavailable, malformed, ambiguous, or mismatched, stop the proposed action and return `blocked`. Preserve already-published - implementation as a transferable handoff even when publication + implementation as a transferable handoff even when publication or deployment acknowledgement failed. A denial preserves the prior sequence and token. -6. Validate the terminal result against the invocation and the caller's durable - ledger tail. Never return stale checkpoint state, a state the caller +7. Validate the terminal result against the invocation, the caller's durable + ledger tail, and its latest verified deployment observation when deployment + evidence is required. Never return stale checkpoint state, a state the caller excluded, or a claim that a local-only candidate is transferable. A material ticket observation change blocks this invocation; reevaluation starts a fresh one. @@ -221,13 +269,15 @@ or implementing children. Proceed only when the selected ticket: -- is open and is not already implemented, superseded, or represented by a - canonical open or merged PR or branch; +- is open, or was auto-closed while required acceptance remains missing, and is + not already accepted, superseded, or represented by a canonical implementation + owned elsewhere; - has no unresolved native blocker; - has every required closed-blocker outcome verified in its authoritative repository, artifact registry, tracker, or environment; - has a clear observable goal, acceptance criteria, non-goals, preserved - behavior, and required verification; + behavior, required verification, and enough detail to classify each evidence + item as pre-merge or post-merge; - contains no unresolved product, data, authorization, migration, destructive, or architecture decision; - represents one coherent candidate that is expected to fit one reviewable PR, @@ -243,9 +293,11 @@ unimplemented sibling is required; never absorb that sibling into this PR. When an open canonical PR or branch already owns the ticket, return `blocked` with its identity and require explicit ownership transfer before modifying it; do not report another worker's candidate as this run's `ready_pr` or -`ready_prs`. When a merged PR or stack is verified on the base and the ticket is -already complete, return `merged` with that evidence without creating new -implementation state. +`ready_prs`. When a merged PR or stack is verified on the base, return `merged` +without new implementation state only if the criterion-specific acceptance +ledger is current and complete and the tracker transition is correct. Otherwise +report merged delivery with acceptance pending and continue only within the +available post-merge verification and tracker authority. When ticket editing is authorized, make an unclear ticket implementation-ready and re-read it. Otherwise stop with the missing decision rather than @@ -294,9 +346,12 @@ Discover commands from repository instructions and tooling. Run: 4. integration tests with documented real dependencies; and 5. required build, packaging, or manual checks. -Report commands and exact outcomes. Distinguish bootstrap or environment -failures from feature failures. Do not claim completion while required -validation is failing or unavailable. +Report commands and exact outcomes. Add each ticket-required check to the +acceptance ledger with its evidence category, candidate or deployed SHA, +environment, source, and status. Distinguish bootstrap or environment failures +from feature failures. Do not claim completion while required validation or +acceptance evidence is failing, missing, unavailable, stale, or +category-mismatched. ### 4. Run bounded repository-owned review @@ -358,12 +413,15 @@ to its merge-and-propagate boundary and `all_merged`. `implement-ticket` performs no direct `babysit-pr` handoff, watcher, retry, feedback, fix, or merge loop for any stack PR. Exactly one watcher owner exists per PR. -In either path, describe the ticket-wide outcome, important non-goals, and -actual validation. Use the owning tracker's correct closing syntax on the one -ordinary PR or only on the final changeset PR. Intermediate stack PRs use a -non-closing reference and must remain behaviorally safe under the -`carve-changesets` equivalence contract. Verify the ticket transition only after -the ordinary PR merges or `carve-changesets` returns `all_merged`. +In either path, describe the ticket-wide outcome, important non-goals, actual +validation, and acceptance-ledger state. Use closing syntax on the one ordinary +PR or final changeset PR only when explicit tracker-transition authority exists +and every required acceptance item can pass before merge. Without that +authority, or when any required item is post-merge, use `Refs #`, +`Supports #`, or the tracker's established non-closing equivalent on all +PRs. Transition the ticket manually only after the ledger passes and transition +authority exists. Intermediate stack PRs always use a non-closing reference and +remain behaviorally safe under the `carve-changesets` equivalence contract. Normal ticket execution never uses `watch_until_closed`. Ordinary pending CI or review time is not a blocker; retain task ownership through the selected @@ -372,12 +430,33 @@ user-help-required condition occurs. Validate the returned identity and evidence against live GitHub state. After an authorized ordinary merge or `all_merged`, independently verify remote merge -state, complete mainline representation, and the owning tracker's ticket -transition before cleanup. A mid-stack material redesign that invalidates an -earlier merged changeset returns `blocked`; never paper it over by mutating -merged history. For an epic child, reread affected native dependency -relationships and report newly unblocked work without selecting or mutating it. -Never close or verify a parent epic from this skill. +state and complete mainline representation, then run every required post-merge +acceptance item with the separately granted environment and deployment +authority. If any item is missing, failed, unavailable, stale, or bound to the +wrong SHA/environment, return `blocked` with merged delivery preserved and keep +the ticket open. + +If closing automation already transitioned the ticket while required evidence +remains missing, do not treat the closed state as proof. Reopen it when manual +transition authority permits; otherwise return `blocked` and name the missing +reopen authority. Close manually only after the ledger passes and close +authority exists. If merge delivery and acceptance pass but tracker-transition +authority is absent, preserve the merged candidate, keep the tracker open, and +return `blocked` naming that authority gap. A mid-stack material redesign that +invalidates an earlier merged changeset returns `blocked`; never paper it over +by mutating merged history. For an epic child, reread affected native dependency +relationships only after acceptance and the ticket transition pass, and report +newly unblocked work without selecting or mutating it. Never close or verify a +parent epic from this skill. + +## Revalidate escaped acceptance defects + +When a ticket was supposedly complete and is reopened for an escaped acceptance +defect, require a focused corrective ticket, a regression test at the escaped +boundary, and revalidation of the full affected customer journey. The journey +scope comes from the escaped behavior and authored requirements; do not impose +unrelated full-system testing. Rebuild the affected acceptance entries against +the corrective candidate or deployment before reclosure. ## Stop conditions @@ -385,11 +464,12 @@ Stop and return `blocked` when: - ticket scope and native relationships conflict materially; - implementation requires an unresolved product or architecture choice; -- a prerequisite outcome, authority, credential, approval, or required - infrastructure is missing; +- a prerequisite outcome, authority, credential, approval, required + infrastructure, or required acceptance evidence is missing; - correctness would materially exceed one-ticket scope; - review feedback requires redesigning the ticket; or -- required validation remains unavailable after documented bootstrap attempts. +- required validation or acceptance evidence remains unavailable after + documented bootstrap attempts. Difficulty, a long test suite, ordinary CI wait time, or independently ready sibling work is not a blocker. @@ -400,20 +480,23 @@ Follow [cleanup and result](references/cleanup-and-result.md). Return exactly one terminal state: - `ready_pr`: the ticket's one ordinary PR is open and mergeable at the reported - candidate, every applicable current-candidate non-merge gate has passed, merge - was withheld, and this run owns or was explicitly handed ownership of the - candidate; -- `ready_prs`: the ticket's carved stack is open with verified topology and - every PR at its applicable non-merge gate, merge was withheld, and - `carve-changesets` returned current-candidate `prs_open` evidence; -- `merged`: either the ordinary PR is verified on the base or the full carved - stack is verified there after `all_merged`; in both cases the ticket - transition and authorized cleanup are verified; + candidate, every required pre-merge acceptance entry and applicable + current-candidate non-merge gate has passed, merge was withheld, and this run + owns or was explicitly handed ownership of the candidate; +- `ready_prs`: the ticket's carved stack is open with verified topology, every + required pre-merge acceptance entry and per-PR non-merge gate has passed, + merge was withheld, and `carve-changesets` returned current-candidate + `prs_open` evidence; +- `merged`: the ordinary PR or full carved stack is verified on the base, every + required pre-merge and post-merge acceptance entry passes for the current + candidate/deployment and environment, and the authorized ticket transition and + cleanup are verified; - `blocked`: give one concrete blocking reason and next action, preserving any - partial artifacts; or + partial or merged delivery artifacts and identifying acceptance-pending state; + or - `requires_epic`: no mutation occurred and the handoff names `implement-epic` with its stable routing marker. -When this ticket is an epic child, report newly unblocked downstream work after -merge but do not select or implement it. Never claim whole-epic acceptance or -close a parent. +When this ticket is an epic child, report newly unblocked downstream work only +after the child acceptance ledger and authorized tracker transition pass; do not +select or implement it. Never claim whole-epic acceptance or close a parent. diff --git a/skills/implement-ticket/evals/cases.json b/skills/implement-ticket/evals/cases.json index 897a04f..b53a0ff 100644 --- a/skills/implement-ticket/evals/cases.json +++ b/skills/implement-ticket/evals/cases.json @@ -145,5 +145,44 @@ "request": "Finish the authorized carved stack for G-45.", "authority": "merge_after_gates plus decompose oversized candidates into stacked changesets", "candidate_state": "Changeset 1 is merged; material feedback on changeset 2 requires redesign that invalidates changeset 1." + }, + { + "id": "auto-closed-missing-postmerge-acceptance", + "request": "Finish GitHub issue G-46 after its PR auto-closed the ticket.", + "tracker_state": "G-46 is closed and its PR is merged, but the required production deployment evidence is missing.", + "authority": "merge and manual transition granted; deployment authority absent", + "candidate_state": "Pre-merge evidence passes for the merged head; post-merge deployment evidence is missing." + }, + { + "id": "authenticated-browser-unavailable", + "request": "Complete deployed GitHub issue G-47.", + "tracker_state": "G-47 requires an authenticated production journey.", + "candidate_state": "The exact deployed SHA is known, but no authenticated browser capability is available." + }, + { + "id": "functional-browser-without-visual-evidence", + "request": "Close GitHub issue G-48 after its browser flow passes.", + "tracker_state": "G-48 explicitly requires both functional behavior and visual panel alignment.", + "candidate_state": "The functional browser flow passes on the exact deployment; screenshot or computed-layout evidence is missing." + }, + { + "id": "backend-only-acceptance", + "request": "Implement and merge backend-only GitHub issue G-49.", + "tracker_state": "G-49 requires only parser contract tests and repository review; it specifies no UI, browser, or deployment criterion.", + "authority": "merge_after_gates", + "candidate_state": "The required command evidence, review, and current-head gates pass." + }, + { + "id": "merge-without-deploy-or-close-authority", + "request": "Implement and merge GitHub issue G-50.", + "tracker_state": "G-50 has a required post-merge deployment criterion.", + "authority": "merge granted; deployment and tracker-transition authority absent", + "candidate_state": "Every pre-merge gate passes and the PR merges, but deployment evidence remains missing." + }, + { + "id": "stale-acceptance-evidence", + "request": "Publish GitHub issue G-51 using the prior acceptance run.", + "tracker_state": "G-51 is open and independently shippable.", + "candidate_state": "Required command evidence passed for an older head; the candidate head changed afterward." } ] diff --git a/skills/implement-ticket/evals/expectations.json b/skills/implement-ticket/evals/expectations.json index 803498f..f43a6e4 100644 --- a/skills/implement-ticket/evals/expectations.json +++ b/skills/implement-ticket/evals/expectations.json @@ -118,5 +118,35 @@ "case_id": "mid-stack-material-redesign", "terminal_state": "blocked", "required_actions": ["preserve partial stack state", "report the invalidated merged changeset", "require product or architecture resolution", "do not rewrite merged history"] + }, + { + "case_id": "auto-closed-missing-postmerge-acceptance", + "terminal_state": "blocked", + "required_actions": ["record merged delivery separately", "reopen G-46 when authorized", "keep acceptance pending", "name missing deployment authority and evidence"] + }, + { + "case_id": "authenticated-browser-unavailable", + "terminal_state": "blocked", + "required_actions": ["record authenticated browser evidence as missing", "keep the tracker open", "do not substitute unauthenticated or CI evidence"] + }, + { + "case_id": "functional-browser-without-visual-evidence", + "terminal_state": "blocked", + "required_actions": ["preserve the passing functional evidence", "require screenshot or geometry evidence", "do not claim visual acceptance"] + }, + { + "case_id": "backend-only-acceptance", + "terminal_state": "merged", + "required_actions": ["build the criterion ledger", "use the current command and review evidence", "add no browser visual or deployment gate", "merge after current gates pass"] + }, + { + "case_id": "merge-without-deploy-or-close-authority", + "terminal_state": "blocked", + "required_actions": ["use non-closing PR syntax", "report merged delivery separately", "preserve deployment and transition authority boundaries", "keep acceptance pending"] + }, + { + "case_id": "stale-acceptance-evidence", + "terminal_state": "blocked", + "required_actions": ["reject old-head evidence", "rebuild required acceptance for the current head", "do not publish or claim ready_pr"] } ] diff --git a/skills/implement-ticket/evals/forward_cases.json b/skills/implement-ticket/evals/forward_cases.json index fe320f1..a902357 100644 --- a/skills/implement-ticket/evals/forward_cases.json +++ b/skills/implement-ticket/evals/forward_cases.json @@ -6,7 +6,7 @@ "authority": {}, "capabilities": {"review_code_change": false, "babysit_pr": false}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-301", "state": "open", "whole_epic": true, "subissues": ["G-302"]}, + "ticket": {"tracker": "github", "id": "G-301", "state": "open", "whole_epic": true, "subissues": ["G-302"], "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "one ticket per PR"}, "pr": {"state": "absent", "merged": false, "head": null, "base": "base-1", "mergeable": null}, "diff": {"base": "base-1", "head": null, "patch": "", "resulting_tree": null}, @@ -24,7 +24,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": false}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-303", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-303", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "one ticket per PR"}, "pr": {"state": "absent", "merged": false, "head": null, "base": "base-1", "mergeable": null}, "diff": {"base": "base-1", "head": null, "patch": "", "resulting_tree": null}, @@ -42,7 +42,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-304", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-304", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "CI and clean review required"}, "pr": {"state": "open", "merged": false, "head": "head-304", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-304", "patch": "ticket-scoped patch", "resulting_tree": "tree-304"}, @@ -57,10 +57,10 @@ "id": "authorized-merge-closeout", "target_skill": "implement-ticket", "request": "Implement and merge G-305.", - "authority": {"merge": true}, + "authority": {"merge": true, "tracker_transition": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-305", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-305", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "merge commits allowed"}, "pr": {"state": "open", "merged": false, "head": "head-305", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-305", "patch": "ticket-scoped patch", "resulting_tree": "tree-305"}, @@ -75,10 +75,10 @@ "id": "linear-ticket-github-pr", "target_skill": "implement-ticket", "request": "Implement and merge Linear ticket LIN-306 through a GitHub PR.", - "authority": {"merge": true}, + "authority": {"merge": true, "tracker_transition": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "linear", "id": "LIN-306", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "linear", "id": "LIN-306", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "linear", "pr_host": "github", "instructions": "Linear owns status"}, "pr": {"state": "open", "merged": false, "head": "head-306", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-306", "patch": "ticket-scoped patch", "resulting_tree": "tree-306"}, @@ -96,7 +96,7 @@ "authority": {"merge": false, "reply": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-307", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-307", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "resolve addressed threads"}, "pr": {"state": "open", "merged": false, "head": "head-307b", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-307b", "patch": "ticket fix", "resulting_tree": "tree-307b"}, @@ -111,10 +111,10 @@ "id": "branch-caused-ci-fix", "target_skill": "implement-ticket", "request": "Implement and merge G-308 after repairing its CI regression.", - "authority": {"merge": true}, + "authority": {"merge": true, "tracker_transition": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-308", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-308", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "full tests required"}, "pr": {"state": "open", "merged": false, "head": "head-308b", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-308b", "patch": "CI fix", "resulting_tree": "tree-308b"}, @@ -132,7 +132,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-309", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-309", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "bounded retries"}, "pr": {"state": "open", "merged": false, "head": "head-309", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-309", "patch": "ticket patch", "resulting_tree": "tree-309"}, @@ -150,7 +150,7 @@ "authority": {"merge": false, "reply": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-310", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-310", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "human replies require authority"}, "pr": {"state": "open", "merged": false, "head": "head-310", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-310", "patch": "ticket patch", "resulting_tree": "tree-310"}, @@ -168,7 +168,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-311", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-311", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "current-head connector required"}, "pr": {"state": "open", "merged": false, "head": "head-311b", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-311b", "patch": "ticket patch", "resulting_tree": "tree-311b"}, @@ -183,10 +183,10 @@ "id": "unrelated-base-drift", "target_skill": "implement-ticket", "request": "Merge G-312 after unrelated base drift.", - "authority": {"merge": true}, + "authority": {"merge": true, "tracker_transition": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-312", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-312", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "unrelated drift may retain evidence"}, "pr": {"state": "open", "merged": false, "head": "head-312", "base": "base-2", "mergeable": true}, "diff": {"base": "base-2", "head": "head-312", "base_drift": "unrelated", "patch": "unchanged patch", "resulting_tree": "tree-312"}, @@ -201,10 +201,10 @@ "id": "relevant-base-drift", "target_skill": "implement-ticket", "request": "Merge G-313 after overlapping base drift.", - "authority": {"merge": true}, + "authority": {"merge": true, "tracker_transition": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-313", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-313", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "relevant drift invalidates gates"}, "pr": {"state": "open", "merged": false, "head": "head-313b", "base": "base-2", "mergeable": true}, "diff": {"base": "base-2", "head": "head-313b", "base_drift": "relevant", "patch": "reconciled patch", "resulting_tree": "tree-313b"}, @@ -222,7 +222,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-314", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-314", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "head changes reset gates"}, "pr": {"state": "open", "merged": false, "head": "head-314b", "base": "base-1", "mergeable": true, "external_head_change": true}, "diff": {"base": "base-1", "head": "head-314b", "patch": "new patch", "resulting_tree": "tree-314b"}, @@ -240,7 +240,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-315", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-315", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "preserve closed PR artifacts"}, "pr": {"state": "closed", "merged": false, "head": "head-315", "base": "base-1", "mergeable": false}, "diff": {"base": "base-1", "head": "head-315", "patch": "ticket patch", "resulting_tree": "tree-315"}, @@ -258,7 +258,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-316", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-316", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "candidate-bound results"}, "pr": {"state": "open", "merged": false, "head": "head-316b", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-316b", "patch": "ticket patch", "resulting_tree": "tree-316b"}, @@ -276,7 +276,7 @@ "authority": {"merge": false}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-317", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-317", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "one mutating owner"}, "pr": {"state": "open", "merged": false, "head": "head-317", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-317", "patch": "ticket patch", "resulting_tree": "tree-317"}, @@ -291,10 +291,10 @@ "id": "resumed-pr-deduplication", "target_skill": "implement-ticket", "request": "Resume and merge G-318 from its canonical PR.", - "authority": {"merge": true}, + "authority": {"merge": true, "tracker_transition": true}, "capabilities": {"review_code_change": true, "babysit_pr": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-318", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-318", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "adopt canonical PR only"}, "pr": {"state": "open", "merged": false, "head": "head-318", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-318", "patch": "ticket patch", "resulting_tree": "tree-318"}, @@ -312,7 +312,7 @@ "authority": {"merge": true}, "capabilities": {"implement_ticket": true, "babysit_pr": false}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-319", "state": "open", "whole_epic": true, "children": ["G-320", "G-321", "G-322"]}, + "ticket": {"tracker": "github", "id": "G-319", "state": "open", "whole_epic": true, "children": ["G-320", "G-321", "G-322"], "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "epic invokes implement-ticket only"}, "pr": {"state": "multiple", "merged": false, "head": null, "base": "base-2", "mergeable": null}, "diff": {"base": "base-2", "head": null, "patch": "", "resulting_tree": "tree-epic"}, @@ -330,7 +330,7 @@ "authority": {"merge": false, "decompose_oversized": false}, "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-323", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-323", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "one ticket per candidate"}, "pr": {"state": "absent", "merged": false, "head": "head-323", "base": "base-1", "mergeable": null}, "diff": {"base": "base-1", "head": "head-323", "patch": "large coherent candidate", "resulting_tree": "tree-323", "guardrail": "oversized"}, @@ -348,7 +348,7 @@ "authority": {"merge": false, "decompose_oversized": true}, "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-324", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-324", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "closing syntax only on final stack PR"}, "pr": {"state": "multiple_open", "merged": false, "head": "stack-tip-324", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-324", "patch": "large coherent candidate", "resulting_tree": "tree-324", "guardrail": "oversized"}, @@ -366,7 +366,7 @@ "authority": {"merge": false, "decompose_oversized": true, "ticket_management": true}, "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-325", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-325", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "split independently valuable outcomes at tracker level"}, "pr": {"state": "absent", "merged": false, "head": "head-325", "base": "base-1", "mergeable": null}, "diff": {"base": "base-1", "head": "head-325", "patch": "three independently valuable outcomes", "resulting_tree": "tree-325", "guardrail": "oversized"}, @@ -384,7 +384,7 @@ "authority": {"merge": true, "decompose_oversized": true}, "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-326", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-326", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "never rewrite merged history"}, "pr": {"state": "partial_stack", "merged": false, "head": "stack-326-2", "base": "base-1", "mergeable": false}, "diff": {"base": "base-1", "head": "head-326", "patch": "large coherent candidate", "resulting_tree": "tree-326", "guardrail": "oversized"}, @@ -402,7 +402,7 @@ "authority": {"merge": true, "decompose_oversized": true}, "capabilities": {"implement_ticket": true, "babysit_pr": false, "carve_changesets": false}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-327", "state": "open", "whole_epic": true, "children": ["G-328", "G-329"]}, + "ticket": {"tracker": "github", "id": "G-327", "state": "open", "whole_epic": true, "children": ["G-328", "G-329"], "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "epic verifies ticket results without decomposition mechanics"}, "pr": {"state": "multiple_merged", "merged": true, "head": "stack-tip-328", "base": "base-2", "mergeable": null}, "diff": {"base": "base-2", "head": "stack-tip-328", "patch": "three changesets", "resulting_tree": "tree-328"}, @@ -420,7 +420,7 @@ "authority": {"merge": false, "decompose_oversized": true}, "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": false}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-330", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-330", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "fail closed when the publication dependency is unavailable"}, "pr": {"state": "absent", "merged": false, "head": "head-330", "base": "base-1", "mergeable": null}, "diff": {"base": "base-1", "head": "head-330", "patch": "large coherent candidate", "resulting_tree": "tree-330", "guardrail": "oversized"}, @@ -438,7 +438,7 @@ "authority": {"merge": false, "decompose_oversized": true}, "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, "artifacts": { - "ticket": {"tracker": "github", "id": "G-331", "state": "open", "whole_epic": false}, + "ticket": {"tracker": "github", "id": "G-331", "state": "open", "whole_epic": false, "acceptance_evidence": []}, "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "candidate-bound stack results"}, "pr": {"state": "multiple_open", "merged": false, "head": "stack-tip-331b", "base": "base-1", "mergeable": true}, "diff": {"base": "base-1", "head": "head-331b", "patch": "large coherent candidate", "resulting_tree": "tree-331b", "guardrail": "oversized"}, @@ -448,5 +448,293 @@ "worktree": {"path": "/worktrees/g331", "exclusive_owner": true, "tracked": [], "untracked": []}, "handoff": {"created": true, "result_well_formed": false, "result_stale": true, "rubric": "carve_coherent_branch", "carve_terminal": "prs_open", "result_head": "stack-tip-331a"} } + }, + { + "id": "epic-closed-children-missing-manual-browser", + "target_skill": "implement-epic", + "request": "Close epic G-340 after every child PR merged and every child ticket closed.", + "authority": {"merge": true, "tracker_transition": true}, + "capabilities": {"implement_ticket": true, "babysit_pr": false}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-340", "state": "open", "whole_epic": true, "children": ["G-341", "G-342"], "acceptance_requirements": [{"criterion": "Authenticated workflow succeeds", "required": true, "evidence_category": "manual_browser", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "authenticated browser run"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "closed children are delivery state, not acceptance"}, + "pr": {"state": "multiple_merged", "merged": true, "head": "head-340", "base": "main-340", "mergeable": null}, + "diff": {"base": "main-339", "head": "head-340", "patch": "epic changes", "resulting_tree": "tree-340"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-340", "current_deployment_candidate_sha": "head-340", "children_closed": true, "acceptance_observations": []} + } + }, + { + "id": "auto-closed-missing-postmerge-deployment", + "target_skill": "implement-ticket", + "request": "Finish ticket G-350, which auto-closed when its PR merged.", + "authority": {"merge": true, "tracker_transition": true, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-350", "state": "closed", "whole_epic": false, "acceptance_requirements": [{"criterion": "Tests pass", "required": true, "evidence_category": "ci", "stage": "pre_merge", "identity": "candidate", "environment": "ci", "url": null, "source": "check-run"}, {"criterion": "Production deploy is verified", "required": true, "evidence_category": "deployment", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "deployment-check"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "auto-close does not prove acceptance"}, + "pr": {"state": "merged", "merged": true, "head": "head-350", "base": "main", "mergeable": null}, + "diff": {"base": "base-350", "head": "head-350", "patch": "feature", "resulting_tree": "tree-350"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": null, "closing_syntax": "Fixes #350", "acceptance_observations": [{"criterion": "Tests pass", "evidence_category": "ci", "stage": "pre_merge", "candidate_sha": "head-350", "deployed_sha": null, "environment": "ci", "url": null, "source": "check-run", "outcome": "passed"}]} + } + }, + { + "id": "authenticated-deployed-browser-unavailable", + "target_skill": "implement-ticket", + "request": "Verify ticket G-360 after deployment when the authenticated browser is unavailable.", + "authority": {"merge": true, "tracker_transition": true, "deploy": true}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-360", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Authenticated production journey succeeds", "required": true, "evidence_category": "authenticated_browser", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "authenticated browser capability probe"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "required unavailable evidence fails closed"}, + "pr": {"state": "merged", "merged": true, "head": "head-360", "base": "main", "mergeable": null}, + "diff": {"base": "base-360", "head": "head-360", "patch": "auth flow", "resulting_tree": "tree-360"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-360", "current_deployment_candidate_sha": "head-360", "acceptance_observations": [{"criterion": "Authenticated production journey succeeds", "evidence_category": "authenticated_browser", "stage": "post_merge", "candidate_sha": null, "deployed_sha": "deploy-360", "environment": "production", "url": null, "source": "authenticated browser capability probe", "outcome": "unavailable"}]} + } + }, + { + "id": "functional-browser-missing-visual-layout", + "target_skill": "implement-ticket", + "request": "Close G-370 after its functional browser test passes.", + "authority": {"merge": true, "tracker_transition": true, "deploy": true}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-370", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Controls work", "required": true, "evidence_category": "functional_browser", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "browser-run"}, {"criterion": "Panel is visually aligned", "required": true, "evidence_category": "visual_layout", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "layout evidence"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "functional evidence does not satisfy visual layout"}, + "pr": {"state": "merged", "merged": true, "head": "head-370", "base": "main", "mergeable": null}, + "diff": {"base": "base-370", "head": "head-370", "patch": "panel", "resulting_tree": "tree-370"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-370", "current_deployment_candidate_sha": "head-370", "acceptance_observations": [{"criterion": "Controls work", "evidence_category": "functional_browser", "stage": "post_merge", "candidate_sha": null, "deployed_sha": "deploy-370", "environment": "production", "url": null, "source": "browser-run", "outcome": "passed"}]} + } + }, + { + "id": "all-acceptance-current", + "target_skill": "implement-ticket", + "request": "Merge and complete G-380 after all authored acceptance evidence passes.", + "authority": {"merge": true, "tracker_transition": true, "deploy": true}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-380", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Contract tests pass", "required": true, "evidence_category": "command", "stage": "pre_merge", "identity": "candidate", "environment": "worktree", "url": null, "source": "just test"}, {"criterion": "Current deployment responds", "required": true, "evidence_category": "deployment", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "deployment-check"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "complete only with current criterion evidence"}, + "pr": {"state": "merged", "merged": true, "head": "head-380", "base": "main", "mergeable": null}, + "diff": {"base": "base-380", "head": "head-380", "patch": "feature", "resulting_tree": "tree-380"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": "/worktrees/g380", "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-380", "current_deployment_candidate_sha": "head-380", "deployment_observed_after_merge": true, "acceptance_observations": [{"criterion": "Contract tests pass", "evidence_category": "command", "stage": "pre_merge", "candidate_sha": "head-380", "deployed_sha": null, "environment": "worktree", "url": null, "source": "just test", "outcome": "passed"}, {"criterion": "Current deployment responds", "evidence_category": "deployment", "stage": "post_merge", "candidate_sha": null, "deployed_sha": "deploy-380", "environment": "production", "url": null, "source": "deployment-check", "outcome": "passed"}]} + } + }, + { + "id": "backend-only-no-ui-gates", + "target_skill": "implement-ticket", + "request": "Merge backend-only G-390 after its authored command acceptance passes.", + "authority": {"merge": true, "tracker_transition": true, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-390", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Parser contract suite passes", "required": true, "evidence_category": "command", "stage": "pre_merge", "identity": "candidate", "environment": "worktree", "url": null, "source": "just test-parser"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "do not invent UI acceptance"}, + "pr": {"state": "open", "merged": false, "head": "head-390", "base": "main", "mergeable": true}, + "diff": {"base": "base-390", "head": "head-390", "patch": "parser", "resulting_tree": "tree-390"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": "/worktrees/g390", "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "acceptance_observations": [{"criterion": "Parser contract suite passes", "evidence_category": "command", "stage": "pre_merge", "candidate_sha": "head-390", "deployed_sha": null, "environment": "worktree", "url": null, "source": "just test-parser", "outcome": "passed"}]} + } + }, + { + "id": "merge-without-deploy-or-close-authority", + "target_skill": "implement-ticket", + "request": "Merge G-400; deployment and tracker transition were not authorized.", + "authority": {"merge": true, "tracker_transition": false, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-400", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Tests pass", "required": true, "evidence_category": "ci", "stage": "pre_merge", "identity": "candidate", "environment": "ci", "url": null, "source": "check-run"}, {"criterion": "Production deploy is verified", "required": true, "evidence_category": "deployment", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "deployment-check"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "merge authority does not grant deploy or close authority"}, + "pr": {"state": "merged", "merged": true, "head": "head-400", "base": "main", "mergeable": null}, + "diff": {"base": "base-400", "head": "head-400", "patch": "feature", "resulting_tree": "tree-400"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": null, "acceptance_observations": [{"criterion": "Tests pass", "evidence_category": "ci", "stage": "pre_merge", "candidate_sha": "head-400", "deployed_sha": null, "environment": "ci", "url": null, "source": "check-run", "outcome": "passed"}]} + } + }, + { + "id": "reopened-epic-correction-without-journey-revalidation", + "target_skill": "implement-epic", + "request": "Reclose G-410 after its focused corrective child merged.", + "authority": {"merge": true, "tracker_transition": true, "deploy": true}, + "capabilities": {"implement_ticket": true, "babysit_pr": false}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-410", "state": "open", "whole_epic": true, "children": ["G-411"], "acceptance_requirements": [{"criterion": "Escaped boundary regression passes", "required": true, "evidence_category": "regression", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "regression-run"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "corrective merge alone does not reaccept the journey"}, + "pr": {"state": "merged", "merged": true, "head": "head-411", "base": "main", "mergeable": null}, + "diff": {"base": "base-411", "head": "head-411", "patch": "focused correction", "resulting_tree": "tree-411"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-410", "current_deployment_candidate_sha": "head-411", "escaped_acceptance_defect": true, "affected_journey_revalidated": false, "acceptance_observations": [{"criterion": "Escaped boundary regression passes", "evidence_category": "regression", "stage": "post_merge", "candidate_sha": null, "deployed_sha": "deploy-410", "environment": "production", "url": null, "source": "regression-run", "outcome": "passed"}]} + } + }, + { + "id": "stale-acceptance-evidence", + "target_skill": "implement-ticket", + "request": "Publish G-420 using its prior successful acceptance run.", + "authority": {"merge": false, "tracker_transition": false, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-420", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Contract tests pass", "required": true, "evidence_category": "command", "stage": "pre_merge", "identity": "candidate", "environment": "worktree", "url": null, "source": "just test"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "acceptance evidence is candidate bound"}, + "pr": {"state": "open", "merged": false, "head": "head-420-new", "base": "main", "mergeable": true}, + "diff": {"base": "base-420", "head": "head-420-new", "patch": "new head", "resulting_tree": "tree-420-new"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": "/worktrees/g420", "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "acceptance_observations": [{"criterion": "Contract tests pass", "evidence_category": "command", "stage": "pre_merge", "candidate_sha": "head-420-old", "deployed_sha": null, "environment": "worktree", "url": null, "source": "just test", "outcome": "passed"}]} + } + }, + { + "id": "epic-auto-closed-child-incomplete", + "target_skill": "implement-epic", + "request": "Continue epic G-430 by recovering auto-closed child G-431, whose required production acceptance is missing.", + "authority": {"merge": true, "tracker_transition": true, "deploy": false}, + "capabilities": {"implement_ticket": true, "babysit_pr": false}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-431", "state": "closed", "whole_epic": false, "acceptance_requirements": [{"criterion": "Tests pass", "required": true, "evidence_category": "ci", "stage": "pre_merge", "identity": "candidate", "environment": "ci", "url": null, "source": "check-run"}, {"criterion": "Production journey passes", "required": true, "evidence_category": "manual_browser", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "production browser run"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "auto-closed child remains selectable until acceptance is complete"}, + "pr": {"state": "merged", "merged": true, "head": "head-431", "base": "main", "mergeable": null}, + "diff": {"base": "base-431", "head": "head-431", "patch": "child feature", "resulting_tree": "tree-431"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": null, "auto_closed_incomplete_child": true, "acceptance_observations": [{"criterion": "Tests pass", "evidence_category": "ci", "stage": "pre_merge", "candidate_sha": "head-431", "deployed_sha": null, "environment": "ci", "url": null, "source": "check-run", "outcome": "passed"}]} + } + }, + { + "id": "prior-unrelated-deployment-evidence", + "target_skill": "implement-ticket", + "request": "Complete G-440 using the production deployment that was already live before this candidate merged.", + "authority": {"merge": true, "tracker_transition": true, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-440", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Tests pass", "required": true, "evidence_category": "ci", "stage": "pre_merge", "identity": "candidate", "environment": "ci", "url": null, "source": "check-run"}, {"criterion": "Production deployment responds", "required": true, "evidence_category": "deployment", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "prior-deployment-check"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "prior deployment belongs to a different candidate"}, + "pr": {"state": "merged", "merged": true, "head": "head-440", "base": "main", "mergeable": null}, + "diff": {"base": "base-440", "head": "head-440", "patch": "feature", "resulting_tree": "tree-440"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-old", "current_deployment_candidate_sha": "head-old", "acceptance_observations": [{"criterion": "Tests pass", "evidence_category": "ci", "stage": "pre_merge", "candidate_sha": "head-440", "deployed_sha": null, "environment": "ci", "url": null, "source": "check-run", "outcome": "passed"}, {"criterion": "Production deployment responds", "evidence_category": "deployment", "stage": "post_merge", "candidate_sha": null, "deployed_sha": "deploy-old", "environment": "production", "url": null, "source": "prior-deployment-check", "outcome": "passed"}]} + } + }, + { + "id": "missing-acceptance-ledger", + "target_skill": "implement-ticket", + "request": "Publish G-450 as ready without recording its authored acceptance contract.", + "authority": {"merge": false, "tracker_transition": false, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-450", "state": "open", "whole_epic": false}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "acceptance contract observation is required before readiness"}, + "pr": {"state": "open", "merged": false, "head": "head-450", "base": "main", "mergeable": true}, + "diff": {"base": "base-450", "head": "head-450", "patch": "feature", "resulting_tree": "tree-450"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": "/worktrees/g450", "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true} + } + }, + { + "id": "merge-without-tracker-transition-authority", + "target_skill": "implement-ticket", + "request": "Merge G-451 after acceptance passes, but do not transition its tracker.", + "authority": {"merge": true, "tracker_transition": false, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-451", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Contract tests pass", "required": true, "evidence_category": "command", "stage": "pre_merge", "identity": "candidate", "environment": "worktree", "url": null, "source": "just test"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "merge authority does not grant tracker transition authority"}, + "pr": {"state": "merged", "merged": true, "head": "head-451", "base": "main", "mergeable": null}, + "diff": {"base": "base-451", "head": "head-451", "patch": "feature", "resulting_tree": "tree-451"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": "/worktrees/g451", "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "closing_syntax": "Refs #451", "acceptance_observations": [{"criterion": "Contract tests pass", "evidence_category": "command", "stage": "pre_merge", "candidate_sha": "head-451", "deployed_sha": null, "environment": "worktree", "url": null, "source": "just test", "outcome": "passed"}]} + } + }, + { + "id": "wrong-source-acceptance-evidence", + "target_skill": "implement-ticket", + "request": "Publish G-460 using a delegate summary in place of the authored command source.", + "authority": {"merge": false, "tracker_transition": false, "deploy": false}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-460", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Contract tests pass", "required": true, "evidence_category": "command", "stage": "pre_merge", "identity": "candidate", "environment": "worktree", "url": null, "source": "just test"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "the authored evidence source is part of the acceptance contract"}, + "pr": {"state": "open", "merged": false, "head": "head-460", "base": "main", "mergeable": true}, + "diff": {"base": "base-460", "head": "head-460", "patch": "feature", "resulting_tree": "tree-460"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": "/worktrees/g460", "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "acceptance_observations": [{"criterion": "Contract tests pass", "evidence_category": "command", "stage": "pre_merge", "candidate_sha": "head-460", "deployed_sha": null, "environment": "worktree", "url": null, "source": "delegate summary", "outcome": "passed"}]} + } + }, + { + "id": "deployment-requirement-rejects-candidate-fallback", + "target_skill": "implement-ticket", + "request": "Complete G-470 because its candidate matches even though the observed deployment is not current.", + "authority": {"merge": true, "tracker_transition": true, "deploy": true}, + "capabilities": {"review_code_change": true, "babysit_pr": true, "carve_changesets": true}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-470", "state": "open", "whole_epic": false, "acceptance_requirements": [{"criterion": "Production deployment responds", "required": true, "evidence_category": "deployment", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "deployment-check"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "deployment identity cannot fall back to candidate identity"}, + "pr": {"state": "merged", "merged": true, "head": "head-470", "base": "main", "mergeable": null}, + "diff": {"base": "base-470", "head": "head-470", "patch": "feature", "resulting_tree": "tree-470"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "current_deployed_sha": "deploy-current", "current_deployment_candidate_sha": "head-470", "acceptance_observations": [{"criterion": "Production deployment responds", "evidence_category": "deployment", "stage": "post_merge", "candidate_sha": "head-470", "deployed_sha": "deploy-wrong", "environment": "production", "url": null, "source": "deployment-check", "outcome": "passed"}]} + } + }, + { + "id": "epic-refreshes-after-blocked-merged-delivery", + "target_skill": "implement-epic", + "request": "Continue G-480 after a child delivery merged but required production acceptance remains missing.", + "authority": {"merge": true, "tracker_transition": true, "deploy": false}, + "capabilities": {"implement_ticket": true, "babysit_pr": false}, + "artifacts": { + "ticket": {"tracker": "github", "id": "G-480", "state": "open", "whole_epic": true, "children": ["G-481"], "acceptance_requirements": [{"criterion": "Production journey passes", "required": true, "evidence_category": "manual_browser", "stage": "post_merge", "identity": "deployment", "environment": "production", "url": null, "source": "production browser run"}]}, + "repository": {"repo": "example/project", "tracker": "github", "pr_host": "github", "instructions": "refresh the complete graph after verified delivery even when acceptance blocks completion"}, + "pr": {"state": "merged", "merged": true, "head": "head-481", "base": "main", "mergeable": null}, + "diff": {"base": "base-481", "head": "head-481", "patch": "child feature", "resulting_tree": "tree-481"}, + "checks": {"status": "success", "items": []}, + "reviews": {"initial": "clean", "items": []}, + "threads": {"items": [], "unresolved": 0}, + "worktree": {"path": null, "exclusive_owner": true, "tracked": [], "untracked": []}, + "handoff": {"result_well_formed": true, "verified_merged_delivery": true, "graph_state_changed": true, "acceptance_observations": []} + } } ] diff --git a/skills/implement-ticket/evals/forward_expectations.json b/skills/implement-ticket/evals/forward_expectations.json index 5025550..77dcc93 100644 --- a/skills/implement-ticket/evals/forward_expectations.json +++ b/skills/implement-ticket/evals/forward_expectations.json @@ -285,5 +285,133 @@ "required_actions": ["reject_stale_or_malformed_result", "reread_live_pr"], "target_skill": "implement-ticket", "terminal_state": "blocked" + }, + { + "case_id": "epic-closed-children-missing-manual-browser", + "acceptance_statuses": {"Authenticated workflow succeeds": "missing"}, + "forbidden_actions": ["allow_acceptance_completion", "invoke_merge_when_ready"], + "required_actions": ["build_acceptance_ledger", "verify_child_acceptance_ledgers", "verify_epic_acceptance", "reject_missing_required_acceptance", "keep_tracker_open", "report_delivery_acceptance_separately"], + "target_skill": "implement-epic", + "terminal_state": "blocked" + }, + { + "case_id": "auto-closed-missing-postmerge-deployment", + "acceptance_statuses": {"Tests pass": "pass", "Production deploy is verified": "missing"}, + "forbidden_actions": ["allow_acceptance_completion", "invoke_merge_when_ready"], + "required_actions": ["build_acceptance_ledger", "reject_missing_required_acceptance", "reopen_auto_closed_ticket", "keep_tracker_open", "use_non_closing_reference", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "authenticated-deployed-browser-unavailable", + "acceptance_statuses": {"Authenticated production journey succeeds": "missing"}, + "forbidden_actions": ["allow_acceptance_completion", "caller_verifies_mainline_tracker_cleanup"], + "required_actions": ["build_acceptance_ledger", "reject_missing_required_acceptance", "keep_tracker_open", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "functional-browser-missing-visual-layout", + "acceptance_statuses": {"Controls work": "pass", "Panel is visually aligned": "missing"}, + "forbidden_actions": ["allow_acceptance_completion", "invoke_merge_when_ready"], + "required_actions": ["build_acceptance_ledger", "require_visual_layout_evidence", "reject_missing_required_acceptance", "keep_tracker_open"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "all-acceptance-current", + "acceptance_statuses": {"Contract tests pass": "pass", "Current deployment responds": "pass"}, + "forbidden_actions": ["reject_missing_required_acceptance", "keep_tracker_open", "invoke_merge_when_ready"], + "required_actions": ["build_acceptance_ledger", "allow_acceptance_completion", "verify_live_deployment_candidate_binding", "verify_merge_live", "caller_verifies_mainline_tracker_cleanup"], + "target_skill": "implement-ticket", + "terminal_state": "merged" + }, + { + "case_id": "backend-only-no-ui-gates", + "acceptance_statuses": {"Parser contract suite passes": "pass"}, + "forbidden_actions": ["require_visual_layout_evidence", "reject_missing_required_acceptance"], + "required_actions": ["build_acceptance_ledger", "avoid_irrelevant_ui_gates", "allow_acceptance_completion", "invoke_merge_when_ready"], + "target_skill": "implement-ticket", + "terminal_state": "merged" + }, + { + "case_id": "merge-without-deploy-or-close-authority", + "acceptance_statuses": {"Tests pass": "pass", "Production deploy is verified": "missing"}, + "forbidden_actions": ["allow_acceptance_completion", "reopen_auto_closed_ticket"], + "required_actions": ["build_acceptance_ledger", "reject_missing_required_acceptance", "keep_tracker_open", "use_non_closing_reference", "preserve_acceptance_authority_boundaries", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "reopened-epic-correction-without-journey-revalidation", + "acceptance_statuses": {"Escaped boundary regression passes": "pass"}, + "forbidden_actions": ["allow_acceptance_completion", "refresh_graph_after_merged_only"], + "required_actions": ["build_acceptance_ledger", "verify_child_acceptance_ledgers", "verify_epic_acceptance", "require_escape_journey_revalidation", "keep_tracker_open"], + "target_skill": "implement-epic", + "terminal_state": "blocked" + }, + { + "case_id": "stale-acceptance-evidence", + "acceptance_statuses": {"Contract tests pass": "pass"}, + "forbidden_actions": ["allow_acceptance_completion", "invoke_ready_to_merge"], + "required_actions": ["build_acceptance_ledger", "reject_stale_acceptance_evidence", "reject_missing_required_acceptance", "keep_tracker_open"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "epic-auto-closed-child-incomplete", + "acceptance_statuses": {"Tests pass": "pass", "Production journey passes": "missing"}, + "forbidden_actions": ["allow_acceptance_completion", "refresh_graph_after_merged_only"], + "required_actions": ["build_acceptance_ledger", "verify_child_acceptance_ledgers", "select_auto_closed_incomplete_child", "invoke_implement_ticket_for_recovery", "reopen_auto_closed_ticket", "reject_missing_required_acceptance", "keep_tracker_open", "use_non_closing_reference"], + "target_skill": "implement-epic", + "terminal_state": "blocked" + }, + { + "case_id": "prior-unrelated-deployment-evidence", + "acceptance_statuses": {"Tests pass": "pass", "Production deployment responds": "pass"}, + "forbidden_actions": ["allow_acceptance_completion", "caller_verifies_mainline_tracker_cleanup"], + "required_actions": ["build_acceptance_ledger", "reject_stale_acceptance_evidence", "reject_missing_required_acceptance", "keep_tracker_open", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "missing-acceptance-ledger", + "acceptance_statuses": {}, + "forbidden_actions": ["allow_acceptance_completion", "invoke_ready_to_merge"], + "required_actions": ["build_acceptance_ledger", "reject_missing_required_acceptance", "keep_tracker_open", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "merge-without-tracker-transition-authority", + "acceptance_statuses": {"Contract tests pass": "pass"}, + "forbidden_actions": ["reject_missing_required_acceptance", "invoke_merge_when_ready"], + "required_actions": ["build_acceptance_ledger", "allow_acceptance_completion", "keep_tracker_open", "use_non_closing_reference", "preserve_acceptance_authority_boundaries", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "wrong-source-acceptance-evidence", + "acceptance_statuses": {"Contract tests pass": "fail"}, + "forbidden_actions": ["allow_acceptance_completion", "invoke_ready_to_merge"], + "required_actions": ["build_acceptance_ledger", "reject_missing_required_acceptance", "keep_tracker_open", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "deployment-requirement-rejects-candidate-fallback", + "acceptance_statuses": {"Production deployment responds": "pass"}, + "forbidden_actions": ["allow_acceptance_completion", "caller_verifies_mainline_tracker_cleanup"], + "required_actions": ["build_acceptance_ledger", "reject_stale_acceptance_evidence", "reject_missing_required_acceptance", "keep_tracker_open", "report_delivery_acceptance_separately"], + "target_skill": "implement-ticket", + "terminal_state": "blocked" + }, + { + "case_id": "epic-refreshes-after-blocked-merged-delivery", + "acceptance_statuses": {"Production journey passes": "missing"}, + "forbidden_actions": ["allow_acceptance_completion"], + "required_actions": ["build_acceptance_ledger", "verify_child_acceptance_ledgers", "verify_epic_acceptance", "reject_missing_required_acceptance", "keep_tracker_open", "refresh_graph_after_verified_delivery"], + "target_skill": "implement-epic", + "terminal_state": "blocked" } ] diff --git a/skills/implement-ticket/references/babysit-pr-handoff.md b/skills/implement-ticket/references/babysit-pr-handoff.md index 73b5a65..6c0f5af 100644 --- a/skills/implement-ticket/references/babysit-pr-handoff.md +++ b/skills/implement-ticket/references/babysit-pr-handoff.md @@ -59,7 +59,10 @@ conclusions. Immediately before delegation, reread the live PR and verify all of: - ticket identity, owning tracker, observable goal, acceptance criteria, - non-goals, and allowed fix scope; + required verification items, non-goals, and allowed fix scope; +- the criterion-specific acceptance ledger, including every required pre-merge + entry passing for the supplied head and every post-merge entry identified as + caller-owned follow-up; - repository instructions and named architecture, design, contract, migration, and rollout documents; - GitHub repository and PR identity; @@ -77,8 +80,8 @@ Immediately before delegation, reread the live PR and verify all of: - mutation, push, retry, reply, resolution, draft/ready transition, merge, and branch-deletion authorities; - exclusive mutation ownership or read-only status; and -- correct tracker reference or closing behavior and the transition expected on - merge. +- tracker reference/closing behavior chosen from the acceptance stage, including + any caller-owned post-merge verification and manual transition. The babysitter must reject stale or conflicting repository, PR, head, base, branch, worktree, scope, or ownership identity. Use this documented shape @@ -89,25 +92,28 @@ without adding a larger mandatory schema unless tests demonstrate a need. - `ready PR only` invokes `babysit-pr` with `ready_to_merge`; merge authority is withheld. - `merge after gates` invokes it with `merge_when_ready` and passes merge - authority. + authority. Post-merge acceptance and tracker transition remain caller-owned. - `merge plus manual transition` also uses `merge_when_ready`; the separately - authorized manual transition stays with `implement-ticket` after merge - verification. + authorized verification and manual transition stay with `implement-ticket` + after merge verification. - Normal ticket execution never uses `watch_until_closed`. Pass authority through without expansion. Ready-PR authority permits babysitting to readiness and evidence-based ticket-scoped repair, but not merge. Merge -authority does not imply human-authored communication, tracker mutation, branch -deletion, deployment, production mutation, or parent closure. Preserve stricter -repository communication and thread-resolution rules. +authority does not imply human-authored communication, tracker mutation, +post-merge verification, deployment, branch deletion, production mutation, or +parent closure. Preserve stricter repository communication and thread-resolution +rules. ## Candidate and review integrity -The supplied initial review is reusable only for its exact head and applicable -base. A babysitter-authored or external head change invalidates head-bound -evidence. `babysit-pr` must then run affected and required validation, commit -and push any authorized fix, invoke fresh repository-owned `review-code-change` -with raw current evidence, and rebuild every invalidated remote gate. +The supplied initial review and pre-merge acceptance evidence are reusable only +for their exact head and applicable base. A babysitter-authored or external head +change invalidates affected head-bound entries. `babysit-pr` must then run +affected and required validation, commit and push any authorized fix, invoke +fresh repository-owned `review-code-change` with raw current evidence, and +rebuild invalidated pre-merge and remote gates. It reports post-merge acceptance +as caller-owned work rather than attempting it. Never pass expected findings, implementation transcripts, or prior reviewer conclusions into a fresh review. A missing, malformed, stale, blocked, or @@ -120,10 +126,13 @@ branch/worktree, policy, ownership, validation, review, CI, feedback, gate, and merge evidence before mapping: - `ready_to_merge` maps to `ready_pr` only when the exact PR remains open and - mergeable, every applicable current-candidate non-merge gate passes, merge was - withheld, and ownership is consistent. -- `merged` maps to `merged` only after independent remote merge, mainline - representation, tracker transition, dependency refresh, and cleanup checks. + mergeable, every required pre-merge acceptance entry and applicable + current-candidate non-merge gate passes, merge was withheld, and ownership is + consistent. +- `merged` maps to `merged` only after the caller independently verifies remote + merge, mainline representation, complete post-merge acceptance, tracker + transition, dependency refresh, and cleanup. Until then it is merged delivery, + not accepted ticket completion. - `closed` maps to `blocked` with `PR closed without merge`; preserve local artifacts unless another canonical merged implementation is independently proven complete. diff --git a/skills/implement-ticket/references/cleanup-and-result.md b/skills/implement-ticket/references/cleanup-and-result.md index ee42d8e..cc26a3e 100644 --- a/skills/implement-ticket/references/cleanup-and-result.md +++ b/skills/implement-ticket/references/cleanup-and-result.md @@ -37,13 +37,19 @@ After merge, verify: - the remote base advanced or otherwise contains the complete ordinary or stacked result; - the implemented behavior and tests exist on the base; -- the owning tracker transitioned the ticket as expected; +- every required post-merge acceptance entry passes for the exact deployed or + candidate SHA, environment, evidence category, and source; +- the owning tracker transitioned only after acceptance passed; - for an epic child, affected native dependency relationships were reread after - the transition and newly unblocked work was reported without selection or - mutation; + acceptance and transition, and newly unblocked work was reported without + selection or mutation; - no required check or review state invalidated the claimed result; and - every performed cleanup action passed its preconditions. +A merged candidate with missing required post-merge evidence is delivered but +not accepted: keep or reopen the ticket, return `blocked`, preserve the merged +publication identity, and name the next verification or authority needed. + Do not close a parent epic, verify whole-epic acceptance, or implement newly unblocked work. Report newly ready work only as context. @@ -59,7 +65,11 @@ before return. Otherwise include every applicable field: - ticket identity, tracker, repository, PR host, and base identity; - branch, worktree, candidate head, publication path, and PR or ordered stack identity when created; +- delivery state separately from tracker/acceptance state; - completion policy and the authority actually used; +- the criterion-specific acceptance ledger: criterion, required flag, evidence + category, pre/post-merge stage, candidate/deployed SHA, environment/URL, + source, and `pass`/`fail`/`missing` status; - focused and full validation commands, outcomes, and limitations; - initial `review-code-change` verdict and reviewed candidate identity; - `babysit-pr` policy, terminal state, returned candidate identity, authority @@ -81,34 +91,40 @@ pr: #91 open, mergeable base: main @ 7be0…44c2 branch: scott/lin-482-rate-limits worktree: ../wt-lin-482 head: 4f2c…9a1d completion_policy: ready PR only authority_used: implement + push + PR create +acceptance: API regression test (required, pre-merge, automated-test) pass; + head 4f2c…9a1d; source `just test`; no post-merge items validation: `just test` pass @ head; `just check` full gate pass @ head initial_review: review-code-change clean @ head 4f2c…9a1d vs base 7be0…44c2 babysit_pr: ready_to_merge @ head, verified against live GitHub state; CI 6/6 pass, human review approved, 0 unresolved threads merge: withheld (not authorized) tracker: LIN-482 still In Progress cleanup: none performed (PR open) deferred: one defer-severity naming finding -next_action: caller may merge; merging will move LIN-482 to Done via PR link +next_action: caller may merge; integration will transition LIN-482 only because + all acceptance is pre-merge ``` For `ready_pr`, require a verified `babysit-pr: ready_to_merge` result for the -still-current open and mergeable PR. Every applicable non-merge gate must pass; -the only withheld action is merge. Do not list ordinary pending CI or review as -a remaining gate on a terminal `ready_pr`. +still-current open and mergeable PR and passing required pre-merge acceptance +entries. Every applicable non-merge gate must pass; the only withheld action is +merge. Post-merge entries may remain pending only with non-closing tracker +syntax. Do not list ordinary pending CI or review as a remaining gate. For `ready_prs`, require a verified `carve-changesets: prs_open` result for the -still-current ordered stack. Every PR must be open, correctly based, mergeable, -and at its applicable non-merge gate; whole-chain equivalence and final-only -closing syntax must be verified. Report each exact base ref, base SHA, head ref, -and head SHA so the first PR starts at the candidate base, each later PR starts -at the prior PR head, and the final PR head equals the candidate. The only -withheld actions are merge and propagation. Do not list ordinary pending CI or -review as a remaining gate. +still-current ordered stack and passing required pre-merge acceptance entries. +Every PR must be open, correctly based, mergeable, and at its applicable +non-merge gate; whole-chain equivalence and correct closing/non-closing syntax +must be verified. Report each exact base ref, base SHA, head ref, and head SHA +so the first PR starts at the candidate base, each later PR starts at the prior +PR head, and the final PR head equals the candidate. The only withheld actions +are merge and propagation. For `merged`, require a verified `babysit-pr: merged` or -`carve-changesets: all_merged` result plus the independent mainline, -tracker-transition, dependency-refresh, and cleanup checks above. A `closed` -babysitter result becomes `blocked` with `PR closed without merge` and preserves -local artifacts unless another canonical completion is proven. +`carve-changesets: all_merged` result plus independent mainline, complete +criterion-specific acceptance evidence, tracker transition, dependency refresh, +and cleanup checks. A merged delivery with pending acceptance remains `blocked`, +even if automation closed the tracker. A `closed` babysitter result becomes +`blocked` with `PR closed without merge` and preserves local artifacts unless +another canonical completion is proven. For `requires_epic`, require all of: diff --git a/skills/implement-ticket/references/delegated-execution/CONTRACT.md b/skills/implement-ticket/references/delegated-execution/CONTRACT.md index 0cb5f86..3e17613 100644 --- a/skills/implement-ticket/references/delegated-execution/CONTRACT.md +++ b/skills/implement-ticket/references/delegated-execution/CONTRACT.md @@ -6,7 +6,7 @@ the coordinator is opaque to Agent Scripts, and no Atelier concept appears in the protocol. The capability identifier is -`agent-scripts.implement-ticket/delegated-execution/v1`. +`agent-scripts.implement-ticket/delegated-execution/v2`. ## Contract ownership @@ -34,6 +34,11 @@ binds the run to: - one caller-owned work identifier and revision; - opaque caller-owned approval evidence; - intent, scope, non-goals, constraints, and done definition; +- a structured acceptance contract with one stable criterion identity, required + flag, evidence category, stage, candidate/deployment identity basis, and + applicable environment/URL requirement per item; +- the caller-observed starting deployment's candidate SHA, deployed SHA, + environment, and URL, or explicit null when none applies; - required validation and review expectations; - a finite authority ceiling; - one desired delivery outcome; @@ -66,6 +71,7 @@ sends a `pre_external_mutation` request. The finite action vocabulary is: - `review.reply` - `review.resolve` - `ticket.update` +- `tracker.auto_close.authorize` - `ticket.dependencies.update` - `ticket.followup.create` - `changeset.carve` @@ -122,6 +128,25 @@ may report that verified published candidate whether acknowledgement succeeded or failed. It becomes shared coordinator state only after the caller records it in a later verified transition. +## Deployment observation acknowledgement + +A starting deployment is only an invocation-time baseline; it is not evidence +for a different candidate. After merge and after any authorized deployment, the +delegate sends a `deployment_observed` checkpoint containing the exact candidate +and proposed deployed SHA, environment, and URL. The coordinator rereads the +current deployment from its authoritative source and returns `allow` with +`observed_deployment` only when every field matches and the deployment +represents the exact candidate. A denial or mismatch blocks acceptance. + +The caller persists that observation in the checkpoint ledger and passes it to +`validate_result_checkpoint_state` for combined terminal and durable-ledger-tail +validation. That helper forwards the live observation to +`validate_result_for_invocation`; the live observation overrides the starting +snapshot. Without a caller-verified live observation, only a starting deployment +already bound to the exact candidate may satisfy a deployment-based criterion. +Passing post-merge evidence also requires merged PR state; an open candidate +cannot carry post-merge acceptance. + ## Terminal result The terminal result is always validated before return. It records: @@ -132,17 +157,50 @@ The terminal result is always validated before return. It records: - whether the handoff is transferable; - checkpoint sequence and final continuation token; - validation and review observations; +- a criterion-specific acceptance ledger with required flag, evidence category, + pre/post-merge stage, candidate/deployed SHA, environment/URL, source, and + `pass`/`fail`/`missing` status; +- the caller-verified final tracker state, transition mode, and observation + time; - authority actually used; - unresolved obligations; and - one next action or blocking reason. `ready_pr`, `ready_prs`, and `merged` require published, transferable candidate -state. `ready_pr` requires exactly one PR; `ready_prs` requires a stack. -`requires_epic` requires no implementation state. - -Delivery terminals must report every required validation command as passed at -the exact candidate, satisfy requested independent review, and report zero -unresolved material feedback when requested. +state and at least one acceptance record. `ready_pr` requires exactly one PR; +`ready_prs` requires a stack. `requires_epic` requires no implementation state +and may have an empty ledger. + +Except for `requires_epic`, the terminal ledger must cover the invocation's +acceptance contract one-to-one: it may neither omit a criterion nor invent one, +and its required flag, category, stage, identity basis, exact required source, +and applicable environment/URL must match the caller-owned requirement. A +passing record's source must equal the authored source; a delegate summary or +other nonempty substitute does not satisfy it. Passing evidence must match the +required identity basis: candidate-bound evidence names the exact candidate; +deployment-bound evidence names the exact caller-observed deployment SHA, +environment, and URL whose candidate SHA matches the result candidate. Delivery +terminals must report every required pre-merge entry as passing at the exact +candidate, every required validation command as passed, satisfy requested +independent review, and report zero unresolved material feedback when requested. +Every passing post-merge record requires merged PR state. `merged` additionally +requires every required post-merge entry to pass with its declared candidate or +caller-verified deployment identity. A merged publication with pending +acceptance returns `blocked` while preserving its transferable candidate. + +`merged` is also a tracker-completion claim. The result's `tracker_transition` +is delegate-supplied evidence, not an authoritative observation. The caller must +reread the owning tracker and pass a final observation bound to provider, ticket +ID, state, transition mode, and observation time into +`validate_result_for_invocation` or `validate_result_checkpoint_state`; the +helper requires an exact match. It also requires the caller's consumed-authority +ledger to match `authority_used` exactly. A manual transition requires +`ticket.update`; an automatic transition requires the distinct +`tracker.auto_close.authorize` grant for closing syntax and a subsequent live +closed-state observation. Automatic closing authority does not imply manual +`ticket.update` authority. A merged publication with an open, stale, delegate- +only, mismatched, or unauthorized tracker transition returns `blocked` while +preserving delivery. Published implementation must report the candidate push in `authority_used`. A result containing a pull request must also report the corresponding pull-request @@ -164,8 +222,10 @@ never describe a local-only SHA as transferable. The caller must validate every terminal result against the durable checkpoint ledger tail, not merely the invocation's starting position. The bundled `validate_result_checkpoint_state` helper requires the terminal sequence and -continuation token to equal that caller-supplied tail. A stale terminal -checkpoint blocks handoff. +continuation token to equal that caller-supplied tail. For `merged`, it also +requires the caller-observed final tracker record and consumed-authority ledger. +The CLI accepts those JSON values through `--observed-tracker` and +`--consumed-authority`. A stale terminal checkpoint blocks handoff. A material ticket-observation change always causes the caller to deny the current invocation. Eligibility may be reevaluated only before starting a fresh @@ -176,8 +236,10 @@ truthfully. ## Compatibility and failure Standalone invocations remain unchanged and may return the documented human -handoff. Delegated execution applies only when the caller supplies a valid v1 -invocation. +handoff. Delegated execution applies only when the caller supplies a valid v2 +invocation. Version 2 adds the required acceptance-evidence ledger to terminal +results; v1 manifests, invocations, checkpoints, and results are rejected rather +than silently interpreted under the stronger closeout contract. There is no daemon, callback server, or background lease. The checkpoint command is synchronous and caller-owned. If the caller disappears, execution fails diff --git a/skills/implement-ticket/references/delegated-execution/capability.json b/skills/implement-ticket/references/delegated-execution/capability.json index 9516a16..63fd43d 100644 --- a/skills/implement-ticket/references/delegated-execution/capability.json +++ b/skills/implement-ticket/references/delegated-execution/capability.json @@ -1,6 +1,6 @@ { - "schema": "agent-scripts.implement-ticket/capability-manifest/v1", - "id": "agent-scripts.implement-ticket/delegated-execution/v1", + "schema": "agent-scripts.implement-ticket/capability-manifest/v2", + "id": "agent-scripts.implement-ticket/delegated-execution/v2", "contract": "CONTRACT.md", "invocation_schema": "invocation.schema.json", "checkpoint_request_schema": "checkpoint-request.schema.json", @@ -16,6 +16,7 @@ ], "checkpoint_phases": [ "pre_external_mutation", - "candidate_published" + "candidate_published", + "deployment_observed" ] } diff --git a/skills/implement-ticket/references/delegated-execution/capability.schema.json b/skills/implement-ticket/references/delegated-execution/capability.schema.json index a7b3d4a..95c70de 100644 --- a/skills/implement-ticket/references/delegated-execution/capability.schema.json +++ b/skills/implement-ticket/references/delegated-execution/capability.schema.json @@ -17,10 +17,10 @@ ], "properties": { "schema": { - "const": "agent-scripts.implement-ticket/capability-manifest/v1" + "const": "agent-scripts.implement-ticket/capability-manifest/v2" }, "id": { - "const": "agent-scripts.implement-ticket/delegated-execution/v1" + "const": "agent-scripts.implement-ticket/delegated-execution/v2" }, "contract": { "const": "CONTRACT.md" @@ -54,7 +54,8 @@ "type": "array", "const": [ "pre_external_mutation", - "candidate_published" + "candidate_published", + "deployment_observed" ] } } diff --git a/skills/implement-ticket/references/delegated-execution/checkpoint-request.schema.json b/skills/implement-ticket/references/delegated-execution/checkpoint-request.schema.json index bbc8a3a..a07971e 100644 --- a/skills/implement-ticket/references/delegated-execution/checkpoint-request.schema.json +++ b/skills/implement-ticket/references/delegated-execution/checkpoint-request.schema.json @@ -13,14 +13,15 @@ "action", "ticket_observation", "candidate", + "deployment", "proposed_effect" ], "properties": { "schema": { - "const": "agent-scripts.implement-ticket/checkpoint-request/v1" + "const": "agent-scripts.implement-ticket/checkpoint-request/v2" }, "capability": { - "const": "agent-scripts.implement-ticket/delegated-execution/v1" + "const": "agent-scripts.implement-ticket/delegated-execution/v2" }, "invocation_id": { "type": "string", @@ -35,7 +36,7 @@ "minimum": 1 }, "phase": { - "enum": ["pre_external_mutation", "candidate_published"] + "enum": ["pre_external_mutation", "candidate_published", "deployment_observed"] }, "action": { "enum": [ @@ -46,6 +47,7 @@ "review.reply", "review.resolve", "ticket.update", + "tracker.auto_close.authorize", "ticket.dependencies.update", "ticket.followup.create", "changeset.carve", @@ -93,6 +95,28 @@ } } }, + "deployment": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["candidate_sha", "deployed_sha", "environment", "url"], + "properties": { + "candidate_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "deployed_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "environment": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": ["string", "null"] + } + } + }, "proposed_effect": { "type": "string", "minLength": 1 diff --git a/skills/implement-ticket/references/delegated-execution/checkpoint-response.schema.json b/skills/implement-ticket/references/delegated-execution/checkpoint-response.schema.json index 221f2c2..2977541 100644 --- a/skills/implement-ticket/references/delegated-execution/checkpoint-response.schema.json +++ b/skills/implement-ticket/references/delegated-execution/checkpoint-response.schema.json @@ -11,11 +11,12 @@ "continuation_token", "decision", "reason", - "acknowledged_candidate_sha" + "acknowledged_candidate_sha", + "observed_deployment" ], "properties": { "schema": { - "const": "agent-scripts.implement-ticket/checkpoint-response/v1" + "const": "agent-scripts.implement-ticket/checkpoint-response/v2" }, "invocation_id": { "type": "string", @@ -42,6 +43,28 @@ "acknowledged_candidate_sha": { "type": ["string", "null"], "pattern": "^[0-9a-f]{40,64}$" + }, + "observed_deployment": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["candidate_sha", "deployed_sha", "environment", "url"], + "properties": { + "candidate_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "deployed_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "environment": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": ["string", "null"] + } + } } } } diff --git a/skills/implement-ticket/references/delegated-execution/invocation.schema.json b/skills/implement-ticket/references/delegated-execution/invocation.schema.json index 03530d7..559a53f 100644 --- a/skills/implement-ticket/references/delegated-execution/invocation.schema.json +++ b/skills/implement-ticket/references/delegated-execution/invocation.schema.json @@ -15,14 +15,16 @@ "authority", "desired_outcome", "accepted_terminal_states", + "acceptance_requirements", + "starting_deployment", "checkpoint" ], "properties": { "schema": { - "const": "agent-scripts.implement-ticket/delegated-invocation/v1" + "const": "agent-scripts.implement-ticket/delegated-invocation/v2" }, "capability": { - "const": "agent-scripts.implement-ticket/delegated-execution/v1" + "const": "agent-scripts.implement-ticket/delegated-execution/v2" }, "invocation_id": { "type": "string", @@ -88,6 +90,35 @@ ] } }, + "acceptance_requirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/acceptance_requirement" + } + }, + "starting_deployment": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["candidate_sha", "deployed_sha", "environment", "url"], + "properties": { + "candidate_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "deployed_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40,64}$" + }, + "environment": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": ["string", "null"] + } + } + }, "checkpoint": { "type": "object", "additionalProperties": false, @@ -113,6 +144,49 @@ } }, "$defs": { + "acceptance_requirement": { + "type": "object", + "additionalProperties": false, + "required": [ + "criterion", + "required", + "evidence_category", + "stage", + "identity", + "environment", + "url", + "source" + ], + "properties": { + "criterion": { + "type": "string", + "minLength": 1 + }, + "required": { + "type": "boolean" + }, + "evidence_category": { + "type": "string", + "minLength": 1 + }, + "stage": { + "enum": ["pre_merge", "post_merge"] + }, + "identity": { + "enum": ["candidate", "deployment"] + }, + "environment": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "source": { + "type": "string", + "minLength": 1 + } + } + }, "action": { "enum": [ "repository.candidate.create", @@ -122,6 +196,7 @@ "review.reply", "review.resolve", "ticket.update", + "tracker.auto_close.authorize", "ticket.dependencies.update", "ticket.followup.create", "changeset.carve", diff --git a/skills/implement-ticket/references/delegated-execution/result.schema.json b/skills/implement-ticket/references/delegated-execution/result.schema.json index 7507431..440687f 100644 --- a/skills/implement-ticket/references/delegated-execution/result.schema.json +++ b/skills/implement-ticket/references/delegated-execution/result.schema.json @@ -10,6 +10,7 @@ "terminal_state", "ticket", "repository", + "tracker_transition", "implementation_state", "candidate", "handoff", @@ -18,16 +19,17 @@ "reviews", "feedback", "authority_used", + "acceptance_evidence", "unresolved_obligations", "blocking_reason", "next_action" ], "properties": { "schema": { - "const": "agent-scripts.implement-ticket/delegated-result/v1" + "const": "agent-scripts.implement-ticket/delegated-result/v2" }, "capability": { - "const": "agent-scripts.implement-ticket/delegated-execution/v1" + "const": "agent-scripts.implement-ticket/delegated-execution/v2" }, "invocation_id": { "type": "string", @@ -83,6 +85,30 @@ } } }, + "tracker_transition": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "ticket_id", "mode", "state", "observed_at"], + "properties": { + "provider": { + "enum": ["github", "linear"] + }, + "ticket_id": { + "type": "string", + "minLength": 1 + }, + "mode": { + "enum": ["none", "manual", "automatic"] + }, + "state": { + "enum": ["open", "closed"] + }, + "observed_at": { + "type": "string", + "minLength": 1 + } + } + }, "implementation_state": { "enum": ["none", "local", "published"] }, @@ -241,6 +267,12 @@ "$ref": "#/$defs/action" } }, + "acceptance_evidence": { + "type": "array", + "items": { + "$ref": "#/$defs/acceptance_record" + } + }, "unresolved_obligations": { "type": "array", "items": { @@ -257,6 +289,58 @@ } }, "$defs": { + "acceptance_record": { + "type": "object", + "additionalProperties": false, + "required": [ + "criterion", + "required", + "evidence_category", + "stage", + "candidate_sha", + "deployed_sha", + "environment", + "url", + "source", + "status" + ], + "properties": { + "criterion": { + "type": "string", + "minLength": 1 + }, + "required": { + "type": "boolean" + }, + "evidence_category": { + "type": "string", + "minLength": 1 + }, + "stage": { + "enum": ["pre_merge", "post_merge"] + }, + "candidate_sha": { + "type": ["string", "null"], + "pattern": "^[0-9a-f]{40,64}$" + }, + "deployed_sha": { + "type": ["string", "null"], + "pattern": "^[0-9a-f]{40,64}$" + }, + "environment": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "source": { + "type": ["string", "null"] + }, + "status": { + "enum": ["pass", "fail", "missing"] + } + } + }, "action": { "enum": [ "repository.candidate.create", @@ -266,6 +350,7 @@ "review.reply", "review.resolve", "ticket.update", + "tracker.auto_close.authorize", "ticket.dependencies.update", "ticket.followup.create", "changeset.carve", diff --git a/skills/implement-ticket/references/delegated-execution/validate.py b/skills/implement-ticket/references/delegated-execution/validate.py index 66767cd..ff5ddab 100644 --- a/skills/implement-ticket/references/delegated-execution/validate.py +++ b/skills/implement-ticket/references/delegated-execution/validate.py @@ -28,6 +28,7 @@ "changeset.carve", "pull_request.merge", "repository.branch.delete", + "deployment.execute", } PUBLICATION_ACTIONS = CANDIDATE_REQUIRED_ACTIONS | { "deployment.execute", @@ -132,6 +133,16 @@ def _validate_invocation(value: dict[str, Any]) -> list[str]: errors.append("$.desired_outcome: must appear in accepted_terminal_states") if "blocked" not in accepted: errors.append("$.accepted_terminal_states: must include blocked") + requirements = value.get("acceptance_requirements", []) + criteria = [entry["criterion"] for entry in requirements] + duplicate_criteria = sorted( + {criterion for criterion in criteria if criteria.count(criterion) > 1} + ) + if duplicate_criteria: + errors.append( + "$.acceptance_requirements: duplicate criteria " + + ", ".join(duplicate_criteria) + ) return errors @@ -139,6 +150,7 @@ def _validate_checkpoint_request(value: dict[str, Any]) -> list[str]: errors: list[str] = [] phase = value.get("phase") candidate = value.get("candidate") + deployment = value.get("deployment") if value.get("action") in CANDIDATE_REQUIRED_ACTIONS and candidate is None: errors.append("$.candidate: action requires exact candidate") if phase == "candidate_published": @@ -148,6 +160,17 @@ def _validate_checkpoint_request(value: dict[str, Any]) -> list[str]: errors.append( "$.action: candidate_published requires repository.candidate.push" ) + if phase == "deployment_observed": + if candidate is None or deployment is None: + errors.append( + "$.deployment: deployment_observed requires candidate and deployment" + ) + elif deployment["candidate_sha"] != candidate["head_sha"]: + errors.append("$.deployment.candidate_sha: does not match candidate") + if value.get("action") != "deployment.execute": + errors.append("$.action: deployment_observed requires deployment.execute") + elif deployment is not None: + errors.append("$.deployment: only deployment_observed permits deployment") return errors @@ -165,6 +188,8 @@ def _validate_checkpoint_response(value: dict[str, Any]) -> list[str]: errors.append("$.continuation_token: allow must rotate the token") if decision == "deny" and continuation != prior: errors.append("$.continuation_token: deny must preserve the prior token") + if decision == "deny" and value.get("observed_deployment") is not None: + errors.append("$.observed_deployment: deny requires null") return errors @@ -178,6 +203,8 @@ def _validate_result(value: dict[str, Any]) -> list[str]: reason = handoff.get("reason") blocking_reason = value.get("blocking_reason") authority_used = set(value.get("authority_used", [])) + acceptance = value.get("acceptance_evidence", []) + tracker_transition = value.get("tracker_transition", {}) for collection in ("validation", "reviews"): names = [observation["name"] for observation in value[collection]] @@ -187,6 +214,75 @@ def _validate_result(value: dict[str, Any]) -> list[str]: f"$.{collection}: duplicate observation names " + ", ".join(duplicates) ) + criteria = [entry["criterion"] for entry in acceptance] + duplicate_criteria = sorted( + {criterion for criterion in criteria if criteria.count(criterion) > 1} + ) + if duplicate_criteria: + errors.append( + "$.acceptance_evidence: duplicate criteria " + ", ".join(duplicate_criteria) + ) + + if terminal in {"ready_pr", "ready_prs", "merged"} and not acceptance: + errors.append("$.acceptance_evidence: delivery terminal requires evidence") + if terminal in {"ready_pr", "ready_prs", "merged"}: + pre_merge_incomplete = [ + entry["criterion"] + for entry in acceptance + if entry["required"] + and entry["stage"] == "pre_merge" + and entry["status"] != "pass" + ] + if pre_merge_incomplete: + errors.append( + "$.acceptance_evidence: required pre-merge evidence incomplete for " + + ", ".join(pre_merge_incomplete) + ) + if terminal == "merged": + incomplete = [ + entry["criterion"] + for entry in acceptance + if entry["required"] and entry["status"] != "pass" + ] + if incomplete: + errors.append( + "$.acceptance_evidence: merged requires complete evidence for " + + ", ".join(incomplete) + ) + ticket = value.get("ticket", {}) + if tracker_transition.get("provider") != ticket.get("provider"): + errors.append("$.tracker_transition.provider: does not match ticket") + if tracker_transition.get("ticket_id") != ticket.get("id"): + errors.append("$.tracker_transition.ticket_id: does not match ticket") + if tracker_transition.get("state") != "closed": + errors.append("$.tracker_transition.state: merged requires closed tracker") + transition_mode = tracker_transition.get("mode") + if transition_mode not in {"manual", "automatic"}: + errors.append( + "$.tracker_transition.mode: merged requires verified manual or automatic transition" + ) + elif transition_mode == "manual" and "ticket.update" not in authority_used: + errors.append( + "$.authority_used: manual tracker transition requires ticket.update" + ) + elif ( + transition_mode == "automatic" + and "tracker.auto_close.authorize" not in authority_used + ): + errors.append( + "$.authority_used: automatic tracker transition requires " + "tracker.auto_close.authorize" + ) + missing_source = [ + entry["criterion"] + for entry in acceptance + if entry["status"] == "pass" and not entry["source"] + ] + if missing_source: + errors.append( + "$.acceptance_evidence: passing evidence requires source for " + + ", ".join(missing_source) + ) if implementation == "published": if candidate is None or transferable is not True: errors.append( @@ -229,6 +325,30 @@ def _validate_result(value: dict[str, Any]) -> list[str]: "$.implementation_state: delivery terminal requires published candidate" ) if candidate is not None: + stale_acceptance = [ + entry["criterion"] + for entry in acceptance + if entry["status"] == "pass" + and entry["candidate_sha"] is not None + and entry["candidate_sha"] != candidate["head_sha"] + ] + if stale_acceptance: + errors.append( + "$.acceptance_evidence: candidate mismatch for " + + ", ".join(stale_acceptance) + ) + missing_identity = [ + entry["criterion"] + for entry in acceptance + if entry["status"] == "pass" + and not entry["candidate_sha"] + and not entry["deployed_sha"] + ] + if missing_identity: + errors.append( + "$.acceptance_evidence: passing evidence requires candidate or deployed SHA for " + + ", ".join(missing_identity) + ) publication = candidate["publication"] kind = publication["kind"] pull_requests = publication["pull_requests"] @@ -352,6 +472,7 @@ def validate_checkpoint_exchange( if request[request_field] != response[response_field]: errors.append(f"$.{response_field}: does not match request {request_field}") acknowledged = response["acknowledged_candidate_sha"] + observed_deployment = response["observed_deployment"] if response["decision"] == "deny": if acknowledged is not None: errors.append("$.acknowledged_candidate_sha: deny requires null") @@ -362,7 +483,16 @@ def validate_checkpoint_exchange( ) elif acknowledged is not None: errors.append( - "$.acknowledged_candidate_sha: pre-mutation response requires null" + "$.acknowledged_candidate_sha: non-publication response requires null" + ) + if response["decision"] == "allow" and request["phase"] == "deployment_observed": + if observed_deployment != request["deployment"]: + errors.append( + "$.observed_deployment: does not match caller-verified deployment" + ) + elif observed_deployment is not None: + errors.append( + "$.observed_deployment: only deployment_observed allow may include it" ) return errors @@ -385,10 +515,42 @@ def validate_checkpoint_progress( def validate_result_for_invocation( invocation: dict[str, Any], result: dict[str, Any], + observed_deployment: dict[str, Any] | None = None, + observed_tracker: dict[str, Any] | None = None, + consumed_authority: list[str] | None = None, ) -> list[str]: - """Validate a terminal result against its delegated invocation.""" + """Validate a terminal result against its invocation and caller observations.""" errors = validate("invocation", invocation) errors.extend(validate("result", result)) + if observed_deployment is not None: + response_schema = json.loads(SCHEMAS["checkpoint-response"].read_text()) + errors.extend( + validate_schema( + observed_deployment, + response_schema["properties"]["observed_deployment"], + response_schema, + "$.observed_deployment", + ) + ) + result_schema = json.loads(SCHEMAS["result"].read_text()) + if observed_tracker is not None: + errors.extend( + validate_schema( + observed_tracker, + result_schema["properties"]["tracker_transition"], + result_schema, + "$.observed_tracker", + ) + ) + if consumed_authority is not None: + errors.extend( + validate_schema( + consumed_authority, + result_schema["properties"]["authority_used"], + result_schema, + "$.consumed_authority", + ) + ) if errors: return errors if result["invocation_id"] != invocation["invocation_id"]: @@ -397,6 +559,19 @@ def validate_result_for_invocation( errors.append("$.terminal_state: caller does not accept this state") if result["ticket"] != invocation["ticket"]: errors.append("$.ticket: does not match invocation") + if result["terminal_state"] == "merged": + if observed_tracker is None: + errors.append("$.observed_tracker: merged requires caller observation") + elif result["tracker_transition"] != observed_tracker: + errors.append( + "$.tracker_transition: does not match caller-observed tracker state" + ) + if consumed_authority is None: + errors.append( + "$.consumed_authority: merged requires caller authority ledger" + ) + elif set(result["authority_used"]) != set(consumed_authority): + errors.append("$.authority_used: does not match caller authority ledger") expected_repository = { "identity": invocation["repository"]["identity"], "base_ref": invocation["repository"]["base_ref"], @@ -422,6 +597,94 @@ def validate_result_for_invocation( errors.append( "$.candidate.publication: first PR base_ref does not match invocation" ) + if result["terminal_state"] != "requires_epic": + requirements = { + item["criterion"]: item for item in invocation["acceptance_requirements"] + } + evidence = {item["criterion"]: item for item in result["acceptance_evidence"]} + missing_criteria = sorted(set(requirements) - set(evidence)) + if missing_criteria: + errors.append( + "$.acceptance_evidence: missing invocation criteria " + + ", ".join(missing_criteria) + ) + unexpected_criteria = sorted(set(evidence) - set(requirements)) + if unexpected_criteria: + errors.append( + "$.acceptance_evidence: criteria not present in invocation " + + ", ".join(unexpected_criteria) + ) + deployment = observed_deployment or invocation["starting_deployment"] + for criterion in sorted(set(requirements) & set(evidence)): + requirement = requirements[criterion] + entry = evidence[criterion] + for field in ("required", "evidence_category", "stage"): + if entry[field] != requirement[field]: + errors.append( + f"$.acceptance_evidence: {field} mismatch for {criterion}" + ) + for field in ("environment", "url"): + expected = requirement[field] + if expected is not None and entry[field] != expected: + errors.append( + f"$.acceptance_evidence: {field} mismatch for {criterion}" + ) + if entry["status"] == "pass" and entry["source"] != requirement["source"]: + errors.append(f"$.acceptance_evidence: source mismatch for {criterion}") + if entry["status"] != "pass": + continue + if entry["stage"] == "post_merge": + pull_requests = ( + candidate["publication"]["pull_requests"] + if candidate is not None + else [] + ) + if not pull_requests or any( + pull_request["state"] != "merged" for pull_request in pull_requests + ): + errors.append( + "$.acceptance_evidence: post-merge evidence requires merged " + f"candidate for {criterion}" + ) + if requirement["identity"] == "candidate": + if candidate is None or entry["candidate_sha"] != candidate["head_sha"]: + errors.append( + "$.acceptance_evidence: required candidate identity mismatch " + f"for {criterion}" + ) + continue + if entry["deployed_sha"] is None: + errors.append( + "$.acceptance_evidence: required deployment identity missing " + f"for {criterion}" + ) + continue + if deployment is None: + errors.append( + "$.acceptance_evidence: passing deployment evidence lacks " + f"caller-observed deployment for {criterion}" + ) + continue + if ( + candidate is None + or deployment["candidate_sha"] != candidate["head_sha"] + ): + errors.append( + "$.acceptance_evidence: deployment candidate mismatch " + f"for {criterion}" + ) + continue + deployment_fields = { + "deployed_sha": "deployed_sha", + "environment": "environment", + "url": "url", + } + for evidence_field, deployment_field in deployment_fields.items(): + if entry[evidence_field] != deployment[deployment_field]: + errors.append( + "$.acceptance_evidence: current deployment " + f"{evidence_field} mismatch for {criterion}" + ) allowed = set(invocation["authority"]["allow"]) excess = sorted(set(result["authority_used"]) - allowed) if excess: @@ -470,9 +733,18 @@ def validate_result_checkpoint_state( result: dict[str, Any], last_sequence: int, current_token: str, + observed_deployment: dict[str, Any] | None = None, + observed_tracker: dict[str, Any] | None = None, + consumed_authority: list[str] | None = None, ) -> list[str]: - """Validate a terminal result against its invocation and caller ledger tail.""" - errors = validate_result_for_invocation(invocation, result) + """Validate a terminal result against live observations and the ledger tail.""" + errors = validate_result_for_invocation( + invocation, + result, + observed_deployment, + observed_tracker, + consumed_authority, + ) if errors: return errors if result["checkpoint"]["last_sequence"] != last_sequence: @@ -500,6 +772,16 @@ def main() -> int: type=Path, help="Validate a checkpoint response against this request", ) + parser.add_argument( + "--observed-tracker", + type=Path, + help="Caller-observed final tracker transition for a merged result", + ) + parser.add_argument( + "--consumed-authority", + type=Path, + help="Caller authorization ledger for a merged result", + ) args = parser.parse_args() try: @@ -514,7 +796,22 @@ def main() -> int: if args.kind != "result": parser.error("--invocation requires kind=result") invocation = json.loads(args.invocation.read_text()) - errors = validate_result_for_invocation(invocation, value) + observed_tracker = ( + json.loads(args.observed_tracker.read_text()) + if args.observed_tracker + else None + ) + consumed_authority = ( + json.loads(args.consumed_authority.read_text()) + if args.consumed_authority + else None + ) + errors = validate_result_for_invocation( + invocation, + value, + observed_tracker=observed_tracker, + consumed_authority=consumed_authority, + ) elif args.request: if args.kind != "checkpoint-response": parser.error("--request requires kind=checkpoint-response") diff --git a/skills/implement-ticket/references/github.md b/skills/implement-ticket/references/github.md index 1dd7ebb..5aaa3d5 100644 --- a/skills/implement-ticket/references/github.md +++ b/skills/implement-ticket/references/github.md @@ -38,10 +38,11 @@ When duplicate branches or PRs exist, compare actual patches or resulting trees and retain one canonical implementation path. For an open canonical path owned by another worker, return `blocked` with its identity unless ownership is explicitly transferred; never claim its candidate as this run's `ready_pr` or -`ready_prs`. When a merged PR is verified on the base and the ticket is already -complete, return `merged` with that evidence without creating new state. Return -`blocked` rather than creating a competing PR when canonical ownership is -unresolved. +`ready_prs`. A merged PR and `CLOSED` issue prove delivery/administrative state, +not acceptance. Return `merged` without new implementation only when the +criterion-specific ledger passes for the current candidate/deployment and the +tracker transition is correct. Return `blocked` rather than creating a competing +PR when canonical ownership is unresolved. ## PR-host preflight and contract @@ -51,19 +52,22 @@ unresolved. - Inspect open and merged PRs that reference the owning tracker ticket. - Use one tracker ticket per candidate. Publish that candidate as exactly one ordinary PR or one ordered carved stack. -- When GitHub owns ticket state, use the repository's closing syntax, normally - `Fixes #`. -- Determine whether that syntax will automatically close or transition the - ticket on merge and disclose the consequence in the resolved completion policy - before publishing or merging. Use a non-closing reference when automatic - closure would conflict with that policy. -- For a carved stack, put closing syntax only on the final changeset PR; every - intermediate PR uses a non-closing reference. Verify transition only after the - full stack is merged. +- When GitHub owns ticket state and every required acceptance item can pass + before merge, use the repository's closing syntax, normally `Fixes #`. +- When any required item needs merged code, deployment, authentication, or + another post-merge environment, use `Refs #`, `Supports #`, or + the repository's established non-closing equivalent. Close manually only after + the current acceptance ledger passes and close authority exists. +- If closing automation transitions the issue before required acceptance passes, + treat it as incomplete. Reopen it when authorized; otherwise report missing + reopen authority as a blocker. +- For a carved stack, closing syntax is permitted only on the final changeset PR + and only under the same pre-merge-acceptance rule. Every intermediate PR uses + a non-closing reference. - When another tracker owns state, use its required reference and avoid GitHub closing syntax unless a real GitHub issue is also intentionally in scope. - Describe the branch as a whole, preserve material non-goals, and report actual - validation. + validation plus pre-merge and post-merge acceptance-ledger state. - Confirm the PR base and head match the ticket worktree. Use file-based commit and PR messages when shell interpolation could alter @@ -88,10 +92,10 @@ state. Do not also poll, mutate, reply, resolve, or merge from this caller after ownership transfer. After a babysitter `merged` result or a carve `all_merged` result, independently -verify PR or stack state, complete candidate representation on the base, and the -GitHub issue transition before cleanup. When the ticket is an epic child, reread -its affected native `blocking` and sibling `blockedBy` relationships and report -newly unblocked work without selecting or mutating it. If local worktree -ownership prevents the CLI from switching to the base, use a read-only remote -verification path and perform local cleanup separately. Never close a parent -issue from this skill. +verify PR or stack state and complete candidate representation on the base, then +run every required post-merge acceptance item with separately granted authority. +Transition the GitHub issue only after the ledger passes. When the ticket is an +epic child, reread affected native relationships only after acceptance and the +transition pass. If local worktree ownership prevents switching to the base, use +a read-only remote verification path and perform local cleanup separately. Never +close a parent issue from this skill. diff --git a/skills/implement-ticket/references/linear.md b/skills/implement-ticket/references/linear.md index 6f4353f..27c2cbc 100644 --- a/skills/implement-ticket/references/linear.md +++ b/skills/implement-ticket/references/linear.md @@ -34,21 +34,24 @@ independently. ## PR linkage and transition -- Treat the live Linear issue body as the scope contract. +- Treat the live Linear issue body as the scope and acceptance contract. - Preserve exact product rules such as timing, timezone, idempotency, skip, - unavailable, migration, compatibility, and rollout behavior. + unavailable, migration, compatibility, rollout, and verification behavior. - Use the repository's required Linear reference in branch, commit, and PR metadata. - Keep one ticket per candidate. Publish it as one ordinary PR or one ordered - carved stack, with the Linear reference on every PR and any transition-causing - closing integration only on the final PR. -- Update Linear status only when that workflow state was actually reached and - the completion policy authorizes a manual transition. -- After the ordinary merge or a verified `all_merged` stack result, verify the - complete result on the base and the expected Linear transition. + carved stack. Allow an integration-driven completion transition only when all + required acceptance can pass before merge; otherwise use a non-transitioning + reference and close manually after post-merge evidence passes. +- Update or reopen Linear status only when that workflow state was actually + reached, the criterion-specific ledger passes, and the completion policy + authorizes the transition. A completed state is not acceptance evidence. +- After the ordinary merge or verified `all_merged` result, verify the complete + result on the base and run every required post-merge acceptance item with + separately granted environment and deployment authority. - When the ticket is an epic child, reread affected native dependency - relationships after transition and report newly unblocked work without - selecting or mutating it. + relationships only after acceptance and the transition pass, then report newly + unblocked work without selecting or mutating it. Do not close or verify the parent epic. Report newly unblocked downstream work without selecting it. diff --git a/skills/implement-ticket/references/review-and-merge-gates.md b/skills/implement-ticket/references/review-and-merge-gates.md index f362666..7777c10 100644 --- a/skills/implement-ticket/references/review-and-merge-gates.md +++ b/skills/implement-ticket/references/review-and-merge-gates.md @@ -22,7 +22,10 @@ Before delegation, capture HEAD, comparison base, commit history, and tracked, staged, unstaged, untracked, and ignored state. Invoke `review-code-change` in a fresh or minimally inherited read-only context with: -- the live ticket and acceptance criteria; +- the live ticket, every acceptance criterion, and every required verification + item classified as pre-merge or post-merge; +- the criterion-specific acceptance ledger with required evidence category, + current SHA/environment, source, and status; - every named architecture, design, contract, migration, and rollout document; - repository instructions and representative nearby code and tests; - the exact captured head and comparison-base SHAs plus the complete @@ -54,6 +57,8 @@ Before invoking either delegate: - verify the initial review is clean for the exact live head and applicable base; +- verify every required pre-merge acceptance entry passes and choose closing or + non-closing tracker syntax from whether post-merge entries exist; - evaluate the exact candidate against the live `carve-changesets` guardrails without duplicating their thresholds; - verify the selected single-PR or stack identity, effective diff, resulting @@ -62,7 +67,8 @@ Before invoking either delegate: - assemble every field required by the applicable [babysit-pr](babysit-pr-handoff.md) or [carve-changesets](carve-changesets-handoff.md) handoff contract; -- map the completion policy without broadening authority; and +- map the completion policy without broadening merge, deployment, verification, + or tracker-transition authority; and - establish one exclusive mutating owner. Treat a missing dependency, malformed result, `blocked` verdict, reviewer @@ -75,14 +81,17 @@ initial review is clean. After the selected delegate returns, reread live GitHub state and apply the applicable [babysit-pr](babysit-pr-handoff.md) or [carve-changesets](carve-changesets-handoff.md) result mapping. A `ready_pr` -requires a validated current `ready_to_merge` result. A `ready_prs` requires a -validated current `prs_open` result. A `merged` result requires independent -remote merge or `all_merged`, mainline, tracker-transition, dependency-refresh, -and cleanup verification by `implement-ticket`. - -If the live head, base, PR state, ownership, or gate evidence differs from the -result, reconcile the live candidate or fail closed. Never carry stale evidence -through a head change or accept a closed-unmerged PR as complete. +requires a validated current `ready_to_merge` result plus passing required +pre-merge acceptance evidence. A `ready_prs` requires a validated current +`prs_open` result plus the same evidence. A `merged` result requires independent +remote merge or `all_merged`, mainline, complete current acceptance evidence, +tracker transition, dependency refresh, and cleanup verification by +`implement-ticket`. + +If the live head, base, PR state, ownership, acceptance ledger, or gate evidence +differs from the result, reconcile the live candidate or fail closed. Never +carry stale evidence through a head/deployment change or accept a closed issue, +merged PR, or closed-unmerged PR as acceptance proof. ## Findings that must not expand the ticket diff --git a/skills/implement-ticket/scripts/evals/claude_executor.py b/skills/implement-ticket/scripts/evals/claude_executor.py index 294ff78..4e95934 100644 --- a/skills/implement-ticket/scripts/evals/claude_executor.py +++ b/skills/implement-ticket/scripts/evals/claude_executor.py @@ -5,7 +5,8 @@ `run_forward.py`), asks a fresh `claude -p` process to act as the target skill's runtime, and prints one JSON result to stdout: - {"target_skill": ..., "terminal_state": ..., "actions": [...]} + {"target_skill": ..., "terminal_state": ..., "actions": [...], + "acceptance_ledger": [...]} The evaluated model receives the skill prompt, the request, and raw scenario artifacts, plus the closed action vocabulary below so its choices are gradable @@ -38,6 +39,9 @@ # must decide which obligations apply, not invent matching strings. ACTION_VOCABULARY = ( "adopt_verified_canonical_pr", + "allow_acceptance_completion", + "avoid_irrelevant_ui_gates", + "build_acceptance_ledger", "caller_verifies_mainline_tracker_cleanup", "consume_ticket_states_unchanged", "deduplicate_prior_actions", @@ -54,6 +58,7 @@ "invoke_merge_when_ready", "invoke_ready_to_merge", "invoke_carve_changesets", + "invoke_implement_ticket_for_recovery", "make_no_code_mutation", "name_missing_babysit_pr", "name_missing_carve_changesets", @@ -61,16 +66,23 @@ "place_closing_syntax_final_pr_only", "preserve_artifacts", "preserve_feedback_gate", + "preserve_acceptance_authority_boundaries", "preserve_partial_stack", "preserve_tracker_pr_host_separation", "rebuild_remote_gates", "record_guardrail_evidence", "refresh_graph_after_merged_only", + "refresh_graph_after_verified_delivery", "reject_concurrent_mutation", + "reject_missing_required_acceptance", "reject_stale_connector_verdict", "reject_stale_or_malformed_result", + "reject_stale_acceptance_evidence", "report_closed_without_merge", + "report_delivery_acceptance_separately", "report_mid_stack_redesign", + "report_missing_reopen_authority", + "reopen_auto_closed_ticket", "reread_live_pr", "retain_only_proven_unaffected_evidence", "retry_diagnosed_run_only", @@ -78,15 +90,24 @@ "revalidate_commit_push", "route_before_ticket_dependencies", "route_to_tracker_split", + "select_auto_closed_incomplete_child", + "skill_contract_incomplete", "skip_direct_babysit_handoff", "stop_before_publication", + "keep_tracker_open", + "require_escape_journey_revalidation", + "require_visual_layout_evidence", "ticket_scoped_fix", "transfer_exclusive_mutation_ownership", + "verify_live_deployment_candidate_binding", "verify_merge_live", "verify_non_merge_gates", "verify_each_pr_gate", "verify_full_stack_on_base", "verify_stack_topology", + "verify_child_acceptance_ledgers", + "verify_epic_acceptance", + "use_non_closing_reference", ) @@ -117,7 +138,8 @@ def build_prompt(payload: dict) -> str: "Return ONLY one JSON object, no prose and no code fence:", '{"target_skill": "' + payload["target_skill"] + '",', ' "terminal_state": ,", - ' "actions": }', + ' "actions": ,', + ' "acceptance_ledger": }', json.dumps(list(ACTION_VOCABULARY), indent=2), ] ) @@ -160,6 +182,16 @@ def normalize(payload: dict, observed: dict) -> dict: actions = observed.get("actions") if not isinstance(actions, list): actions = [] + ledger = observed.get("acceptance_ledger") + if not isinstance(ledger, list): + ledger = [] + normalized_ledger = [ + entry + for entry in ledger + if isinstance(entry, dict) + and isinstance(entry.get("criterion"), str) + and entry.get("status") in {"pass", "fail", "missing"} + ] return { # Report exactly what the model claimed; backfilling from the payload # would make the grader's target_skill check vacuous. @@ -168,6 +200,7 @@ def normalize(payload: dict, observed: dict) -> dict: "actions": sorted( {str(action) for action in actions if str(action) in ACTION_VOCABULARY} ), + "acceptance_ledger": normalized_ledger, } diff --git a/skills/implement-ticket/scripts/evals/fixture_executor.py b/skills/implement-ticket/scripts/evals/fixture_executor.py index b84fa31..b883820 100644 --- a/skills/implement-ticket/scripts/evals/fixture_executor.py +++ b/skills/implement-ticket/scripts/evals/fixture_executor.py @@ -21,6 +21,212 @@ def compact(text: str) -> str: return re.sub(r"\s+", " ", text) +def build_acceptance_ledger(ticket: dict, handoff: dict) -> list[dict] | None: + """Map authored requirements and raw observations into evidence records.""" + requirements = ticket.get("acceptance_requirements") + if requirements is None: + # Legacy composition controls predate the acceptance scenarios. They keep + # their explicit empty ledger so unrelated cases remain stable. + return ticket.get("acceptance_evidence") + + observations = handoff.get("acceptance_observations") or [] + ledger = [] + for requirement in requirements: + matches = [ + observation + for observation in observations + if observation.get("criterion") == requirement.get("criterion") + ] + observation = matches[0] if len(matches) == 1 else None + contract_fields = ( + "evidence_category", + "stage", + "environment", + "url", + "source", + ) + contract_matches = observation is not None and all( + observation.get(field) == requirement.get(field) + for field in contract_fields + ) + identity = requirement.get("identity") + identity_present = observation is not None and ( + identity == "candidate" + and bool(observation.get("candidate_sha")) + or identity == "deployment" + and bool(observation.get("deployed_sha")) + ) + source_present = observation is not None and bool(observation.get("source")) + outcome = observation.get("outcome") if observation is not None else None + if ( + len(matches) == 1 + and outcome == "passed" + and contract_matches + and identity_present + and source_present + ): + status = "pass" + elif observation is None or outcome in {None, "unavailable"}: + status = "missing" + else: + status = "fail" + ledger.append( + { + "criterion": requirement["criterion"], + "required": requirement.get("required", True), + "evidence_category": requirement["evidence_category"], + "stage": requirement["stage"], + "identity": requirement["identity"], + "candidate_sha": ( + observation.get("candidate_sha") + if observation is not None + else None + ), + "deployed_sha": ( + observation.get("deployed_sha") if observation is not None else None + ), + "environment": requirement.get("environment"), + "url": requirement.get("url"), + "source": observation.get("source") + if observation is not None + else None, + "status": status, + } + ) + return ledger + + +def acceptance_result( + target: str, + ticket: dict, + pr: dict, + handoff: dict, + authority: dict, +) -> tuple[list[str], bool, list[dict]]: + """Evaluate criterion-specific acceptance artifacts and authority boundaries.""" + evidence = build_acceptance_ledger(ticket, handoff) + actions = ["build_acceptance_ledger"] + if target == "implement-epic": + actions.extend(["verify_child_acceptance_ledgers", "verify_epic_acceptance"]) + if evidence is None: + actions.extend( + [ + "reject_missing_required_acceptance", + "keep_tracker_open", + "report_delivery_acceptance_separately", + ] + ) + return actions, True, [] + + required = [entry for entry in evidence if entry.get("required", True)] + missing = any(entry.get("status") != "pass" for entry in required) + stale = any( + entry.get("status") == "pass" + and ( + ( + entry.get("stage") == "pre_merge" + and ( + entry.get("identity") != "candidate" + or entry.get("candidate_sha") != pr.get("head") + ) + ) + or ( + entry.get("stage") == "post_merge" + and ( + not pr.get("merged") + or ( + entry.get("identity") == "candidate" + and entry.get("candidate_sha") != pr.get("head") + ) + or ( + entry.get("identity") == "deployment" + and ( + not entry.get("deployed_sha") + or entry.get("deployed_sha") + != handoff.get("current_deployed_sha") + or handoff.get("current_deployment_candidate_sha") + != pr.get("head") + ) + ) + ) + ) + ) + for entry in required + ) + visual_missing = any( + entry.get("required", True) + and entry.get("evidence_category") == "visual_layout" + and entry.get("status") != "pass" + for entry in evidence + ) + escape_incomplete = handoff.get("escaped_acceptance_defect") and not handoff.get( + "affected_journey_revalidated" + ) + + categories = {entry.get("evidence_category") for entry in required} + if categories and categories <= {"command", "ci", "unit_test", "integration_test"}: + actions.append("avoid_irrelevant_ui_gates") + if visual_missing: + actions.append("require_visual_layout_evidence") + if escape_incomplete: + actions.append("require_escape_journey_revalidation") + if stale: + actions.append("reject_stale_acceptance_evidence") + elif any( + entry.get("status") == "pass" and entry.get("deployed_sha") + for entry in required + ): + actions.append("verify_live_deployment_candidate_binding") + + acceptance_blocked = missing or stale or escape_incomplete + tracker_blocked = ( + target == "implement-ticket" + and (pr.get("merged") or authority.get("merge") and pr.get("mergeable") is True) + and not authority.get("tracker_transition") + ) + if acceptance_blocked: + actions.extend( + [ + "reject_missing_required_acceptance", + "keep_tracker_open", + "report_delivery_acceptance_separately", + ] + ) + if target == "implement-epic" and ( + handoff.get("verified_merged_delivery") + or handoff.get("tracker_transition_observed") + ): + actions.append("refresh_graph_after_verified_delivery") + if ticket.get("state") == "closed" and authority.get("tracker_transition"): + actions.append("reopen_auto_closed_ticket") + if target == "implement-epic" and handoff.get("auto_closed_incomplete_child"): + actions.extend( + [ + "select_auto_closed_incomplete_child", + "invoke_implement_ticket_for_recovery", + ] + ) + if not authority.get("tracker_transition"): + actions.append("report_missing_reopen_authority") + if any(entry.get("stage") == "post_merge" for entry in required): + actions.append("use_non_closing_reference") + if not authority.get("deploy") or not authority.get("tracker_transition"): + actions.append("preserve_acceptance_authority_boundaries") + elif tracker_blocked: + actions.extend( + [ + "allow_acceptance_completion", + "keep_tracker_open", + "report_delivery_acceptance_separately", + "preserve_acceptance_authority_boundaries", + "use_non_closing_reference", + ] + ) + else: + actions.append("allow_acceptance_completion") + return actions, acceptance_blocked or tracker_blocked, evidence + + def action_result(payload: dict) -> dict: target = payload["target_skill"] prompt = compact(payload["skill_prompt"]) @@ -31,12 +237,14 @@ def action_result(payload: dict) -> dict: "`prs_open`", "`ready_prs`", "Normal ticket execution never uses `watch_until_closed`", + "Build the acceptance evidence ledger", ), "implement-epic": ( "Do not make this skill invoke", "`carve-changesets` itself", "`ready_pr`", "`ready_prs`", + "every required child's criterion-specific acceptance ledger", ), }[target] if not all(compact(fragment) in prompt for fragment in required_contract): @@ -44,6 +252,7 @@ def action_result(payload: dict) -> dict: "target_skill": target, "terminal_state": "blocked", "actions": ["skill_contract_incomplete"], + "acceptance_ledger": [], } artifacts = payload["artifacts"] @@ -55,8 +264,16 @@ def action_result(payload: dict) -> dict: handoff = artifacts["handoff"] authority = payload.get("authority") or {} capabilities = payload.get("capabilities") or {} - actions = [] - + actions, acceptance_blocked, acceptance_ledger = acceptance_result( + target, ticket, pr, handoff, authority + ) + if acceptance_blocked: + return { + "target_skill": target, + "terminal_state": "blocked", + "actions": sorted(set(actions)), + "acceptance_ledger": acceptance_ledger, + } if target == "implement-epic": if handoff.get("stack_child_result"): return { @@ -260,7 +477,12 @@ def action_result(payload: dict) -> dict: "actions": actions + ["reject_concurrent_mutation"], } - if authority.get("merge"): + if pr.get("merged"): + actions.extend( + ["verify_merge_live", "caller_verifies_mainline_tracker_cleanup"] + ) + terminal_state = "merged" + elif authority.get("merge"): actions.extend( [ "invoke_merge_when_ready", @@ -277,6 +499,7 @@ def action_result(payload: dict) -> dict: "target_skill": target, "terminal_state": terminal_state, "actions": sorted(set(actions)), + "acceptance_ledger": acceptance_ledger, } diff --git a/skills/implement-ticket/scripts/evals/run_forward.py b/skills/implement-ticket/scripts/evals/run_forward.py index 572b580..bd6c350 100644 --- a/skills/implement-ticket/scripts/evals/run_forward.py +++ b/skills/implement-ticket/scripts/evals/run_forward.py @@ -81,14 +81,50 @@ def grade(case_id: str, observed: dict, expected: dict) -> list[str]: f"target_skill: expected {expected.get('target_skill')!r}, " f"got {observed.get('target_skill')!r}" ) + if "acceptance_statuses" in expected: + ledger = observed.get("acceptance_ledger") or [] + criteria = [ + entry.get("criterion") for entry in ledger if isinstance(entry, dict) + ] + duplicates = sorted( + {criterion for criterion in criteria if criteria.count(criterion) > 1} + ) + if duplicates: + failures.append( + "acceptance_ledger: duplicate criteria " + ", ".join(duplicates) + ) + observed_statuses = { + entry.get("criterion"): entry.get("status") + for entry in ledger + if isinstance(entry, dict) and isinstance(entry.get("criterion"), str) + } + if observed_statuses != expected["acceptance_statuses"]: + failures.append( + "acceptance_statuses: expected " + f"{expected['acceptance_statuses']!r}, got {observed_statuses!r}" + ) return [f"{case_id}: {failure}" for failure in failures] -def evaluate(cases_path: Path, expectations_path: Path, command: list[str]): +def evaluate( + cases_path: Path, + expectations_path: Path, + command: list[str], + target_skill: str | None = None, +): cases = load_json(cases_path) expectations = {item["case_id"]: item for item in load_json(expectations_path)} if {case["id"] for case in cases} != set(expectations): raise ValueError("forward case and expectation IDs differ") + if target_skill: + cases = [case for case in cases if case["target_skill"] == target_skill] + expectations = { + case_id: expected + for case_id, expected in expectations.items() + if expected["target_skill"] == target_skill + } + if not cases: + raise ValueError(f"no forward cases target {target_skill}") observations = {} failures = [] @@ -109,6 +145,11 @@ def parse_args(): default=f"{shlex.quote(sys.executable)} {shlex.quote(str(DEFAULT_EXECUTOR))}", help="Fresh-process evaluator command; receives one result-blind JSON packet on stdin", ) + parser.add_argument( + "--target-skill", + choices=("implement-ticket", "implement-epic"), + help="Run only cases targeting one skill after validating the full corpus", + ) parser.add_argument("--output-dir", type=Path) return parser.parse_args() @@ -116,7 +157,9 @@ def parse_args(): def main() -> int: args = parse_args() command = shlex.split(args.executor) - observations, failures = evaluate(args.cases, args.expectations, command) + observations, failures = evaluate( + args.cases, args.expectations, command, target_skill=args.target_skill + ) if args.output_dir: args.output_dir.mkdir(parents=True, exist_ok=True) diff --git a/skills/implement-ticket/scripts/tests/test_delegated_execution_contract.py b/skills/implement-ticket/scripts/tests/test_delegated_execution_contract.py index 0ef9abb..82ce7ca 100644 --- a/skills/implement-ticket/scripts/tests/test_delegated_execution_contract.py +++ b/skills/implement-ticket/scripts/tests/test_delegated_execution_contract.py @@ -30,8 +30,8 @@ def load_validator() -> ModuleType: def invocation() -> dict[str, object]: return { - "schema": "agent-scripts.implement-ticket/delegated-invocation/v1", - "capability": "agent-scripts.implement-ticket/delegated-execution/v1", + "schema": "agent-scripts.implement-ticket/delegated-invocation/v2", + "capability": "agent-scripts.implement-ticket/delegated-execution/v2", "invocation_id": "run-123", "ticket": { "provider": "github", @@ -76,6 +76,19 @@ def invocation() -> dict[str, object]: "blocked", "requires_epic", ], + "acceptance_requirements": [ + { + "criterion": "Automated regression suite passes", + "required": True, + "evidence_category": "automated_test", + "stage": "pre_merge", + "identity": "candidate", + "environment": "local", + "url": None, + "source": "just test", + } + ], + "starting_deployment": None, "checkpoint": { "command": ["example-coordinator", "checkpoint"], "last_sequence": 0, @@ -111,8 +124,8 @@ def candidate(kind: str = "ordinary") -> dict[str, object]: def result() -> dict[str, object]: source = invocation() return { - "schema": "agent-scripts.implement-ticket/delegated-result/v1", - "capability": "agent-scripts.implement-ticket/delegated-execution/v1", + "schema": "agent-scripts.implement-ticket/delegated-result/v2", + "capability": "agent-scripts.implement-ticket/delegated-execution/v2", "invocation_id": "run-123", "terminal_state": "ready_pr", "ticket": source["ticket"], @@ -122,6 +135,13 @@ def result() -> dict[str, object]: "base_sha": SHA_A, }, "implementation_state": "published", + "tracker_transition": { + "provider": "github", + "ticket_id": "123", + "mode": "none", + "state": "open", + "observed_at": "2026-07-25T12:11:30Z", + }, "candidate": candidate(), "handoff": {"transferable": True, "reason": None}, "checkpoint": { @@ -154,18 +174,51 @@ def result() -> dict[str, object]: "repository.candidate.push", "pull_request.create", ], + "acceptance_evidence": [ + { + "criterion": "Automated regression suite passes", + "required": True, + "evidence_category": "automated_test", + "stage": "pre_merge", + "candidate_sha": SHA_B, + "deployed_sha": None, + "environment": "local", + "url": None, + "source": "just test", + "status": "pass", + } + ], "unresolved_obligations": [], "blocking_reason": None, "next_action": "Caller may accept the ready PR", } +def record_tracker_transition( + value: dict[str, object], + source: dict[str, object] | None = None, + mode: str = "manual", +) -> None: + """Record tracker closure and the authority action that enabled it.""" + value["tracker_transition"] = { + "provider": value["ticket"]["provider"], + "ticket_id": value["ticket"]["id"], + "mode": mode, + "state": "closed", + "observed_at": "2026-07-25T12:12:00Z", + } + action = "ticket.update" if mode == "manual" else "tracker.auto_close.authorize" + value["authority_used"].append(action) + if source is not None: + source["authority"]["allow"].append(action) + + def checkpoint_request(phase: str = "pre_external_mutation") -> dict[str, object]: published_candidate = candidate() published_candidate.pop("publication") return { - "schema": "agent-scripts.implement-ticket/checkpoint-request/v1", - "capability": "agent-scripts.implement-ticket/delegated-execution/v1", + "schema": "agent-scripts.implement-ticket/checkpoint-request/v2", + "capability": "agent-scripts.implement-ticket/delegated-execution/v2", "invocation_id": "run-123", "continuation_token": "token-1", "sequence": 2, @@ -173,13 +226,14 @@ def checkpoint_request(phase: str = "pre_external_mutation") -> dict[str, object "action": "repository.candidate.push", "ticket_observation": "sha256:ticket", "candidate": published_candidate, + "deployment": None, "proposed_effect": "Publish the exact candidate", } def checkpoint_response() -> dict[str, object]: return { - "schema": "agent-scripts.implement-ticket/checkpoint-response/v1", + "schema": "agent-scripts.implement-ticket/checkpoint-response/v2", "invocation_id": "run-123", "request_sequence": 2, "prior_continuation_token": "token-1", @@ -187,6 +241,7 @@ def checkpoint_response() -> dict[str, object]: "decision": "allow", "reason": None, "acknowledged_candidate_sha": None, + "observed_deployment": None, } @@ -201,7 +256,7 @@ def test_all_contract_files_are_valid_json_or_documented_markdown(self) -> None: manifest = json.loads((CONTRACT_ROOT / "capability.json").read_text()) self.assertEqual([], self.validator.validate("capability", manifest)) self.assertIn( - "agent-scripts.implement-ticket/delegated-execution/v1", + "agent-scripts.implement-ticket/delegated-execution/v2", (CONTRACT_ROOT / "CONTRACT.md").read_text(), ) @@ -260,6 +315,36 @@ def test_candidate_publication_requires_exact_acknowledgement(self) -> None: self.validator.validate_checkpoint_exchange(request, response), ) + def test_deployment_observation_requires_caller_verified_candidate_binding( + self, + ) -> None: + request = checkpoint_request("deployment_observed") + request["action"] = "deployment.execute" + request["deployment"] = { + "candidate_sha": SHA_B, + "deployed_sha": SHA_C, + "environment": "production", + "url": "https://example.test", + } + response = checkpoint_response() + response["observed_deployment"] = copy.deepcopy(request["deployment"]) + + self.assertEqual( + [], self.validator.validate_checkpoint_exchange(request, response) + ) + + response["observed_deployment"]["candidate_sha"] = SHA_A + self.assertIn( + "$.observed_deployment: does not match caller-verified deployment", + self.validator.validate_checkpoint_exchange(request, response), + ) + + request["deployment"]["candidate_sha"] = SHA_A + self.assertIn( + "$.deployment.candidate_sha: does not match candidate", + self.validator.validate("checkpoint-request", request), + ) + def test_checkpoint_identity_and_token_must_match(self) -> None: request = checkpoint_request() response = checkpoint_response() @@ -392,6 +477,152 @@ def test_terminal_checkpoint_must_match_caller_ledger_tail(self) -> None: errors, ) + def test_checkpoint_tail_accepts_live_candidate_bound_deployment(self) -> None: + source = invocation() + source["validation"] = ["just test"] + source["desired_outcome"] = "merged" + source["accepted_terminal_states"].append("merged") + source["acceptance_requirements"].append( + { + "criterion": "Production deployment smoke passes", + "required": True, + "evidence_category": "deployed_integration", + "stage": "post_merge", + "identity": "deployment", + "environment": "production", + "url": "https://example.test", + "source": "production smoke run", + } + ) + value = result() + value["terminal_state"] = "merged" + record_tracker_transition(value, source) + value["candidate"]["publication"]["pull_requests"][0]["state"] = "merged" + value["acceptance_evidence"].append( + { + "criterion": "Production deployment smoke passes", + "required": True, + "evidence_category": "deployed_integration", + "stage": "post_merge", + "candidate_sha": None, + "deployed_sha": SHA_C, + "environment": "production", + "url": "https://example.test", + "source": "production smoke run", + "status": "pass", + } + ) + observed_deployment = { + "candidate_sha": SHA_B, + "deployed_sha": SHA_C, + "environment": "production", + "url": "https://example.test", + } + observed_tracker = copy.deepcopy(value["tracker_transition"]) + consumed_authority = list(value["authority_used"]) + + self.assertEqual( + [], + self.validator.validate_result_checkpoint_state( + source, + value, + 4, + "token-4", + observed_deployment, + observed_tracker, + consumed_authority, + ), + ) + + unauthorized = copy.deepcopy(source) + unauthorized["authority"]["allow"].remove("ticket.update") + self.assertIn( + "$.authority_used: exceeds invocation: ticket.update", + self.validator.validate_result_checkpoint_state( + unauthorized, + value, + 4, + "token-4", + observed_deployment, + observed_tracker, + consumed_authority, + ), + ) + + unverified_tracker = copy.deepcopy(observed_tracker) + unverified_tracker["observed_at"] = "2026-07-25T12:13:00Z" + self.assertIn( + "$.tracker_transition: does not match caller-observed tracker state", + self.validator.validate_result_checkpoint_state( + source, + value, + 4, + "token-4", + observed_deployment, + unverified_tracker, + consumed_authority, + ), + ) + + no_transition_authority = copy.deepcopy(value) + no_transition_authority["authority_used"].remove("ticket.update") + self.assertIn( + "$.authority_used: manual tracker transition requires ticket.update", + self.validator.validate_result_checkpoint_state( + source, + no_transition_authority, + 4, + "token-4", + observed_deployment, + observed_tracker, + no_transition_authority["authority_used"], + ), + ) + + automatic_source = copy.deepcopy(source) + automatic_source["authority"]["allow"].remove("ticket.update") + automatic_source["authority"]["allow"].append("tracker.auto_close.authorize") + automatic = copy.deepcopy(value) + automatic["tracker_transition"]["mode"] = "automatic" + automatic["authority_used"].remove("ticket.update") + automatic["authority_used"].append("tracker.auto_close.authorize") + automatic_tracker = copy.deepcopy(automatic["tracker_transition"]) + self.assertEqual( + [], + self.validator.validate_result_checkpoint_state( + automatic_source, + automatic, + 4, + "token-4", + observed_deployment, + automatic_tracker, + automatic["authority_used"], + ), + ) + self.assertIn( + "$.observed_tracker: merged requires caller observation", + self.validator.validate_result_checkpoint_state( + source, + value, + 4, + "token-4", + observed_deployment, + consumed_authority=consumed_authority, + ), + ) + self.assertIn( + "$.authority_used: does not match caller authority ledger", + self.validator.validate_result_checkpoint_state( + automatic_source, + automatic, + 4, + "token-4", + observed_deployment, + automatic_tracker, + consumed_authority, + ), + ) + def test_candidate_must_match_invocation_repository(self) -> None: value = result() value["candidate"]["repository"] = "github:other/project" @@ -467,6 +698,274 @@ def test_duplicate_observation_names_fail_closed(self) -> None: self.validator.validate("result", value), ) + def test_acceptance_evidence_is_required_and_candidate_bound(self) -> None: + value = result() + value.pop("acceptance_evidence") + self.assertIn( + "$: missing required property 'acceptance_evidence'", + self.validator.validate("result", value), + ) + value = result() + value["acceptance_evidence"][0]["candidate_sha"] = SHA_A + self.assertIn( + "$.acceptance_evidence: candidate mismatch for Automated regression suite passes", + self.validator.validate("result", value), + ) + + def test_ready_pr_requires_pre_merge_acceptance(self) -> None: + value = result() + value["acceptance_evidence"][0].update({"status": "missing", "source": None}) + self.assertIn( + "$.acceptance_evidence: required pre-merge evidence incomplete for " + "Automated regression suite passes", + self.validator.validate("result", value), + ) + + def test_merged_requires_post_merge_acceptance(self) -> None: + value = result() + value["terminal_state"] = "merged" + record_tracker_transition(value) + value["candidate"]["publication"]["pull_requests"][0]["state"] = "merged" + value["acceptance_evidence"].append( + { + "criterion": "Production deployment smoke passes", + "required": True, + "evidence_category": "deployed_integration", + "stage": "post_merge", + "candidate_sha": None, + "deployed_sha": None, + "environment": "production", + "url": "https://example.test", + "source": None, + "status": "missing", + } + ) + self.assertIn( + "$.acceptance_evidence: merged requires complete evidence for " + "Production deployment smoke passes", + self.validator.validate("result", value), + ) + value["acceptance_evidence"][-1].update( + { + "candidate_sha": None, + "deployed_sha": SHA_A, + "source": "production smoke run", + "status": "pass", + } + ) + self.assertEqual([], self.validator.validate("result", value)) + value["acceptance_evidence"][-1]["candidate_sha"] = SHA_A + self.assertIn( + "$.acceptance_evidence: candidate mismatch for " + "Production deployment smoke passes", + self.validator.validate("result", value), + ) + + def test_invocation_anchors_acceptance_contract_and_deployment(self) -> None: + source = invocation() + source["desired_outcome"] = "merged" + source["accepted_terminal_states"].append("merged") + source["acceptance_requirements"].append( + { + "criterion": "Production deployment smoke passes", + "required": True, + "evidence_category": "deployed_integration", + "stage": "post_merge", + "identity": "deployment", + "environment": "production", + "url": "https://example.test", + "source": "production smoke run", + } + ) + source["starting_deployment"] = { + "candidate_sha": SHA_B, + "deployed_sha": SHA_C, + "environment": "production", + "url": "https://example.test", + } + value = result() + value["terminal_state"] = "merged" + record_tracker_transition(value, source) + value["candidate"]["publication"]["pull_requests"][0]["state"] = "merged" + value["validation"].append( + { + "name": "just lint", + "outcome": "passed", + "candidate_sha": SHA_B, + "observed_at": "2026-07-25T12:01:00Z", + } + ) + value["acceptance_evidence"].append( + { + "criterion": "Production deployment smoke passes", + "required": True, + "evidence_category": "deployed_integration", + "stage": "post_merge", + "candidate_sha": None, + "deployed_sha": SHA_C, + "environment": "production", + "url": "https://example.test", + "source": "production smoke run", + "status": "pass", + } + ) + observed_tracker = copy.deepcopy(value["tracker_transition"]) + consumed_authority = list(value["authority_used"]) + self.assertEqual( + [], + self.validator.validate_result_for_invocation( + source, + value, + observed_tracker=observed_tracker, + consumed_authority=consumed_authority, + ), + ) + + dynamic_source = copy.deepcopy(source) + dynamic_source["starting_deployment"] = None + observed_deployment = copy.deepcopy(source["starting_deployment"]) + self.assertEqual( + [], + self.validator.validate_result_for_invocation( + dynamic_source, + value, + observed_deployment, + observed_tracker, + consumed_authority, + ), + ) + self.assertIn( + "$.acceptance_evidence: passing deployment evidence lacks " + "caller-observed deployment for Production deployment smoke passes", + self.validator.validate_result_for_invocation( + dynamic_source, + value, + observed_tracker=observed_tracker, + consumed_authority=consumed_authority, + ), + ) + observed_deployment["candidate_sha"] = SHA_A + self.assertIn( + "$.acceptance_evidence: deployment candidate mismatch for " + "Production deployment smoke passes", + self.validator.validate_result_for_invocation( + dynamic_source, + value, + observed_deployment, + observed_tracker, + consumed_authority, + ), + ) + + omitted = copy.deepcopy(value) + omitted["acceptance_evidence"].pop() + self.assertIn( + "$.acceptance_evidence: missing invocation criteria " + "Production deployment smoke passes", + self.validator.validate_result_for_invocation( + source, + omitted, + observed_tracker=observed_tracker, + consumed_authority=consumed_authority, + ), + ) + for field, replacement, expected_error in ( + ( + "deployed_sha", + None, + "$.acceptance_evidence: passing evidence requires candidate or " + "deployed SHA for Production deployment smoke passes", + ), + ( + "deployed_sha", + SHA_A, + "$.acceptance_evidence: current deployment deployed_sha mismatch for " + "Production deployment smoke passes", + ), + ( + "environment", + "staging", + "$.acceptance_evidence: environment mismatch for " + "Production deployment smoke passes", + ), + ( + "evidence_category", + "browser_functional", + "$.acceptance_evidence: evidence_category mismatch for " + "Production deployment smoke passes", + ), + ( + "source", + "delegate summary", + "$.acceptance_evidence: source mismatch for " + "Production deployment smoke passes", + ), + ): + mismatched = copy.deepcopy(value) + mismatched["acceptance_evidence"][-1][field] = replacement + self.assertIn( + expected_error, + self.validator.validate_result_for_invocation( + source, + mismatched, + observed_tracker=observed_tracker, + consumed_authority=consumed_authority, + ), + ) + + def test_postmerge_acceptance_may_bind_to_merged_candidate(self) -> None: + source = invocation() + source["desired_outcome"] = "merged" + source["accepted_terminal_states"].append("merged") + source["acceptance_requirements"].append( + { + "criterion": "Merged result is represented on main", + "required": True, + "evidence_category": "mainline_representation", + "stage": "post_merge", + "identity": "candidate", + "environment": "repository", + "url": None, + "source": "git merge-base --is-ancestor", + } + ) + value = result() + value["terminal_state"] = "merged" + record_tracker_transition(value, source) + value["candidate"]["publication"]["pull_requests"][0]["state"] = "merged" + value["validation"].append( + { + "name": "just lint", + "outcome": "passed", + "candidate_sha": SHA_B, + "observed_at": "2026-07-25T12:01:00Z", + } + ) + value["acceptance_evidence"].append( + { + "criterion": "Merged result is represented on main", + "required": True, + "evidence_category": "mainline_representation", + "stage": "post_merge", + "candidate_sha": SHA_B, + "deployed_sha": None, + "environment": "repository", + "url": None, + "source": "git merge-base --is-ancestor", + "status": "pass", + } + ) + + self.assertEqual( + [], + self.validator.validate_result_for_invocation( + source, + value, + observed_tracker=copy.deepcopy(value["tracker_transition"]), + consumed_authority=list(value["authority_used"]), + ), + ) + def test_ready_prs_require_unique_ordered_topology(self) -> None: value = result() value["terminal_state"] = "ready_prs" @@ -574,6 +1073,7 @@ def test_skill_loads_and_enforces_delegated_contract(self) -> None: self.assertIn("before every action", skill) self.assertIn("invocation's `last_sequence`", skill) self.assertIn("candidate_published", skill) + self.assertIn("deployment_observed", skill) self.assertIn("versioned delegated-execution contract", result_reference) diff --git a/skills/implement-ticket/scripts/tests/test_forward_evals.py b/skills/implement-ticket/scripts/tests/test_forward_evals.py index c7cf4dd..742488b 100644 --- a/skills/implement-ticket/scripts/tests/test_forward_evals.py +++ b/skills/implement-ticket/scripts/tests/test_forward_evals.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import importlib.util import json import sys @@ -15,6 +16,13 @@ assert SPEC and SPEC.loader SPEC.loader.exec_module(RUNNER) +FIXTURE_SPEC = importlib.util.spec_from_file_location( + "implement_ticket_fixture_executor", EXECUTOR_PATH +) +FIXTURE_EXECUTOR = importlib.util.module_from_spec(FIXTURE_SPEC) +assert FIXTURE_SPEC and FIXTURE_SPEC.loader +FIXTURE_SPEC.loader.exec_module(FIXTURE_EXECUTOR) + CLAUDE_EXECUTOR_PATH = SKILL_ROOT / "scripts" / "evals" / "claude_executor.py" CLAUDE_SPEC = importlib.util.spec_from_file_location( "implement_ticket_claude_executor", CLAUDE_EXECUTOR_PATH @@ -42,10 +50,29 @@ def test_every_packet_contains_raw_live_shaped_artifact_categories(self): "worktree", "handoff", } - self.assertEqual(25, len(self.cases)) + self.assertEqual(41, len(self.cases)) for case in self.cases: self.assertEqual(required, set(case["artifacts"]), case["id"]) + def test_acceptance_packets_separate_requirements_from_raw_observations(self): + acceptance_cases = [ + case + for case in self.cases + if "acceptance_requirements" in case["artifacts"]["ticket"] + ] + self.assertGreaterEqual(len(acceptance_cases), 15) + for case in acceptance_cases: + requirements = case["artifacts"]["ticket"]["acceptance_requirements"] + observations = case["artifacts"]["handoff"].get( + "acceptance_observations", [] + ) + with self.subTest(case=case["id"]): + self.assertTrue(requirements) + self.assertTrue(all("status" not in item for item in requirements)) + self.assertTrue(all("status" not in item for item in observations)) + self.assertTrue(all("outcome" not in item for item in requirements)) + self.assertTrue(all(item.get("source") for item in requirements)) + def test_executor_payload_is_result_blind(self): for case in self.cases: payload = RUNNER.build_payload(case) @@ -57,6 +84,20 @@ def test_executor_payload_is_result_blind(self): self.assertNotIn("terminal_state", serialized) self.assertNotIn(self.expectations_text, serialized) + def test_reference_executor_actions_fit_real_runtime_vocabulary(self): + emitted_actions = { + action + for case in self.cases + for action in FIXTURE_EXECUTOR.action_result(RUNNER.build_payload(case))[ + "actions" + ] + } + emitted_actions.add("skill_contract_incomplete") + self.assertEqual( + set(), + emitted_actions - set(CLAUDE_EXECUTOR.ACTION_VOCABULARY), + ) + def test_forward_cases_execute_fresh_and_pass_separate_grading(self): observations, failures = RUNNER.evaluate( RUNNER.DEFAULT_CASES, @@ -64,9 +105,9 @@ def test_forward_cases_execute_fresh_and_pass_separate_grading(self): [sys.executable, str(EXECUTOR_PATH)], ) self.assertEqual([], failures) - self.assertEqual(25, len(observations)) + self.assertEqual(41, len(observations)) process_ids = {result["executor_pid"] for result in observations.values()} - self.assertEqual(25, len(process_ids)) + self.assertEqual(41, len(process_ids)) def test_reference_executor_evaluates_the_supplied_skill_prompt(self): payload = RUNNER.build_payload(self.cases[2]) @@ -78,6 +119,28 @@ def test_reference_executor_evaluates_the_supplied_skill_prompt(self): self.assertEqual("blocked", observed["terminal_state"]) self.assertIn("skill_contract_incomplete", observed["actions"]) + def test_acceptance_cases_depend_on_the_acceptance_skill_contract(self): + for case_id, fragment in ( + ( + "functional-browser-missing-visual-layout", + "Build the acceptance evidence ledger", + ), + ( + "epic-closed-children-missing-manual-browser", + "every required child's criterion-specific acceptance ledger", + ), + ): + case = next(item for item in self.cases if item["id"] == case_id) + payload = RUNNER.build_payload(case) + payload["skill_prompt"] = payload["skill_prompt"].replace(fragment, "") + observed = RUNNER.run_executor( + [sys.executable, str(EXECUTOR_PATH)], payload + ) + with self.subTest(case=case_id): + self.assertEqual("blocked", observed["terminal_state"]) + self.assertEqual([], observed["acceptance_ledger"]) + self.assertIn("skill_contract_incomplete", observed["actions"]) + def test_vocabulary_spam_fails_every_case(self): """An executor emitting the whole action vocabulary must never pass. @@ -148,6 +211,130 @@ def test_required_composition_cases_are_executable(self): observations["stale-carved-result"]["actions"], ) + def test_acceptance_cases_fail_closed_or_complete_from_raw_evidence(self): + observations, failures = RUNNER.evaluate( + RUNNER.DEFAULT_CASES, + RUNNER.DEFAULT_EXPECTATIONS, + [sys.executable, str(EXECUTOR_PATH)], + ) + self.assertEqual([], failures) + for case_id in ( + "epic-closed-children-missing-manual-browser", + "auto-closed-missing-postmerge-deployment", + "authenticated-deployed-browser-unavailable", + "functional-browser-missing-visual-layout", + "merge-without-deploy-or-close-authority", + "reopened-epic-correction-without-journey-revalidation", + "stale-acceptance-evidence", + "epic-auto-closed-child-incomplete", + "prior-unrelated-deployment-evidence", + "wrong-source-acceptance-evidence", + "deployment-requirement-rejects-candidate-fallback", + "epic-refreshes-after-blocked-merged-delivery", + ): + self.assertEqual("blocked", observations[case_id]["terminal_state"]) + self.assertEqual( + "merged", observations["all-acceptance-current"]["terminal_state"] + ) + self.assertEqual( + "merged", observations["backend-only-no-ui-gates"]["terminal_state"] + ) + self.assertIn( + "avoid_irrelevant_ui_gates", + observations["backend-only-no-ui-gates"]["actions"], + ) + self.assertIn( + "require_visual_layout_evidence", + observations["functional-browser-missing-visual-layout"]["actions"], + ) + self.assertIn( + "select_auto_closed_incomplete_child", + observations["epic-auto-closed-child-incomplete"]["actions"], + ) + self.assertIn( + "invoke_implement_ticket_for_recovery", + observations["epic-auto-closed-child-incomplete"]["actions"], + ) + self.assertEqual( + "fail", + observations["wrong-source-acceptance-evidence"]["acceptance_ledger"][0][ + "status" + ], + ) + deployment_fallback = observations[ + "deployment-requirement-rejects-candidate-fallback" + ] + self.assertEqual( + "deployment", deployment_fallback["acceptance_ledger"][0]["identity"] + ) + self.assertIn( + "reject_stale_acceptance_evidence", deployment_fallback["actions"] + ) + self.assertIn( + "refresh_graph_after_verified_delivery", + observations["epic-refreshes-after-blocked-merged-delivery"]["actions"], + ) + + def test_reference_executor_rejects_null_pass_identity(self): + case = copy.deepcopy( + next(item for item in self.cases if item["id"] == "all-acceptance-current") + ) + post_merge = next( + entry + for entry in case["artifacts"]["handoff"]["acceptance_observations"] + if entry["stage"] == "post_merge" + ) + post_merge["deployed_sha"] = None + observed = RUNNER.run_executor( + [sys.executable, str(EXECUTOR_PATH)], RUNNER.build_payload(case) + ) + self.assertEqual("blocked", observed["terminal_state"]) + self.assertIn("reject_missing_required_acceptance", observed["actions"]) + self.assertEqual("fail", observed["acceptance_ledger"][-1]["status"]) + + def test_reference_executor_accepts_candidate_bound_postmerge_identity(self): + case = copy.deepcopy( + next(item for item in self.cases if item["id"] == "all-acceptance-current") + ) + requirement = next( + entry + for entry in case["artifacts"]["ticket"]["acceptance_requirements"] + if entry["stage"] == "post_merge" + ) + observation = next( + entry + for entry in case["artifacts"]["handoff"]["acceptance_observations"] + if entry["stage"] == "post_merge" + ) + requirement["identity"] = "candidate" + observation["candidate_sha"] = case["artifacts"]["pr"]["head"] + observation["deployed_sha"] = None + case["artifacts"]["handoff"]["current_deployed_sha"] = None + + observed = RUNNER.run_executor( + [sys.executable, str(EXECUTOR_PATH)], RUNNER.build_payload(case) + ) + + self.assertEqual("merged", observed["terminal_state"]) + self.assertNotIn("reject_stale_acceptance_evidence", observed["actions"]) + self.assertEqual("pass", observed["acceptance_ledger"][-1]["status"]) + + def test_target_skill_filter_runs_only_epic_cases(self): + observations, failures = RUNNER.evaluate( + RUNNER.DEFAULT_CASES, + RUNNER.DEFAULT_EXPECTATIONS, + [sys.executable, str(EXECUTOR_PATH)], + target_skill="implement-epic", + ) + self.assertEqual([], failures) + self.assertEqual(6, len(observations)) + self.assertTrue( + all( + result["target_skill"] == "implement-epic" + for result in observations.values() + ) + ) + if __name__ == "__main__": unittest.main() diff --git a/skills/implement-ticket/scripts/tests/test_implement_ticket_contract.py b/skills/implement-ticket/scripts/tests/test_implement_ticket_contract.py index 5c1842c..8243559 100644 --- a/skills/implement-ticket/scripts/tests/test_implement_ticket_contract.py +++ b/skills/implement-ticket/scripts/tests/test_implement_ticket_contract.py @@ -161,6 +161,69 @@ def test_eval_expectations_enforce_routing_and_authority(self): "ready_prs", self.expectations["oversized-authorized-carved-stack"]["terminal_state"], ) + for case_id in ( + "auto-closed-missing-postmerge-acceptance", + "authenticated-browser-unavailable", + "functional-browser-without-visual-evidence", + "merge-without-deploy-or-close-authority", + "stale-acceptance-evidence", + ): + self.assertEqual("blocked", self.expectations[case_id]["terminal_state"]) + self.assertEqual( + "merged", self.expectations["backend-only-acceptance"]["terminal_state"] + ) + + def test_acceptance_evidence_is_criterion_specific_and_fail_closed(self): + for field in ( + "criterion text or stable identity", + "evidence category", + "pre-merge or post-merge", + "exact candidate SHA", + "deployed SHA", + "environment and URL", + "source", + "`pass`, `fail`, or `missing`", + ): + self.assertIn(field, self.all_contract) + self.assertIn("wrong-environment", self.skill_compact) + self.assertIn("category-mismatched", self.skill_compact) + self.assertIn("return `blocked`", self.skill_compact) + + def test_closing_syntax_and_post_merge_transition_are_acceptance_gated(self): + self.assertIn("non-closing reference", self.skill_compact) + self.assertIn("`Fixes #`", self.github) + self.assertIn("`Refs #`", self.github) + self.assertIn("`Supports #`", self.github) + self.assertIn( + "Reopen it when manual transition authority permits", self.skill_compact + ) + self.assertIn("Close manually only after the ledger passes", self.skill_compact) + + def test_acceptance_does_not_invent_irrelevant_ui_gates(self): + self.assertIn( + "Do not add browser, deployment, authenticated, integration, manual, visual, or full-system gates that the ticket does not require", + self.skill_compact, + ) + self.assertIn( + "do not satisfy an explicit visual-layout requirement", + self.skill_compact, + ) + + def test_escaped_acceptance_requires_focused_revalidation(self): + self.assertIn("focused corrective ticket", self.skill_compact) + self.assertIn("regression test at the escaped boundary", self.skill_compact) + self.assertIn("full affected customer journey", self.skill_compact) + self.assertIn("do not impose unrelated full-system testing", self.skill_compact) + + def test_epic_child_unblocks_only_after_acceptance_transition(self): + self.assertIn( + "report newly unblocked downstream work only after the child acceptance " + "ledger and authorized tracker transition pass", + self.skill_compact, + ) + self.assertNotIn( + "report newly unblocked downstream work after merge", self.skill_compact + ) def test_runtime_adapters_exist_for_both_products(self): metadata = read(SKILL_ROOT / "agents" / "openai.yaml")