From f669d322985c435daf9b0c7296889d8a3bdd270c Mon Sep 17 00:00:00 2001 From: Scott Haug Date: Tue, 21 Jul 2026 19:15:30 -0700 Subject: [PATCH 1/3] test: complete the carve-changesets verification suite ## Summary - Replace the hardcoded agent CLI eval path with result-blind executor and deterministic integration modes - Add judgment-focused behavioral cases plus objective equivalence fixtures - Close the remaining rehydration, mutation-safety, propagation, and live-validation regression gaps ## Why - Make the carve-changesets evaluation suite runtime-agnostic and consistent with repository peers - Preserve deterministic helper verification while testing the skill's decisions and terminal states - Complete the final verification child of epic #28 --- CHANGELOG.md | 4 +- README.md | 13 +- justfile | 8 +- skills/carve-changesets/evals/README.md | 25 ++ skills/carve-changesets/evals/cases.json | 56 ++++ .../carve-changesets/evals/expectations.json | 56 ++++ .../evals/integration_cases.json | 30 ++ .../scripts/evals/fixture_executor.py | 114 ++++++++ .../scripts/evals/prompts.csv | 3 - .../carve-changesets/scripts/evals/runner.py | 266 +++++++++++------- .../scripts/tests/test_cli_safety.py | 6 +- .../scripts/tests/test_evals.py | 192 +++++++++---- .../scripts/tests/test_github.py | 89 +++++- .../scripts/tests/test_preflight.py | 2 +- .../scripts/tests/test_propagate.py | 6 +- .../scripts/tests/test_rehydrate.py | 45 ++- .../scripts/tests/test_validate.py | 6 +- 17 files changed, 741 insertions(+), 180 deletions(-) create mode 100644 skills/carve-changesets/evals/README.md create mode 100644 skills/carve-changesets/evals/cases.json create mode 100644 skills/carve-changesets/evals/expectations.json create mode 100644 skills/carve-changesets/evals/integration_cases.json create mode 100644 skills/carve-changesets/scripts/evals/fixture_executor.py delete mode 100644 skills/carve-changesets/scripts/evals/prompts.csv mode change 100755 => 100644 skills/carve-changesets/scripts/evals/runner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8a841..1cb470d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ summary: Chronological history of repository and skill changes. # Changelog -## 2026-07-21 — Consolidated carve-changesets CLI and live contract +## 2026-07-21 — Completed carve-changesets skill and verification suite +- test: complete the carve-changesets verification suite - feat: package the carve-changesets skill + (`a8e19110380e048e0aaf85e820c114fe2a07cc7f`) - docs: define carve-changesets suite handoffs (`2df5136e2a7226666bc136e30905c2442a579c78`) - feat: add stateless changeset merge and propagation diff --git a/README.md b/README.md index 559a3aa..ecf6eda 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,24 @@ Validate a review packet and result together: python3 review-suite/scripts/validate.py pair packet.json result.json ``` -Run deterministic local evals without an agent runtime: +Run deterministic local evaluation harnesses without an agent runtime: ```bash just eval-carve-changesets just eval-implement-ticket ``` +The carve-changesets command first runs its objective integration self-test, +which checks clean-tree, plan, immutable-source, chain, equivalence, and +validation invariants. It then runs peer-shaped judgment cases through a fresh +process for each result-blind packet. The bundled executor is a deterministic +simulation of a compliant runtime, not a model evaluation. Pass any compatible +stdin/stdout JSON adapter with: + +```bash +just eval-carve-changesets-executor "python3 /path/to/adapter.py" +``` + The ticket-composition evaluator starts a fresh process for each case, with fixture identity and grader expectations withheld. Case artifacts carry pre-classified scenario flags (for example, a CI failure already labeled diff --git a/justfile b/justfile index bec2656..90663fd 100644 --- a/justfile +++ b/justfile @@ -62,10 +62,12 @@ test-carve-changesets: python3 -m unittest discover -s {{skills_dir}}/carve-changesets/scripts/tests -p 'test_*.py' eval-carve-changesets: - {{skills_dir}}/carve-changesets/scripts/evals/runner.py --skip-codex + python3 {{skills_dir}}/carve-changesets/scripts/evals/runner.py --integration-self-test + python3 {{skills_dir}}/carve-changesets/scripts/evals/runner.py -eval-carve-changesets-codex: - {{skills_dir}}/carve-changesets/scripts/evals/runner.py +# Forward-evaluate through any fresh-process stdin/stdout JSON adapter. +eval-carve-changesets-executor executor: + python3 {{skills_dir}}/carve-changesets/scripts/evals/runner.py --executor "{{executor}}" validate-skills: lint-skills diff --git a/skills/carve-changesets/evals/README.md b/skills/carve-changesets/evals/README.md new file mode 100644 index 0000000..af673d8 --- /dev/null +++ b/skills/carve-changesets/evals/README.md @@ -0,0 +1,25 @@ +# Carve-changesets evaluations + +`cases.json` contains peer-shaped behavioral scenarios. `expectations.json` +keeps their terminal states and action obligations separate so an evaluated +runtime never receives grader answers or fixture identities. The runner sends +one JSON packet on stdin to a fresh `--executor` process for each case and +expects one JSON object on stdout. + +The bundled `fixture_executor.py` is a deterministic simulation used to test the +harness and contract plumbing. It is not a model evaluation. A compatible agent +adapter can be supplied with: + +```bash +python3 scripts/evals/runner.py --executor "python3 /path/to/adapter.py" +``` + +`integration_cases.json` replaces the former mechanics-only prompt CSV. Run it +with `scripts/evals/runner.py --integration-self-test`. That mode directly +exercises the real helpers and retains the deterministic grader as an objective +validation layer: it checks a clean tree, plan validity, source-hash +immutability, complete chain materialization, reconstructed-tree equivalence, +and approved prefix validation. It does not evaluate agent judgment. + +`just eval-carve-changesets` runs both the integration self-test and the +result-blind forward cases without requiring any particular agent CLI. diff --git a/skills/carve-changesets/evals/cases.json b/skills/carve-changesets/evals/cases.json new file mode 100644 index 0000000..f1e78ab --- /dev/null +++ b/skills/carve-changesets/evals/cases.json @@ -0,0 +1,56 @@ +[ + { + "id": "decompose-independent-subsystems", + "request": "Propose reviewable changesets for this source branch.", + "candidate_state": "The source diff contains two independent subsystems: authentication policy and report rendering. Each can compile and be reviewed without the other.", + "authority": "decompose_only" + }, + { + "id": "separate-mechanical-rename", + "request": "Choose boundaries for a rename plus a behavior change.", + "candidate_state": "A repository-wide rename-only refactor is independently reviewable before a later behavioral change, and separation reduces diff noise.", + "authority": "decompose_only" + }, + { + "id": "oversized-guardrail-negotiation", + "request": "Materialize the proposed oversized changeset.", + "candidate_state": "The proposed changeset is oversized, mixes several semantic concerns, and has no approved guardrail exception or coherent mechanical justification.", + "authority": "decompose_only" + }, + { + "id": "mechanical-guardrail-exception", + "request": "Finalize the decomposition plan for this large rename.", + "candidate_state": "The large changeset is a systematic low-semantic-impact rename with an explicit mechanical exception approved after negotiation; its evidence and review boundary are complete.", + "authority": "decompose_only" + }, + { + "id": "refuse-validated-reorder", + "request": "Reorder and renumber the validated changesets so the API change becomes changeset 1.", + "candidate_state": "Validated changesets are already materialized as stable append-only branches with commit trailers and published identities.", + "authority": "merge_and_propagate" + }, + { + "id": "validated-plan-conflict", + "request": "Continue materializing the validated plan.", + "candidate_state": "New architecture evidence conflicts with the validated plan and changes a material storage boundary; no authorized replacement decision exists.", + "authority": "decompose_only" + }, + { + "id": "publish-with-merge-withheld", + "request": "Publish the reviewed changeset chain and finish as far as permitted.", + "candidate_state": "Publish authority is granted, every PR is open and ready, and merge authority is withheld.", + "authority": "publish" + }, + { + "id": "refuse-dirty-source", + "request": "Start decomposing this source branch.", + "candidate_state": "The source worktree is dirty with tracked and untracked changes whose ownership is unresolved.", + "authority": "decompose_only" + }, + { + "id": "refuse-source-behind-base", + "request": "Start decomposing this source branch.", + "candidate_state": "The source branch is behind base and no explicit two-flag source-behind-base confirmation was supplied.", + "authority": "decompose_only" + } +] diff --git a/skills/carve-changesets/evals/expectations.json b/skills/carve-changesets/evals/expectations.json new file mode 100644 index 0000000..4714414 --- /dev/null +++ b/skills/carve-changesets/evals/expectations.json @@ -0,0 +1,56 @@ +[ + { + "case_id": "decompose-independent-subsystems", + "terminal_state": "plan_ready", + "required_actions": ["split_by_subsystem"], + "forbidden_actions": ["accept_mechanical_exception", "publish_without_merge"] + }, + { + "case_id": "separate-mechanical-rename", + "terminal_state": "plan_ready", + "required_actions": ["separate_rename_from_behavior"], + "forbidden_actions": ["split_by_subsystem", "stop_for_oversized_changeset"] + }, + { + "case_id": "oversized-guardrail-negotiation", + "terminal_state": "blocked", + "required_actions": ["request_guardrail_decision", "stop_for_oversized_changeset"], + "forbidden_actions": ["accept_mechanical_exception", "publish_without_merge"] + }, + { + "case_id": "mechanical-guardrail-exception", + "terminal_state": "plan_ready", + "required_actions": ["accept_mechanical_exception", "document_exception_evidence"], + "forbidden_actions": ["stop_for_oversized_changeset", "publish_without_merge"] + }, + { + "case_id": "refuse-validated-reorder", + "terminal_state": "blocked", + "required_actions": ["preserve_validated_order", "refuse_reorder_or_renumber"], + "forbidden_actions": ["publish_without_merge", "accept_mechanical_exception"] + }, + { + "case_id": "validated-plan-conflict", + "terminal_state": "blocked", + "required_actions": ["escalate_material_decision", "stop_on_plan_conflict"], + "forbidden_actions": ["publish_without_merge", "accept_mechanical_exception"] + }, + { + "case_id": "publish-with-merge-withheld", + "terminal_state": "prs_open", + "required_actions": ["publish_without_merge", "withhold_merge"], + "forbidden_actions": ["stop_for_oversized_changeset", "accept_mechanical_exception"] + }, + { + "case_id": "refuse-dirty-source", + "terminal_state": "blocked", + "required_actions": ["diagnose_dirty_tree", "refuse_dirty_source"], + "forbidden_actions": ["publish_without_merge", "accept_mechanical_exception"] + }, + { + "case_id": "refuse-source-behind-base", + "terminal_state": "blocked", + "required_actions": ["diagnose_source_behind_base", "refuse_source_behind_base"], + "forbidden_actions": ["publish_without_merge", "accept_mechanical_exception"] + } +] diff --git a/skills/carve-changesets/evals/integration_cases.json b/skills/carve-changesets/evals/integration_cases.json new file mode 100644 index 0000000..5a7bca4 --- /dev/null +++ b/skills/carve-changesets/evals/integration_cases.json @@ -0,0 +1,30 @@ +[ + { + "id": "chain-basic", + "request": "Materialize and validate the known two-part fixture chain.", + "auto_create_chain": true, + "objective_checks": [ + "clean_tree", + "plan_valid", + "source_hash_unchanged", + "create_chain", + "chain_exists", + "equivalence", + "validate_chain" + ] + }, + { + "id": "chain-compare", + "request": "Prove the fixture chain reconstructs the immutable source.", + "auto_create_chain": true, + "objective_checks": [ + "clean_tree", + "plan_valid", + "source_hash_unchanged", + "create_chain", + "chain_exists", + "equivalence", + "validate_chain" + ] + } +] diff --git a/skills/carve-changesets/scripts/evals/fixture_executor.py b/skills/carve-changesets/scripts/evals/fixture_executor.py new file mode 100644 index 0000000..690f46f --- /dev/null +++ b/skills/carve-changesets/scripts/evals/fixture_executor.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Deterministic fresh-process stand-in for a compatible agent runtime.""" + +from __future__ import annotations + +import json +import os +import re +import sys + +ACTION_VOCABULARY = ( + "accept_mechanical_exception", + "diagnose_dirty_tree", + "diagnose_source_behind_base", + "document_exception_evidence", + "escalate_material_decision", + "preserve_validated_order", + "publish_without_merge", + "refuse_dirty_source", + "refuse_reorder_or_renumber", + "refuse_source_behind_base", + "request_guardrail_decision", + "separate_rename_from_behavior", + "split_by_subsystem", + "stop_for_oversized_changeset", + "stop_on_plan_conflict", + "withhold_merge", +) + + +def compact(value: str) -> str: + return re.sub(r"\s+", " ", value).strip().lower() + + +def action_result(payload: dict) -> dict: + target = payload.get("target_skill") + contract = compact( + payload.get("skill_prompt", "") + + "\n" + + "\n".join((payload.get("contract_documents") or {}).values()) + ) + required_contract = ( + "independently reviewable", + "mechanical refactors", + "not silently reordered or renumbered", + "publish authority does not permit merging", + "source is behind the base", + ) + if target != "carve-changesets" or not all( + fragment in contract for fragment in required_contract + ): + return { + "target_skill": target, + "terminal_state": "blocked", + "actions": [], + } + + request = compact(payload.get("request", "")) + scenario = payload.get("scenario") or {} + state = compact(" ".join(str(value) for value in scenario.values())) + actions: list[str] + terminal_state: str + + if "dirty" in state: + terminal_state = "blocked" + actions = ["diagnose_dirty_tree", "refuse_dirty_source"] + elif "behind base" in state and "two-part override" not in state: + terminal_state = "blocked" + actions = ["diagnose_source_behind_base", "refuse_source_behind_base"] + elif "validated plan" in state and "conflicts" in state: + terminal_state = "blocked" + actions = ["escalate_material_decision", "stop_on_plan_conflict"] + elif "validated changesets" in state and ( + "renumber" in request or "reorder" in request + ): + terminal_state = "blocked" + actions = ["preserve_validated_order", "refuse_reorder_or_renumber"] + elif "publish authority" in state and "merge authority is withheld" in state: + terminal_state = "prs_open" + actions = ["publish_without_merge", "withhold_merge"] + elif "explicit mechanical exception" in state: + terminal_state = "plan_ready" + actions = ["accept_mechanical_exception", "document_exception_evidence"] + elif "oversized" in state: + terminal_state = "blocked" + actions = ["request_guardrail_decision", "stop_for_oversized_changeset"] + elif "rename-only" in state: + terminal_state = "plan_ready" + actions = ["separate_rename_from_behavior"] + elif "independent subsystems" in state: + terminal_state = "plan_ready" + actions = ["split_by_subsystem"] + else: + terminal_state = "blocked" + actions = [] + + return { + "target_skill": target, + "terminal_state": terminal_state, + "actions": sorted(set(actions)), + } + + +def main() -> int: + payload = json.load(sys.stdin) + result = action_result(payload) + result["executor_pid"] = os.getpid() + json.dump(result, sys.stdout, sort_keys=True) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/carve-changesets/scripts/evals/prompts.csv b/skills/carve-changesets/scripts/evals/prompts.csv deleted file mode 100644 index 0dda0ca..0000000 --- a/skills/carve-changesets/scripts/evals/prompts.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,prompt -chain-basic,"You are in a git repo with base branch main, source branch feature/test, and a validated carve-changesets plan at .carve-changesets/plan.json. Using the carve-changesets skill, invoke its scripts/cli.py subcommands to run preflight, create the changeset chain, validate mergeability with tests using python3 -c \"print('ok')\", and compare the fully merged chain to the source branch. Do not modify the source branch." -chain-compare,"Follow the carve-changesets skill strictly. The plan is already validated at .carve-changesets/plan.json for base main and source feature/test. Invoke its scripts/cli.py subcommands to create the changeset chain, run validate-chain with python3 -c \"print('ok')\", and run compare. Keep the source branch immutable." diff --git a/skills/carve-changesets/scripts/evals/runner.py b/skills/carve-changesets/scripts/evals/runner.py old mode 100755 new mode 100644 index 69a54bb..0eb4ef5 --- a/skills/carve-changesets/scripts/evals/runner.py +++ b/skills/carve-changesets/scripts/evals/runner.py @@ -1,22 +1,23 @@ #!/usr/bin/env python3 -"""Local eval runner that optionally invokes codex exec, then grades deterministically.""" +"""Run result-blind forward evals or the deterministic integration self-test.""" from __future__ import annotations import argparse -import csv import json import os -import shutil +import shlex import subprocess import sys from pathlib import Path -from typing import Dict, List THIS_DIR = Path(__file__).resolve().parent -SCRIPTS_DIR = THIS_DIR.parents[0] -DEFAULT_PROMPTS_PATH = THIS_DIR / "prompts.csv" -DEFAULT_OUT_DIR = THIS_DIR / "out" +SCRIPTS_DIR = THIS_DIR.parent +SKILL_ROOT = SCRIPTS_DIR.parent +DEFAULT_CASES = SKILL_ROOT / "evals" / "cases.json" +DEFAULT_EXPECTATIONS = SKILL_ROOT / "evals" / "expectations.json" +DEFAULT_INTEGRATION_CASES = SKILL_ROOT / "evals" / "integration_cases.json" +DEFAULT_EXECUTOR = THIS_DIR / "fixture_executor.py" if str(SCRIPTS_DIR) not in sys.path: sys.path.insert(0, str(SCRIPTS_DIR)) @@ -26,146 +27,195 @@ from evals.helpers import cleanup_repo, init_eval_repo # noqa: E402 -def codex_available(codex_bin: str) -> bool: - return shutil.which(codex_bin) is not None +def load_json(path: Path): + return json.loads(path.read_text()) -def run_codex(prompt: str, *, codex_bin: str, cwd: Path) -> subprocess.CompletedProcess: - cmd = [codex_bin, "exec", "--full-auto", "--json", prompt] - return subprocess.run(cmd, cwd=cwd, text=True, capture_output=True, check=False) +def skill_prompt() -> str: + return (SKILL_ROOT / "SKILL.md").read_text() -def load_prompts(path: Path) -> List[Dict[str, str]]: - with path.open(newline="") as f: - reader = csv.DictReader(f) - return [row for row in reader] +def contract_documents() -> dict[str, str]: + return { + "SPEC.md": (SKILL_ROOT / "references" / "SPEC.md").read_text(), + "suite-handoffs.md": ( + SKILL_ROOT / "references" / "suite-handoffs.md" + ).read_text(), + } + + +def build_payload(case: dict) -> dict: + """Build one raw scenario packet without fixture identity or grader data.""" + scenario = { + key: value for key, value in case.items() if key not in {"id", "request"} + } + return { + "target_skill": "carve-changesets", + "skill_prompt": skill_prompt(), + "contract_documents": contract_documents(), + "request": case["request"], + "scenario": scenario, + } -def run_eval_case( - *, - case_id: str, - prompt: str, - codex_bin: str, - skip_codex: bool, - test_cmd: str, - out_dir: Path, -) -> Dict: - repo_dir, plan, source_hash = init_eval_repo() - plan_path = repo_dir / DEFAULT_PLAN_PATH - codex_result: Dict[str, str] = {} +def run_executor(command: list[str], payload: dict) -> dict: + completed = subprocess.run( + command, + input=json.dumps(payload), + text=True, + capture_output=True, + check=False, + ) + if completed.returncode: + raise RuntimeError( + f"executor exited {completed.returncode}: {completed.stderr.strip()}" + ) + try: + observed = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise RuntimeError("executor did not return one JSON result") from error + if not isinstance(observed, dict): + raise RuntimeError("executor did not return one JSON object") + return observed + + +def grade_forward(case_id: str, observed: dict, expected: dict) -> list[str]: + failures: list[str] = [] + if observed.get("terminal_state") != expected.get("terminal_state"): + failures.append( + f"terminal_state: expected {expected.get('terminal_state')!r}, " + f"got {observed.get('terminal_state')!r}" + ) + if observed.get("target_skill") != "carve-changesets": + failures.append( + "target_skill: expected 'carve-changesets', " + f"got {observed.get('target_skill')!r}" + ) + observed_actions = set(observed.get("actions") or []) + missing = sorted(set(expected.get("required_actions") or []) - observed_actions) + if missing: + failures.append(f"missing actions: {', '.join(missing)}") + forbidden = sorted(set(expected.get("forbidden_actions") or []) & observed_actions) + if forbidden: + failures.append(f"forbidden actions: {', '.join(forbidden)}") + return [f"{case_id}: {failure}" for failure in failures] + + +def evaluate_forward( + cases_path: Path, expectations_path: Path, command: list[str] +) -> tuple[dict[str, dict], list[str]]: + cases = load_json(cases_path) + expectations = {item["case_id"]: item for item in load_json(expectations_path)} + if {case["id"] for case in cases} != set(expectations): + raise ValueError("forward case and expectation IDs differ") + + observations: dict[str, dict] = {} + failures: list[str] = [] + for case in cases: + observed = run_executor(command, build_payload(case)) + observations[case["id"]] = observed + failures.extend(grade_forward(case["id"], observed, expectations[case["id"]])) + return observations, failures + + +def run_integration_case(case: dict, *, test_cmd: str) -> dict: + repo_dir, _plan, source_hash = init_eval_repo() original_cwd = Path.cwd() try: os.chdir(repo_dir) - if not skip_codex: - if not codex_available(codex_bin): - raise RuntimeError(f"codex binary not found on PATH: {codex_bin}") - result = run_codex(prompt, codex_bin=codex_bin, cwd=repo_dir) - codex_result = { - "returncode": str(result.returncode), - "stdout": result.stdout, - "stderr": result.stderr, - } - else: - codex_result = {"skipped": "true"} - grade: GradeResult = grade_repo( - plan_path=plan_path, + plan_path=repo_dir / DEFAULT_PLAN_PATH, expected_source_hash=source_hash, test_cmd=test_cmd, - auto_create_chain=skip_codex, + auto_create_chain=bool(case.get("auto_create_chain", True)), ) - - case_out = { - "id": case_id, - "ok": grade.ok, + expected_checks = set(case.get("objective_checks") or []) + missing_checks = sorted(expected_checks - set(grade.checks)) + failures = list(grade.failures) + if missing_checks: + failures.append("missing objective checks: " + ", ".join(missing_checks)) + return { + "id": case["id"], + "ok": grade.ok and not missing_checks, "checks": grade.checks, - "failures": grade.failures, - "codex": codex_result, + "failures": failures, } - return case_out finally: os.chdir(original_cwd) cleanup_repo(repo_dir) +def evaluate_integration(cases_path: Path, *, test_cmd: str) -> dict[str, dict]: + return { + case["id"]: run_integration_case(case, test_cmd=test_cmd) + for case in load_json(cases_path) + } + + +def write_outputs(output_dir: Path | None, results: dict[str, dict]) -> None: + if output_dir is None: + return + output_dir.mkdir(parents=True, exist_ok=True) + for case_id, result in results.items(): + (output_dir / f"{case_id}.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n" + ) + + def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Run local evals for carve-changesets." - ) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cases", type=Path, default=DEFAULT_CASES) + parser.add_argument("--expectations", type=Path, default=DEFAULT_EXPECTATIONS) parser.add_argument( - "--prompts", - default=str(DEFAULT_PROMPTS_PATH), - help="Prompts CSV path", + "--executor", + default=f"{shlex.quote(sys.executable)} {shlex.quote(str(DEFAULT_EXECUTOR))}", + help="Fresh-process evaluator command; receives result-blind JSON on stdin", ) parser.add_argument( - "--out-dir", - default=str(DEFAULT_OUT_DIR), - help="Directory to write eval results", + "--integration-self-test", + action="store_true", + help="Run only the deterministic helper integration self-test", ) - parser.add_argument("--codex-bin", default="codex", help="codex executable name") parser.add_argument( - "--skip-codex", - action="store_true", - help="Skip invoking codex and grade a deterministic baseline instead.", + "--integration-cases", type=Path, default=DEFAULT_INTEGRATION_CASES ) parser.add_argument( "--test-cmd", default="python3 -c \"print('ok')\"", - help="Test command used by the grader's validate-chain step.", + help="Approved command used by the objective chain grader", ) + parser.add_argument("--output-dir", type=Path) return parser -def main(argv: List[str] | None = None) -> int: - parser = build_parser() - args = parser.parse_args(argv) - - prompts_path = Path(args.prompts) - out_dir = Path(args.out_dir) - out_dir.mkdir(parents=True, exist_ok=True) - - cases = load_prompts(prompts_path) - results: List[Dict] = [] - - for row in cases: - case_id = row.get("id", "case") - prompt = row.get("prompt", "").strip() - if not prompt: - results.append( - {"id": case_id, "ok": False, "failures": ["empty prompt"], "checks": []} - ) - continue - try: - result = run_eval_case( - case_id=case_id, - prompt=prompt, - codex_bin=args.codex_bin, - skip_codex=args.skip_codex, - test_cmd=args.test_cmd, - out_dir=out_dir, - ) - except Exception as exc: # defensive guard for eval runs - result = { - "id": case_id, - "ok": False, - "checks": [], - "failures": [f"exception: {exc}"], - } - results.append(result) - - (out_dir / f"{case_id}.json").write_text(json.dumps(result, indent=2) + "\n") +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.integration_self_test: + results = evaluate_integration(args.integration_cases, test_cmd=args.test_cmd) + failures = [ + f"{case_id}: {failure}" + for case_id, result in results.items() + for failure in result["failures"] + ] + mode = "integration_self_test" + else: + results, failures = evaluate_forward( + args.cases, args.expectations, shlex.split(args.executor) + ) + mode = "forward" + write_outputs(args.output_dir, results) + failed_ids = {item.split(":", 1)[0] for item in failures} summary = { + "mode": mode, "total": len(results), - "passed": sum(1 for r in results if r.get("ok")), - "failed": sum(1 for r in results if not r.get("ok")), - "results": results, + "passed": len(results) - len(failed_ids), + "failed": len(failed_ids), + "failures": failures, } - (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") - - print(json.dumps(summary, indent=2)) - return 0 if summary["failed"] == 0 else 1 + print(json.dumps(summary, indent=2, sort_keys=True)) + return 1 if failures else 0 if __name__ == "__main__": diff --git a/skills/carve-changesets/scripts/tests/test_cli_safety.py b/skills/carve-changesets/scripts/tests/test_cli_safety.py index f3bc2ba..6cad4d7 100644 --- a/skills/carve-changesets/scripts/tests/test_cli_safety.py +++ b/skills/carve-changesets/scripts/tests/test_cli_safety.py @@ -16,7 +16,7 @@ def test_every_operation_has_one_mutation_class(self) -> None: self.assertIn(command, help_text) self.assertIn(f"[{mutation_class}]", help_text) - def test_all_remote_mutations_default_to_dry_run(self) -> None: + def test_issue_30_all_remote_mutations_default_to_dry_run(self) -> None: parser = build_parser() for argv in ( ("pr-create",), @@ -28,7 +28,7 @@ def test_all_remote_mutations_default_to_dry_run(self) -> None: self.assertEqual("remote-mutating", args.mutation_class) self.assertTrue(args.dry_run) - def test_implementation_uses_file_messages_and_no_hard_reset(self) -> None: + def test_issue_30_uses_file_messages_and_never_hard_resets(self) -> None: scripts = Path(__file__).resolve().parents[1] implementation = "\n".join( path.read_text() @@ -40,7 +40,7 @@ def test_implementation_uses_file_messages_and_no_hard_reset(self) -> None: self.assertNotIn('"--body",', implementation) self.assertNotIn('"reset", "--hard"', implementation) - def test_only_github_module_invokes_gh(self) -> None: + def test_issue_30_only_github_chokepoint_invokes_gh(self) -> None: scripts = Path(__file__).resolve().parents[1] for path in scripts.glob("*.py"): if path.name == "github.py": diff --git a/skills/carve-changesets/scripts/tests/test_evals.py b/skills/carve-changesets/scripts/tests/test_evals.py index d3a59a0..42fb8b6 100644 --- a/skills/carve-changesets/scripts/tests/test_evals.py +++ b/skills/carve-changesets/scripts/tests/test_evals.py @@ -1,90 +1,188 @@ from __future__ import annotations +import importlib.util +import json import shutil +import sys import tempfile import unittest from pathlib import Path -from evals.grader import grade_repo -from evals.helpers import cleanup_repo, init_eval_repo -from evals.runner import main as runner_main -from legacy_helpers import chdir, commit, run +SCRIPTS_DIR = Path(__file__).resolve().parents[1] +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from evals.grader import grade_repo # noqa: E402 +from evals.helpers import cleanup_repo, init_eval_repo # noqa: E402 +from legacy_helpers import chdir, commit, run # noqa: E402 + +SKILL_ROOT = Path(__file__).resolve().parents[2] +RUNNER_PATH = SKILL_ROOT / "scripts" / "evals" / "runner.py" +EXECUTOR_PATH = SKILL_ROOT / "scripts" / "evals" / "fixture_executor.py" + +RUNNER_SPEC = importlib.util.spec_from_file_location( + "carve_changesets_eval_runner", RUNNER_PATH +) +RUNNER = importlib.util.module_from_spec(RUNNER_SPEC) +assert RUNNER_SPEC and RUNNER_SPEC.loader +RUNNER_SPEC.loader.exec_module(RUNNER) + +EXECUTOR_SPEC = importlib.util.spec_from_file_location( + "carve_changesets_fixture_executor", EXECUTOR_PATH +) +EXECUTOR = importlib.util.module_from_spec(EXECUTOR_SPEC) +assert EXECUTOR_SPEC and EXECUTOR_SPEC.loader +EXECUTOR_SPEC.loader.exec_module(EXECUTOR) class EvalGraderTests(unittest.TestCase): - def test_grader_passes_on_deterministic_baseline(self) -> None: - repo_dir, plan, source_hash = init_eval_repo() + def test_objective_grader_passes_the_integration_fixture(self) -> None: + repo_dir, _plan, source_hash = init_eval_repo() try: - plan_path = repo_dir / ".carve-changesets/plan.json" with chdir(repo_dir): result = grade_repo( - plan_path=plan_path, + plan_path=repo_dir / ".carve-changesets/plan.json", expected_source_hash=source_hash, test_cmd="python3 -c \"print('ok')\"", auto_create_chain=True, ) self.assertTrue(result.ok, f"grader should pass: {result.failures}") + self.assertIn("equivalence", result.checks) + self.assertIn("source_hash_unchanged", result.checks) finally: cleanup_repo(repo_dir) - def test_grader_detects_source_branch_mutation(self) -> None: - repo_dir, plan, source_hash = init_eval_repo() + def test_objective_grader_detects_source_branch_mutation(self) -> None: + repo_dir, _plan, source_hash = init_eval_repo() try: - plan_path = repo_dir / ".carve-changesets/plan.json" with chdir(repo_dir): - # Mutate the source branch after recording its hash. (repo_dir / "a.txt").write_text("mutated-source\n") run(["git", "add", "a.txt"], cwd=repo_dir) commit(repo_dir, "mutate source") - result = grade_repo( - plan_path=plan_path, + plan_path=repo_dir / ".carve-changesets/plan.json", expected_source_hash=source_hash, test_cmd="python3 -c \"print('ok')\"", auto_create_chain=True, ) - self.assertFalse(result.ok) self.assertTrue( - any("source_hash_unchanged" in failure for failure in result.failures), - f"expected source hash failure, got: {result.failures}", + any("source_hash_unchanged" in item for item in result.failures) ) finally: cleanup_repo(repo_dir) -class EvalRunnerTests(unittest.TestCase): - def test_prompts_use_only_the_consolidated_carve_interface(self) -> None: - prompts_path = Path(__file__).resolve().parents[1] / "evals" / "prompts.csv" - prompts = prompts_path.read_text() - self.assertIn("carve-changesets skill", prompts) - self.assertIn("scripts/cli.py", prompts) - self.assertIn(".carve-changesets/plan.json", prompts) - for stale in ( - "prepare" + "-changesets", - ".prepare" + "-changesets", - "scripts/" + "*.py", - ): - self.assertNotIn(stale, prompts) - - def test_runner_skip_codex_produces_passing_summary(self) -> None: - prompts_path = Path(__file__).resolve().parents[1] / "evals" / "prompts.csv" - out_dir = Path(tempfile.mkdtemp(prefix="pcs-eval-out-")) - try: - rc = runner_main( - [ - "--prompts", - str(prompts_path), - "--out-dir", - str(out_dir), - "--skip-codex", - ] +class ForwardEvaluationTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.cases = json.loads(RUNNER.DEFAULT_CASES.read_text()) + cls.expectations_text = RUNNER.DEFAULT_EXPECTATIONS.read_text() + + def test_cases_match_peer_shape_and_cover_required_judgment(self) -> None: + peer_shape = { + "id", + "request", + "tracker_state", + "authority", + "runtime_profile", + "capabilities", + "candidate_state", + "incoming_handoff", + } + expected_ids = { + "decompose-independent-subsystems", + "separate-mechanical-rename", + "oversized-guardrail-negotiation", + "mechanical-guardrail-exception", + "refuse-validated-reorder", + "validated-plan-conflict", + "publish-with-merge-withheld", + "refuse-dirty-source", + "refuse-source-behind-base", + } + self.assertEqual(expected_ids, {case["id"] for case in self.cases}) + for case in self.cases: + self.assertIn("request", case) + self.assertIn("candidate_state", case) + self.assertLessEqual(set(case), peer_shape, case["id"]) + + def test_executor_payload_is_result_blind(self) -> None: + for case in self.cases: + payload = RUNNER.build_payload(case) + serialized = json.dumps(payload, sort_keys=True) + self.assertNotIn(case["id"], serialized) + self.assertNotIn("case_id", serialized) + self.assertNotIn("required_actions", serialized) + self.assertNotIn("forbidden_actions", serialized) + self.assertNotIn("terminal_state", serialized) + self.assertNotIn(self.expectations_text, serialized) + + def test_forward_cases_execute_in_fresh_processes(self) -> None: + observations, failures = RUNNER.evaluate_forward( + RUNNER.DEFAULT_CASES, + RUNNER.DEFAULT_EXPECTATIONS, + [sys.executable, str(EXECUTOR_PATH)], + ) + self.assertEqual([], failures) + self.assertEqual(len(self.cases), len(observations)) + self.assertEqual( + len(self.cases), + len({result["executor_pid"] for result in observations.values()}), + ) + + def test_reference_executor_uses_the_supplied_contract(self) -> None: + payload = RUNNER.build_payload(self.cases[0]) + payload["contract_documents"]["SPEC.md"] = "" + observed = RUNNER.run_executor([sys.executable, str(EXECUTOR_PATH)], payload) + self.assertEqual("blocked", observed["terminal_state"]) + self.assertEqual([], observed["actions"]) + + def test_vocabulary_spam_cannot_pass_any_case(self) -> None: + expectations = json.loads(self.expectations_text) + for expected in expectations: + observed = { + "target_skill": "carve-changesets", + "terminal_state": expected["terminal_state"], + "actions": list(EXECUTOR.ACTION_VOCABULARY), + } + failures = RUNNER.grade_forward(expected["case_id"], observed, expected) + self.assertTrue( + any("forbidden actions" in failure for failure in failures), + expected["case_id"], ) - self.assertEqual(rc, 0) - summary_path = out_dir / "summary.json" - self.assertTrue(summary_path.exists()) + + def test_integration_self_test_migrates_both_prompt_cases(self) -> None: + cases = json.loads(RUNNER.DEFAULT_INTEGRATION_CASES.read_text()) + self.assertEqual( + {"chain-basic", "chain-compare"}, {case["id"] for case in cases} + ) + results = RUNNER.evaluate_integration( + RUNNER.DEFAULT_INTEGRATION_CASES, + test_cmd="python3 -c \"print('ok')\"", + ) + self.assertTrue(all(result["ok"] for result in results.values()), results) + for result in results.values(): + self.assertIn("equivalence", result["checks"]) + self.assertIn("validate_chain", result["checks"]) + + def test_runner_has_no_agent_cli_specific_dependency(self) -> None: + runner_source = RUNNER_PATH.read_text() + parser_help = RUNNER.build_parser().format_help() + self.assertIn("--executor", parser_help) + self.assertIn("--integration-self-test", parser_help) + self.assertNotIn("--skip-" + "codex", parser_help) + self.assertNotIn("run_" + "codex", runner_source) + self.assertNotIn("codex" + "_available", runner_source) + + def test_runner_writes_results_only_when_requested(self) -> None: + output_dir = Path(tempfile.mkdtemp(prefix="carve-eval-output-")) + try: + rc = RUNNER.main(["--output-dir", str(output_dir)]) + self.assertEqual(0, rc) + self.assertEqual(len(self.cases), len(list(output_dir.glob("*.json")))) finally: - shutil.rmtree(out_dir) + shutil.rmtree(output_dir) if __name__ == "__main__": diff --git a/skills/carve-changesets/scripts/tests/test_github.py b/skills/carve-changesets/scripts/tests/test_github.py index 165f142..5ac5664 100644 --- a/skills/carve-changesets/scripts/tests/test_github.py +++ b/skills/carve-changesets/scripts/tests/test_github.py @@ -1,14 +1,29 @@ from __future__ import annotations +import json +import os import shutil import subprocess +import sys +import tempfile import unittest +from pathlib import Path from unittest import mock -import github as github_mod -from chain import create_chain -from common import CommandError -from legacy_helpers import chdir, commit, init_remote, init_repo, run +SCRIPTS_DIR = Path(__file__).resolve().parents[1] +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +import github as github_mod # noqa: E402 +from chain import create_chain # noqa: E402 +from common import CommandError # noqa: E402 +from legacy_helpers import ( # noqa: E402 + chdir, + commit, + init_remote, + init_repo, + run, +) class GithubTests(unittest.TestCase): @@ -53,7 +68,7 @@ def test_pr_merge_fences_the_exact_number_and_head(self) -> None: capture.call_args.args[0], ) - def test_pr_edit_targets_explicit_number(self) -> None: + def test_issue_33_pr_edit_targets_explicit_number(self) -> None: with ( mock.patch.object( github_mod, @@ -189,7 +204,69 @@ def test_pr_create_binds_and_verifies_exact_remote_candidate(self) -> None: if remote_dir is not None: shutil.rmtree(remote_dir.parent) - def test_pr_create_rejects_local_head_not_on_remote(self) -> None: + def test_non_dry_run_pr_create_executes_against_a_gh_stub(self) -> None: + repo_dir, plan = init_repo() + remote_dir = None + stub_dir = Path(tempfile.mkdtemp(prefix="carve-gh-stub-")) + try: + remote_dir = init_remote(repo_dir) + with chdir(repo_dir): + create_chain(plan) + run(["git", "push", "origin", "feature/test-1"], cwd=repo_dir) + head = run( + ["git", "rev-parse", "feature/test-1"], cwd=repo_dir + ).stdout.strip() + body = github_mod.pr_body_for( + plan, 1, len(plan["changesets"]), plan["changesets"][0] + ) + + stub = stub_dir / "gh" + log_path = stub_dir / "calls.jsonl" + stub.write_text( + "#!/usr/bin/env python3\n" + "import json, os, sys\n" + "with open(os.environ['GH_STUB_LOG'], 'a', encoding='utf-8') as f:\n" + " f.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "if sys.argv[1:3] == ['pr', 'view']:\n" + " print(os.environ['GH_STUB_VIEW'])\n" + ) + stub.chmod(0o755) + created = { + "number": 95, + "url": "https://github.com/acme/widgets/pull/95", + "headRefOid": head, + "baseRefName": "main", + "body": body, + } + environment = { + "PATH": f"{stub_dir}{os.pathsep}{os.environ.get('PATH', '')}", + "GH_STUB_LOG": str(log_path), + "GH_STUB_VIEW": json.dumps(created), + } + with ( + chdir(repo_dir), + mock.patch.dict(os.environ, environment), + mock.patch.object( + github_mod, + "github_repo_for_remote", + return_value="github.com/acme/widgets", + ), + ): + github_mod.pr_create(plan, indices=[1], dry_run=False, remote="origin") + + calls = [json.loads(line) for line in log_path.read_text().splitlines()] + self.assertEqual(["auth", "status"], calls[0][:2]) + create_call = next(call for call in calls if call[:2] == ["pr", "create"]) + self.assertIn("--body-file", create_call) + self.assertNotIn("--body", create_call) + self.assertTrue(any(call[:2] == ["pr", "view"] for call in calls)) + finally: + shutil.rmtree(repo_dir) + shutil.rmtree(stub_dir) + if remote_dir is not None: + shutil.rmtree(remote_dir.parent) + + def test_issue_33_pr_create_rejects_local_head_not_on_remote(self) -> None: repo_dir, plan = init_repo() remote_dir = None try: diff --git a/skills/carve-changesets/scripts/tests/test_preflight.py b/skills/carve-changesets/scripts/tests/test_preflight.py index 3bc0aec..1376a4f 100644 --- a/skills/carve-changesets/scripts/tests/test_preflight.py +++ b/skills/carve-changesets/scripts/tests/test_preflight.py @@ -119,7 +119,7 @@ def test_preflight_allows_recordkeeping_tracked_with_override(self) -> None: finally: shutil.rmtree(repo_dir) - def test_preflight_never_executes_discovered_command(self) -> None: + def test_issue_30_preflight_never_executes_discovered_command(self) -> None: repo_dir, plan = init_repo() try: marker = repo_dir / "test-command-ran" diff --git a/skills/carve-changesets/scripts/tests/test_propagate.py b/skills/carve-changesets/scripts/tests/test_propagate.py index 1b5b681..c48e931 100644 --- a/skills/carve-changesets/scripts/tests/test_propagate.py +++ b/skills/carve-changesets/scripts/tests/test_propagate.py @@ -39,7 +39,7 @@ def test_propagation_push_rejects_remote_head_moved_since_rehydration(self) -> N git_call.assert_not_called() - def test_push_chain_never_sends_base_or_source_to_force_push(self) -> None: + def test_issue_30_and_33_push_chain_never_pushes_base_or_source(self) -> None: repo_dir, plan = init_repo() remote_dir = None try: @@ -278,7 +278,9 @@ def _run_combined(self, strategy: str, *, through_cli: bool = False) -> None: [call.args[0] for call in push.call_args_list], ) - def test_cmd_merge_propagate_rehydrates_without_state_and_rebases(self) -> None: + def test_issue_33_combined_merge_then_propagate_rehydrates_and_rebases( + self, + ) -> None: self._run_combined("rebase", through_cli=True) def test_merge_propagate_supports_cherry_pick(self) -> None: diff --git a/skills/carve-changesets/scripts/tests/test_rehydrate.py b/skills/carve-changesets/scripts/tests/test_rehydrate.py index 69bff8f..b58246d 100644 --- a/skills/carve-changesets/scripts/tests/test_rehydrate.py +++ b/skills/carve-changesets/scripts/tests/test_rehydrate.py @@ -87,6 +87,47 @@ def test_rehydrates_full_chain_after_local_state_is_deleted(self) -> None: ["main", "feature/report-1"], [item.base for item in chain.changesets] ) + def test_edited_pr_prose_rehydrates_from_unchanged_metadata_block(self) -> None: + _, prs = self._materialize() + clone = self._fresh_clone() + edited = [ + PullRequestRecord( + **{ + **pr.__dict__, + "body": ( + "Reviewer-authored context.\n\n" + + pr.body.replace( + "Report API", "Improved report API explanation" + ) + ), + } + ) + for pr in prs + ] + + chain = rehydrate_chain( + source_branch="feature/report", pull_requests=edited, cwd=clone + ) + + self.assertEqual([101, 102], [item.pr_number for item in chain.changesets]) + self.assertEqual( + ["part-1", "part-2"], [item.metadata.slug for item in chain.changesets] + ) + + def test_contiguous_partial_chain_rehydrates_without_inventing_later_items( + self, + ) -> None: + heads, prs = self._materialize(indices=(1,)) + clone = self._fresh_clone() + + chain = rehydrate_chain( + source_branch="feature/report", pull_requests=prs, cwd=clone + ) + + self.assertEqual(1, len(chain.changesets)) + self.assertEqual(heads[1], chain.changesets[0].head) + self.assertEqual(1, chain.changesets[0].metadata.index) + def test_status_is_rendered_from_rehydration_without_local_files(self) -> None: _, prs = self._materialize() clone = self._fresh_clone() @@ -117,7 +158,7 @@ def test_trailers_survive_propagation_rebase(self) -> None: self.assertEqual("part-2", parsed.slug) self.assertEqual(self.source_sha, parsed.source_sha) - def test_missing_branch_index_fails_closed(self) -> None: + def test_partial_chain_with_an_index_gap_fails_closed(self) -> None: self._materialize(indices=(1, 3)) clone = self._fresh_clone() @@ -126,7 +167,7 @@ def test_missing_branch_index_fails_closed(self) -> None: source_branch="feature/report", base_branch="main", cwd=clone ) - def test_missing_commit_trailer_fails_closed(self) -> None: + def test_missing_required_commit_trailers_fail_closed(self) -> None: helpers.run(self.repo, "git", "checkout", "-b", "feature/report-1", "main") (self.repo / "plain.txt").write_text("plain\n") helpers.run(self.repo, "git", "add", "plain.txt") diff --git a/skills/carve-changesets/scripts/tests/test_validate.py b/skills/carve-changesets/scripts/tests/test_validate.py index 107e50e..65dd236 100644 --- a/skills/carve-changesets/scripts/tests/test_validate.py +++ b/skills/carve-changesets/scripts/tests/test_validate.py @@ -58,7 +58,7 @@ def _rehydrate(self): source_branch="feature/report", base_branch="main", cwd=self.repo ) - def test_legitimate_propagation_validates_without_drift_diagnostics(self) -> None: + def test_issue_32_legitimate_propagation_has_no_stale_drift_warning(self) -> None: heads = self._materialize_equivalent_chain() helpers.run(self.repo, "git", "checkout", "feature/report-2") amended_message = self._stamp(2, detail="Refresh the propagated commit.") @@ -92,7 +92,7 @@ def test_legitimate_propagation_validates_without_drift_diagnostics(self) -> Non self.assertEqual("unchanged", result.source_status) self.assertEqual((), result.diagnostics) - def test_rewritten_mid_chain_branch_breaks_live_ancestry(self) -> None: + def test_issue_32_rewritten_mid_chain_branch_breaks_live_ancestry(self) -> None: self._materialize_equivalent_chain() chain = self._rehydrate() helpers.run(self.repo, "git", "checkout", "-b", "replacement", "main") @@ -131,7 +131,7 @@ def test_non_equivalent_chain_tip_is_rejected(self) -> None: "source_equivalence_mismatch", {item.code for item in result.errors} ) - def test_advanced_source_is_distinct_from_different_source_history(self) -> None: + def test_issue_32_source_advance_is_distinct_from_history_mismatch(self) -> None: self._materialize_equivalent_chain() helpers.run(self.repo, "git", "checkout", "feature/report") (self.repo / "later.txt").write_text("later source work\n") From eb8612300d75d1483995677d75f54fe1a32b60d7 Mon Sep 17 00:00:00 2001 From: Scott Haug Date: Tue, 21 Jul 2026 19:36:50 -0700 Subject: [PATCH 2/3] fix: require the two-part source freshness override ## Summary - Require both source-behind allowance and confirmation flags before preflight proceeds - Add direct regressions for each lone flag and the valid two-flag override ## Why - Align the executable safety boundary with the published carve-changesets contract - Prevent behavioral evals from passing while real preflight accepts incomplete authority --- CHANGELOG.md | 2 + skills/carve-changesets/scripts/preflight.py | 15 +++---- .../scripts/tests/test_preflight.py | 43 ++++++++++++++++++- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb470d..5a9f35e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ summary: Chronological history of repository and skill changes. ## 2026-07-21 — Completed carve-changesets skill and verification suite +- fix: require the two-part source freshness override - test: complete the carve-changesets verification suite + (`f669d322985c435daf9b0c7296889d8a3bdd270c`) - feat: package the carve-changesets skill (`a8e19110380e048e0aaf85e820c114fe2a07cc7f`) - docs: define carve-changesets suite handoffs diff --git a/skills/carve-changesets/scripts/preflight.py b/skills/carve-changesets/scripts/preflight.py index eeaffdf..58547db 100755 --- a/skills/carve-changesets/scripts/preflight.py +++ b/skills/carve-changesets/scripts/preflight.py @@ -118,18 +118,13 @@ def preflight( "This workflow assumes the source includes the current base HEAD to avoid churn\n" "while carving changesets.\n\n" f"Fix: merge or rebase {source} onto {base}, then re-run preflight.\n" - "Override (not recommended): re-run with --allow-source-behind-base and record why in the plan." + "Override (not recommended): re-run with both --allow-source-behind-base " + "and --confirm-source-behind-base, then record why in the plan." ) - if allow_source_behind_base: + if allow_source_behind_base and confirm_source_behind_base: print( - "[WARN] Source branch is behind base branch; proceeding by explicit override." - ) - elif confirm_source_behind_base: - response = input("Source is behind base. Proceed anyway? [y/N] ").strip() - if response.lower() not in ("y", "yes"): - raise CommandError(message) - print( - "[WARN] Source branch is behind base branch; proceeding by explicit confirmation." + "[WARN] Source branch is behind base branch; proceeding by explicit " + "two-flag override." ) else: raise CommandError(message) diff --git a/skills/carve-changesets/scripts/tests/test_preflight.py b/skills/carve-changesets/scripts/tests/test_preflight.py index 1376a4f..f573d65 100644 --- a/skills/carve-changesets/scripts/tests/test_preflight.py +++ b/skills/carve-changesets/scripts/tests/test_preflight.py @@ -64,7 +64,47 @@ def test_preflight_fails_when_source_behind_base(self) -> None: finally: shutil.rmtree(repo_dir) - def test_preflight_allows_source_behind_with_override(self) -> None: + def test_source_behind_rejects_allow_flag_without_confirmation(self) -> None: + repo_dir, plan = init_repo() + try: + run(["git", "checkout", plan["base_branch"]], cwd=repo_dir) + (repo_dir / "base.txt").write_text("base-update\n") + run(["git", "add", "base.txt"], cwd=repo_dir) + commit(repo_dir, "base update") + with chdir(repo_dir): + with self.assertRaises(CommandError): + preflight_mod.preflight( + base=plan["base_branch"], + source=plan["source_branch"], + test_cmd="", + skip_tests=True, + skip_merge_check=True, + allow_source_behind_base=True, + ) + finally: + shutil.rmtree(repo_dir) + + def test_source_behind_rejects_confirmation_flag_without_allowance(self) -> None: + repo_dir, plan = init_repo() + try: + run(["git", "checkout", plan["base_branch"]], cwd=repo_dir) + (repo_dir / "base.txt").write_text("base-update\n") + run(["git", "add", "base.txt"], cwd=repo_dir) + commit(repo_dir, "base update") + with chdir(repo_dir): + with self.assertRaises(CommandError): + preflight_mod.preflight( + base=plan["base_branch"], + source=plan["source_branch"], + test_cmd="", + skip_tests=True, + skip_merge_check=True, + confirm_source_behind_base=True, + ) + finally: + shutil.rmtree(repo_dir) + + def test_source_behind_allows_two_flag_override(self) -> None: repo_dir, plan = init_repo() try: run(["git", "checkout", plan["base_branch"]], cwd=repo_dir) @@ -79,6 +119,7 @@ def test_preflight_allows_source_behind_with_override(self) -> None: skip_tests=True, skip_merge_check=True, allow_source_behind_base=True, + confirm_source_behind_base=True, ) finally: shutil.rmtree(repo_dir) From e30b5f1021538d673eb931b2978287cfd21ae4ae Mon Sep 17 00:00:00 2001 From: Scott Haug Date: Tue, 21 Jul 2026 20:39:29 -0700 Subject: [PATCH 3/3] refactor: derive the eval action vocabulary from expectations ## Summary - Remove the duplicated action vocabulary from the deterministic fixture executor - Derive the anti-spam test input from required and forbidden expectation actions ## Why - Keep one authoritative representation of the behavioral evaluation vocabulary - Ensure the adversarial grading check expands automatically as expectations evolve --- CHANGELOG.md | 2 ++ .../scripts/evals/fixture_executor.py | 19 ------------------- .../scripts/tests/test_evals.py | 10 +++++++++- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a9f35e..4085241 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ summary: Chronological history of repository and skill changes. ## 2026-07-21 — Completed carve-changesets skill and verification suite +- refactor: derive the eval action vocabulary from expectations - fix: require the two-part source freshness override + (`eb8612300d75d1483995677d75f54fe1a32b60d7`) - test: complete the carve-changesets verification suite (`f669d322985c435daf9b0c7296889d8a3bdd270c`) - feat: package the carve-changesets skill diff --git a/skills/carve-changesets/scripts/evals/fixture_executor.py b/skills/carve-changesets/scripts/evals/fixture_executor.py index 690f46f..7e5d687 100644 --- a/skills/carve-changesets/scripts/evals/fixture_executor.py +++ b/skills/carve-changesets/scripts/evals/fixture_executor.py @@ -8,25 +8,6 @@ import re import sys -ACTION_VOCABULARY = ( - "accept_mechanical_exception", - "diagnose_dirty_tree", - "diagnose_source_behind_base", - "document_exception_evidence", - "escalate_material_decision", - "preserve_validated_order", - "publish_without_merge", - "refuse_dirty_source", - "refuse_reorder_or_renumber", - "refuse_source_behind_base", - "request_guardrail_decision", - "separate_rename_from_behavior", - "split_by_subsystem", - "stop_for_oversized_changeset", - "stop_on_plan_conflict", - "withhold_merge", -) - def compact(value: str) -> str: return re.sub(r"\s+", " ", value).strip().lower() diff --git a/skills/carve-changesets/scripts/tests/test_evals.py b/skills/carve-changesets/scripts/tests/test_evals.py index 42fb8b6..043237a 100644 --- a/skills/carve-changesets/scripts/tests/test_evals.py +++ b/skills/carve-changesets/scripts/tests/test_evals.py @@ -140,11 +140,19 @@ def test_reference_executor_uses_the_supplied_contract(self) -> None: def test_vocabulary_spam_cannot_pass_any_case(self) -> None: expectations = json.loads(self.expectations_text) + action_vocabulary = sorted( + { + action + for expected in expectations + for field in ("required_actions", "forbidden_actions") + for action in expected.get(field, []) + } + ) for expected in expectations: observed = { "target_skill": "carve-changesets", "terminal_state": expected["terminal_state"], - "actions": list(EXECUTOR.ACTION_VOCABULARY), + "actions": action_vocabulary, } failures = RUNNER.grade_forward(expected["case_id"], observed, expected) self.assertTrue(