From bccfac954aca63e663927806861e7d8a294e5934 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Thu, 9 Jul 2026 19:27:42 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20release-ci=20readiness=20profile=20?= =?UTF-8?q?=E2=80=94=20the=20nightly=20gate's=20CI=20evidence=20scope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of PyAutoLabs/PyAutoBuild#127 (design §5): `pyauto-heart readiness --profile release-ci` evaluates the same gates over a CI-assembled snapshot; gaps in dev-box-local evidence (the local test_run cache and the never-observed slices) are reported as na_reasons — listed, never silently green, non-gating — while present adverse evidence and the required set (library CI, the release-validation report, verify_install) gate exactly as before. Default profile verdict unchanged; profile verdicts are a live compute and never persist to release_ready.json. Also fixes the inner `profile` rebinding in the validation gate (now vr_profile). Co-Authored-By: Claude Fable 5 --- heart/readiness.py | 86 +++++++++++++++++++++++++++++++++----- tests/test_readiness.py | 92 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 10 deletions(-) diff --git a/heart/readiness.py b/heart/readiness.py index d6aff54..d895a48 100644 --- a/heart/readiness.py +++ b/heart/readiness.py @@ -39,7 +39,23 @@ Consumer semantics: the **release gate needs GREEN** — STALE blocks a release exactly like YELLOW, but prescribes refreshing evidence rather than a human -acknowledgement. The **dev-ship gate** (PyAutoBrain ``AUTONOMY.md`` leg 4) +acknowledgement. + +**Profiles** (``--profile``): ``default`` is the dev-box verdict described +above, byte-for-byte unchanged. ``release-ci`` is the scheduled-nightly +release gate (``PyAutoBuild/docs/nightly_release_design.md`` §5): the same +gates evaluated over a snapshot the nightly driver assembles **in CI**, where +dev-box-local evidence (``repo_state``, ``version_skew``, ``script_timing``, +the local ``test_run`` cache, ``manifest_drift``, ``profiling_drift``) does +not exist. Under ``release-ci`` a *gap* in that local-only evidence is +reported in ``na_reasons`` ("out of scope", listed, never silently green) +instead of blocking as stale — while any local evidence actually *present* +still counts in full, and gaps in the **required** evidence set (library +``ci_status``, the release-validation report, ``verify_install``) block +exactly as they always did. The workspace script surface is not weakened by +scoping out the local ``test_run`` cache: the release-validation report *is* +that surface at release fidelity, and required-workflow conclusions gate the +workspaces' ``main``. The **dev-ship gate** (PyAutoBrain ``AUTONOMY.md`` leg 4) treats STALE as passing: an evidence gap is organism-scope, not branch-scope, and the other three legs gate the branch itself. This is what makes GREEN reachable in steady state and keeps YELLOW meaning "something is actually @@ -186,6 +202,7 @@ def compute( snapshot: dict | None, libraries: Sequence[str] | None = None, required_workflows: dict[str, list[str]] | None = None, + profile: str = "default", ) -> dict[str, Any]: """Pure verdict function. Never raises on missing/partial data.""" snapshot = snapshot or {} @@ -193,6 +210,7 @@ def compute( req_wf = required_workflows if required_workflows is not None else load_required_workflows() repos = snapshot.get("repos", {}) or {} ref = _parse_ts(snapshot.get("ts")) or datetime.datetime.now(datetime.timezone.utc) + release_ci = profile == "release-ci" red: list[str] = [] yellow: list[str] = [] @@ -200,11 +218,25 @@ def compute( # remedy is re-running a check. Never receives a reason whose last known # result was adverse (those stay red/yellow); see the module docstring. stale: list[str] = [] + # Out-of-scope-by-profile: dev-box-local evidence gaps under release-ci. + # Listed for transparency (never silently green) but non-gating — the + # module docstring's "Profiles" section is the rule's home. + na: list[str] = [] counts: dict[str, int] = {} def hit(key: str, n: int = 1) -> None: counts[key] = counts.get(key, 0) + n + def scope_local(msg: str, key: str) -> None: + """File a *gap* in dev-box-local evidence: stale on the default + profile; out-of-scope (na, non-gating) under release-ci. Adverse + local evidence never routes through here — it stays red/yellow.""" + if release_ci: + na.append(msg + " (dev-box-local — out of scope for release-ci)") + else: + stale.append(msg) + hit(key) + # --- library gates (RED) --- for lib in libs: body = repos.get(lib) @@ -279,19 +311,16 @@ def hit(key: str, n: int = 1) -> None: # the last known result was good. age = _age_days(test_run.get("ts"), ref) if age is not None and age > TEST_STALE_DAYS: - stale.append(f"test run stale ({int(age)}d old)") - hit("test_stale") + scope_local(f"test run stale ({int(age)}d old)", "test_stale") else: - stale.append("test run status unknown") - hit("test_unknown") + scope_local("test run status unknown", "test_unknown") # parked staleness (YELLOW) parked = _as_int(test_run.get("parked_stale_count", 0)) if parked > 0: yellow.append(f"{parked} stale parked script(s)") hit("parked") else: - stale.append("test run status unknown (no report.json)") - hit("test_unknown") + scope_local("test run status unknown (no report.json)", "test_unknown") # --- version skew (RED ahead / YELLOW behind) --- skew = snapshot.get("version_skew") @@ -396,7 +425,7 @@ def hit(key: str, n: int = 1) -> None: confirmed += 1 else: mismatched.append(lib) - profile = str(vr.get("profile") or "").strip().lower() + vr_profile = str(vr.get("profile") or "").strip().lower() if not commit_shas: stale.append("release validation source unconfirmed (no commit_shas)") hit("validation_unknown") @@ -418,7 +447,7 @@ def hit(key: str, n: int = 1) -> None: "current HEAD: " + ", ".join(unconfirmed) + ")" ) hit("validation_unknown") - elif profile != "release": + elif vr_profile != "release": stale.append( f"release validation profile '{vr.get('profile') or '?'}' is not 'release'" ) @@ -478,6 +507,25 @@ def hit(key: str, n: int = 1) -> None: yellow.append(f"{name}: open PR {_as_int(pr.get('max_age_days'))}d old") hit("open_pr") + # --- release-ci: name the dev-box-local evidence this snapshot never + # carried, so "out of scope" is a stated fact, not an assumption. Slices + # that ARE present were evaluated in full by the gates above. + if release_ci: + unobserved = [ + k for k in ("version_skew", "script_timing", "manifest_drift", "profiling_drift") + if not snapshot.get(k) + ] + if not any( + isinstance(repos.get(lib), dict) and (repos.get(lib) or {}).get("repo_state") + for lib in libs + ): + unobserved.insert(0, "repo_state") + if unobserved: + na.append( + "not observed in this snapshot (dev-box-local — out of scope " + "for release-ci): " + ", ".join(unobserved) + ) + # --- score --- score = 100 for key, n in counts.items(): @@ -489,10 +537,12 @@ def hit(key: str, n: int = 1) -> None: return { "verdict": verdict, "score": score, + "profile": profile, "reasons": red + yellow + stale, "red_reasons": red, "yellow_reasons": yellow, "stale_reasons": stale, + "na_reasons": na, "ts": snapshot.get("ts") or datetime.datetime.now(datetime.timezone.utc).isoformat(), } @@ -530,17 +580,33 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--json", action="store_true", help="print the raw verdict JSON") ap.add_argument("--quiet", action="store_true", help="verdict line + top reason only") ap.add_argument("--no-color", action="store_true", help="disable ANSI colors") + ap.add_argument( + "--profile", + choices=["default", "release-ci"], + default="default", + help="evidence profile: 'release-ci' scopes dev-box-local gaps to " + "na_reasons for the scheduled-nightly gate (design §5); default " + "is unchanged dev-box behaviour", + ) ns = ap.parse_args(argv) if ns.no_color: os.environ["NO_COLOR"] = "1" - verdict = load_verdict() + if ns.profile != "default": + # Profile verdicts are always a live compute and are never persisted: + # release_ready.json stays the default-profile artifact its existing + # consumers (dashboard/status) expect. + verdict = compute(state.load() or {}, profile=ns.profile) + else: + verdict = load_verdict() if ns.json: json.dump(verdict, sys.stdout, indent=2, sort_keys=True) sys.stdout.write("\n") return 0 for line in render_block(verdict, quiet=ns.quiet): print(line) + for line in verdict.get("na_reasons") or []: + print(f" n/a {line}") return 0 diff --git a/tests/test_readiness.py b/tests/test_readiness.py index 76a28d7..7049e70 100644 --- a/tests/test_readiness.py +++ b/tests/test_readiness.py @@ -625,3 +625,95 @@ def test_manifest_drift_unavailable_stays_green(): snap = make_snapshot(manifest_drift={"available": False, "reason": "missing", "checks": {}}) v = compute(snap) assert v["verdict"] == "green" + + +# --------------------------------------------------------------------------- +# release-ci profile (the scheduled-nightly gate — design §5 in +# PyAutoBuild/docs/nightly_release_design.md). The driver assembles a snapshot +# in CI with no dev box: cloud-refreshable evidence only. +# --------------------------------------------------------------------------- + +def make_ci_snapshot(**overrides) -> dict: + """What the nightly driver assembles in CI: library ci_status (no + repo_state), the release-validation report, verify_install — none of the + dev-box-local slices (test_run cache, version_skew, script_timing, ...).""" + snap = { + "ts": "2026-06-01T00:00:00+00:00", + "repos": { + lib: {"ci_status": {"conclusion": "success", "head_sha": SHAS[lib]}} + for lib in LIBS + }, + "verify_install": {"ready": True, "ts": "2026-06-01T00:00:00+00:00", + "version": "2026.6.1.1", "checks": []}, + "validation_report": _green_validation_report(), + } + snap.update(overrides) + return snap + + +def compute_ci(snap): + return readiness.compute(snap, libraries=LIBS, profile="release-ci") + + +def test_release_ci_local_gaps_are_na_not_blocking(): + # The same snapshot is STALE on the default profile (missing local test_run + # cache) but GREEN under release-ci — the gap is scoped out, loudly. + snap = make_ci_snapshot() + assert readiness.compute(snap, libraries=LIBS)["verdict"] == "stale" + v = compute_ci(snap) + assert v["verdict"] == "green" + assert v["profile"] == "release-ci" + assert any("test run" in r for r in v["na_reasons"]) + assert any("not observed in this snapshot" in r for r in v["na_reasons"]) + assert v["stale_reasons"] == [] + + +def test_release_ci_missing_validation_report_still_blocks(): + snap = make_ci_snapshot() + del snap["validation_report"] + v = compute_ci(snap) + assert v["verdict"] == "stale" + assert any("no release validation" in r for r in v["stale_reasons"]) + + +def test_release_ci_missing_verify_install_still_blocks(): + snap = make_ci_snapshot() + del snap["verify_install"] + v = compute_ci(snap) + assert v["verdict"] == "stale" + assert any("install verification not run" in r for r in v["stale_reasons"]) + + +def test_release_ci_adverse_local_evidence_still_counts(): + # A present-and-failing local signal is never scoped out: adverse evidence + # stays yellow/red under every profile. + snap = make_ci_snapshot(test_run={"ready": False, "failed": 3, "run_label": "x"}) + v = compute_ci(snap) + assert v["verdict"] == "yellow" + assert any("workspace validation not passing" in r for r in v["yellow_reasons"]) + + +def test_release_ci_library_ci_failure_is_red(): + snap = make_ci_snapshot() + snap["repos"]["PyAutoLens"]["ci_status"]["conclusion"] = "failure" + v = compute_ci(snap) + assert v["verdict"] == "red" + + +def test_release_ci_stale_validation_sha_still_blocks(): + # Source moved since the rehearsal → stale under release-ci too: the + # required evidence must match the exact source about to ship. + snap = make_ci_snapshot() + snap["repos"]["PyAutoLens"]["ci_status"]["head_sha"] = "f" * 40 + v = compute_ci(snap) + assert v["verdict"] == "stale" + assert any("source moved since rehearsal" in r for r in v["stale_reasons"]) + + +def test_default_profile_output_is_unchanged_shape(): + # Default profile: verdict/reasons exactly as before; the additive keys are + # inert ("default", empty na list). + v = readiness.compute(make_snapshot(), libraries=LIBS) + assert v["verdict"] == "green" + assert v["profile"] == "default" + assert v["na_reasons"] == []