diff --git a/src/scitex_dev/_cli/_root.py b/src/scitex_dev/_cli/_root.py index d31a05df..ed4f8871 100755 --- a/src/scitex_dev/_cli/_root.py +++ b/src/scitex_dev/_cli/_root.py @@ -58,6 +58,7 @@ see_also=("{prog} docs — browse doctrine and package documentation",), ) + def _command_to_dict( cmd: click.Command, parent_ctx: click.Context | None, @@ -96,14 +97,13 @@ def _command_to_dict( out["commands"] = commands return out + def _show_recursive_help(ctx: click.Context) -> None: """Recursively show help for all commands. Honours ctx.obj['json'].""" if ctx.obj and ctx.obj.get("json"): import json as _json - tree = _command_to_dict( - ctx.command, ctx.parent, ctx.info_name or "scitex-dev" - ) + tree = _command_to_dict(ctx.command, ctx.parent, ctx.info_name or "scitex-dev") click.echo(_json.dumps(tree, indent=2)) return @@ -131,6 +131,7 @@ def _show_recursive_help(ctx: click.Context) -> None: click.echo(sub_sub_ctx.get_help()) click.echo() + def _get_version() -> str: try: from importlib.metadata import version @@ -139,6 +140,7 @@ def _get_version() -> str: except Exception: return "0.0.0-unknown" + # Disable Click's auto --help on THIS group only (parameter, not # context — does not propagate to subcommands). Then re-add --help / # -h explicitly via @click.help_option in the desired display slot so @@ -200,6 +202,7 @@ def main( if ctx.invoked_subcommand is None: click.echo(ctx.get_help()) + # ------------------------------------------------------------------- # Ecosystem commands # ------------------------------------------------------------------- @@ -229,6 +232,7 @@ def main( # individual `quality audit-*` callers must update. from .quality import _check as _cli_quality + # These sub-rules belong inside their canonical owner per the # consolidation plan. Hidden until folded in (PR-by-PR) so the public # surface is just five audit-* commands. Removed in 0.11.0. @@ -243,17 +247,20 @@ def _ecosystem_audit_docs(projects_root): """(deprecated) Splits into `audit-python-apis` (README API drift) and `audit-skills` (SKILL.md drift). Removed in 0.11.0.""" raise SystemExit(_cli_quality.audit_docs(projects_root=projects_root)) + @ecosystem_group.command("audit-scope", hidden=True) @click.option("--projects-root", default=None) def _ecosystem_audit_scope(projects_root): """(deprecated) Folds into `audit-project`. Removed in 0.11.0.""" raise SystemExit(_cli_quality.audit_scope(projects_root=projects_root)) + @ecosystem_group.command("audit-lines", hidden=True) def _ecosystem_audit_lines(): """(deprecated) Folds into `audit-project` (LOC-limits). Removed in 0.11.0.""" raise SystemExit(_cli_quality.audit_lines()) + # Umbrella-only pin freshness audit. Designed to fire from the # umbrella package's CI; on any other package it exits 0, so it's # safe to wire into a shared CI step. @@ -266,10 +273,12 @@ def _ecosystem_audit_lines(): # CLI-standardization plan). Warn phase forwards; error phase exits 2. from .._ecosystem.click_compat import deprecated_alias + @main.group("quality", hidden=True) def _quality_deprecated(): """(deprecated) Use `scitex-dev ecosystem audit-*` instead.""" + for _quality_cmd, _quality_target in ( ("audit-docs", _ecosystem_audit_docs), ("audit-scope", _ecosystem_audit_scope), @@ -301,9 +310,8 @@ def _quality_deprecated(): # ------------------------------------------------------------------- # `config` → `show-config` rename, error rung of the deprecation ladder. -deprecated_alias( - main, "config", target="show-config", remove_in="0.11", phase="error" -) +deprecated_alias(main, "config", target="show-config", remove_in="0.11", phase="error") + @main.command( "show-config", @@ -357,6 +365,7 @@ def config_cmd(as_json): else: click.echo(f" {item}") + # rename-symbols + the hidden `rename` deprecation alias live in # _cli/_rename.py. Extracted to keep _root.py under the line budget # and to give the bulk-rename surface a focused module to grow into. @@ -389,6 +398,7 @@ def config_cmd(as_json): docs_grp = docs_click_group(package="scitex-dev") main.add_command(docs_grp) + # `docs search` — canonical home for ecosystem-wide search across APIs, # CLI, MCP tools, and documentation. The legacy top-level `search-docs` # is kept as a hidden deprecation alias (see below). Removed in 0.11.0. @@ -399,15 +409,17 @@ def config_cmd(as_json): summary="Search across APIs, CLI, MCP tools, and documentation.", examples=( Example('{prog} docs search "save figure"', "Full-text search everywhere."), - Example("{prog} docs search version --scope api", "Limit to the API scope."), - Example("{prog} docs search hpc --max-results 20 --json", "More hits, as JSON."), + Example( + "{prog} docs search version --scope api", "Limit to the API scope." + ), + Example( + "{prog} docs search hpc --max-results 20 --json", "More hits, as JSON." + ), ), ), ) @click.argument("query") -@click.option( - "--scope", default="all", help="Search scope: all, api, cli, mcp, docs." -) +@click.option("--scope", default="all", help="Search scope: all, api, cli, mcp, docs.") @click.option("--max-results", default=10, help="Maximum results.") @click.option("--json", "as_json", is_flag=True, help="Output as structured JSON.") def _docs_search(query, scope, max_results, as_json): @@ -422,6 +434,7 @@ def _docs_search(query, scope, max_results, as_json): max_results=max_results, ) + from .skills._manage import register_skills_commands register_skills_commands(main) @@ -459,12 +472,17 @@ def _docs_search(query, scope, max_results, as_json): register_integration_commands(main) # ------------------------------------------------------------------- -# ci runner — self-hosted GitHub Actions runner lifecycle +# ci — self-hosted runner lifecycle (`ci runner`) + CI-failure reading +# (`ci why`), both attached to the one top-level `ci` group. # ------------------------------------------------------------------- from ..ci.runner import register_ci_runner_commands -register_ci_runner_commands(main) +ci_group = register_ci_runner_commands(main) + +from ..ci._why_cli import register_ci_why_command + +register_ci_why_command(ci_group) # ------------------------------------------------------------------- # linter — engine moved here from scitex-linter (soft migration) diff --git a/src/scitex_dev/ci/_why_cli.py b/src/scitex_dev/ci/_why_cli.py new file mode 100755 index 00000000..28d98752 --- /dev/null +++ b/src/scitex_dev/ci/_why_cli.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""``scitex-dev ci why`` — read WHY a CI run is red, cheaply. + +The thin CLI verb over the CI-failure-reading primitive +(``scitex_dev.ci.why``), attached to the top-level ``ci`` group alongside +``ci runner``. Reading CI *status* is one word (``failure``); this reads +the *reason* for a fraction of the log by fetching a failing run's +``--log-failed`` ONCE and distilling it to failing test ids, assertion +lines, or a setup ``##[error]``. + +``why`` resolves a PR number / run id / branch / nothing (current branch) +to the failing run(s) behind it. A target it cannot read raises loudly +(exit 2) — UNKNOWN never reads as green. Exit is tri-state: 0 green, 1 +red (the reason is printed), 2 could-not-read. +""" + +from __future__ import annotations + +import json as _json +import sys + +import click + +from .._ecosystem.help_spec import CliHelp, Example, SpecCommand + +_WHY_HELP = CliHelp( + summary="Extract the real reason a CI run is red — as cheaply as status.", + description=( + "TARGET is a PR number, a run id (>=8 digits), a branch name, or " + "omitted (the current git branch's latest run). Fetches each " + "failing run's log once and distils it to failing test ids, " + "assertion lines, or a setup `##[error]` — a few hundred bytes, " + "not the whole log.", + ), + examples=( + Example("{prog} ci why 712", "Explain PR 712's failing checks."), + Example("{prog} ci why 29446283736", "Explain one run id."), + Example("{prog} ci why -R owner/repo main", "A branch of a repo."), + Example("{prog} ci why 712 --json", "Structured output."), + ), + exit_codes=( + (0, "no failing jobs found (green)"), + (1, "failing jobs found — the distilled reason is printed"), + (2, "could not read the run (gh missing/unauthenticated/unresolved)"), + ), +) + + +def _emit_json(runs) -> None: + click.echo(_json.dumps([r.to_dict() for r in runs], indent=2, default=str)) + + +def _emit_human(runs) -> None: + from .why import render_text + + blocks: list[str] = [] + for run in runs: + head = f"run {run.run_id}" + if run.workflow: + head += f": {run.workflow}" + if run.branch: + head += f" [{run.branch}]" + block = [head, render_text(run)] + if run.url: + block.append(f"-> {run.url}") + blocks.append("\n".join(block)) + click.echo("\n\n".join(blocks)) + + +def register_ci_why_command(ci_group): + """Attach the ``why`` verb to the top-level ``ci`` group (beside runner).""" + + @ci_group.command("why", cls=SpecCommand, help_spec=_WHY_HELP) + @click.argument("target", required=False, default="") + @click.option("-R", "--repo", default=None, help="Target OWNER/REPO (else CWD).") + @click.option("--json", "as_json", is_flag=True, help="Emit structured JSON.") + def ci_why(target, repo, as_json): + from .why import CIWhyError, explain_ci_run + + try: + runs = explain_ci_run(target or None, repo=repo) + except CIWhyError as exc: + click.echo(f"ci why: {exc}", err=True) + sys.exit(2) + + if as_json: + _emit_json(runs) + else: + _emit_human(runs) + + any_failures = any(run.failures for run in runs) + sys.exit(1 if any_failures else 0) + + return ci_why + + +__all__ = ["register_ci_why_command"] + + +# EOF diff --git a/src/scitex_dev/ci/runner/__init__.py b/src/scitex_dev/ci/runner/__init__.py index c1f76b7f..0cab879e 100755 --- a/src/scitex_dev/ci/runner/__init__.py +++ b/src/scitex_dev/ci/runner/__init__.py @@ -29,6 +29,7 @@ def ci(ctx: click.Context) -> None: \b Verbs: runner — self-hosted GitHub Actions runner lifecycle + why — read WHY a CI run is red (distil the failure log) """ if ctx.invoked_subcommand is None: click.echo(ctx.get_help()) diff --git a/src/scitex_dev/ci/why/__init__.py b/src/scitex_dev/ci/why/__init__.py new file mode 100755 index 00000000..93c95829 --- /dev/null +++ b/src/scitex_dev/ci/why/__init__.py @@ -0,0 +1,55 @@ +"""``scitex_dev.ci.why`` — the ecosystem CI-failure-reading primitive. + +Reading CI *status* is one word (``failure``); reading *why* has been +tens of thousands of lines, so a bounded-context agent is steered to the +cheap word and the word replaces the reason instead of summarising it. +This primitive inverts that price: it fetches a failing run's log ONCE +and distils it to a few hundred bytes — failing test ids, assertion +lines, or a setup failure's ``##[error]``. + +The reusable SSOT any SciTeX project (sac, the umbrella, …) consumes via +a thin ``ci why`` verb. Everything except the injectable :func:`run_gh` +gh-seam is pure/string-based (unit-tested, no network); a target that +cannot be read raises :class:`CIWhyError` (UNKNOWN, never a silent +"green"). + +Public entry:: + + from scitex_dev.ci.why import explain_ci_run, render_text + + for run in explain_ci_run("712"): # PR#, run id, or branch + print(render_text(run)) +""" + +from __future__ import annotations + +from ._model import CIWhyError, GhRunner, JobFailure, RunFailures +from ._parse import ( + clean_log_line, + parse_failed_log, + parse_job_context, + split_log_by_job, +) +from ._resolve import ( + explain_ci_run, + explain_run, + render_text, + resolve_run_ids, + run_gh, +) + +__all__ = [ + "CIWhyError", + "GhRunner", + "JobFailure", + "RunFailures", + "clean_log_line", + "split_log_by_job", + "parse_job_context", + "parse_failed_log", + "run_gh", + "resolve_run_ids", + "explain_run", + "explain_ci_run", + "render_text", +] diff --git a/src/scitex_dev/ci/why/_model.py b/src/scitex_dev/ci/why/_model.py new file mode 100755 index 00000000..f8d77980 --- /dev/null +++ b/src/scitex_dev/ci/why/_model.py @@ -0,0 +1,120 @@ +"""Data model for the CI-failure-reading primitive. + +The distilled shapes a red CI run collapses to: :class:`JobFailure` (one +job's few-line reason) and :class:`RunFailures` (every failing job of a +run). :class:`CIWhyError` is the loud, UNKNOWN-not-green error the whole +primitive raises rather than pretending a run it cannot read is fine. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable, Optional + +#: A ``gh`` seam: given ``gh`` argv (without the leading ``gh``), return +#: stdout text. The one injection point that touches the network — every +#: resolver takes it so tests pass a plain callable and stay offline. +GhRunner = Callable[[list[str]], str] + + +class CIWhyError(RuntimeError): + """gh is missing/unauthenticated/errored, or the target won't resolve. + + Raised, never swallowed into a reassuring "no failures": not knowing + WHY a run is red is UNKNOWN, and UNKNOWN must not read as green. A CLI + layer turns this into a loud stderr error plus a non-zero exit. + """ + + +@dataclass +class JobFailure: + """The distilled failure of ONE job — a few lines, not a whole log.""" + + job: str + py: Optional[str] = None + os: Optional[str] = None + failed_tests: list[str] = field(default_factory=list) + assertions: list[str] = field(default_factory=list) + errors: list[str] = field(default_factory=list) # ##[error] annotations + tail: list[str] = field(default_factory=list) + url: str = "" + + @property + def signal(self) -> str: + """Which tier produced the primary evidence (priority order).""" + if self.failed_tests: + return "pytest-summary" + if self.assertions: + return "pytest-assertion" + if self.errors: + return "annotation" + if self.tail: + return "tail" + return "none" + + def context(self) -> str: + """`` (py3.11, ubuntu)`` — the matrix context, or empty.""" + bits = [b for b in (self.py and f"py{self.py}", self.os) if b] + return f" ({', '.join(bits)})" if bits else "" + + def primary_lines( + self, *, max_assertions: int = 8, max_errors: int = 5, max_tests: int = 20 + ) -> list[str]: + """The compact evidence to show under this job's header.""" + lines: list[str] = [] + if self.failed_tests: + lines.extend(self.failed_tests[:max_tests]) + extra = len(self.failed_tests) - max_tests + if extra > 0: + lines.append(f"... and {extra} more failing test(s)") + if self.assertions: + lines.extend(self.assertions[:max_assertions]) + if not self.failed_tests and not self.assertions: + if self.errors: + lines.extend(f"##[error] {e}" for e in self.errors[:max_errors]) + elif self.tail: + lines.append("(no pytest/annotation signal — last log lines:)") + lines.extend(self.tail) + else: + lines.append("(failed, but gh returned no log content)") + return lines + + def to_dict(self) -> dict: + """A JSON-ready view of this job's distilled failure.""" + return { + "job": self.job, + "python": self.py, + "os": self.os, + "signal": self.signal, + "failed_tests": self.failed_tests, + "assertions": self.assertions, + "errors": self.errors, + "tail": self.tail, + "url": self.url, + } + + +@dataclass +class RunFailures: + """Every distilled job failure for ONE run.""" + + run_id: str + workflow: str = "" + title: str = "" + branch: str = "" + url: str = "" + failures: list[JobFailure] = field(default_factory=list) + + def to_dict(self) -> dict: + """A JSON-ready view of the run and its per-job failures.""" + return { + "run_id": self.run_id, + "workflow": self.workflow, + "title": self.title, + "branch": self.branch, + "url": self.url, + "failures": [f.to_dict() for f in self.failures], + } + + +__all__ = ["CIWhyError", "GhRunner", "JobFailure", "RunFailures"] diff --git a/src/scitex_dev/ci/why/_parse.py b/src/scitex_dev/ci/why/_parse.py new file mode 100755 index 00000000..db46122a --- /dev/null +++ b/src/scitex_dev/ci/why/_parse.py @@ -0,0 +1,167 @@ +"""Pure, network-free parsing of ``gh run view --log-failed`` output. + +Everything here is string in, structured out — no subprocess, no gh, no +network — so it is exhaustively unit-testable on real-shaped log strings. +:func:`parse_failed_log` is the core: it distils one job's log into a +:class:`~scitex_dev.ci.why._model.JobFailure` by four priority tiers — +pytest ``short test summary info`` FAILED ids, the ``FAILURES`` block's +``E`` assertion lines, ``##[error]`` annotations, then a tail fallback. +""" + +from __future__ import annotations + +import re +from collections import OrderedDict +from typing import Optional + +from ._model import JobFailure + +# The ISO-8601 runner timestamp that prefixes every raw actions log line +# (after gh's optional "\t\t" prefix). Everything up to and +# including it is scaffolding. +_TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z ?") +_BOM = "" +_ERROR_ANNOT = "##[error]" + +_SUMMARY_RE = re.compile(r"={3,}\s*short test summary info\s*={3,}", re.IGNORECASE) +_FAILURES_HDR_RE = re.compile(r"={3,}\s*(FAILURES|ERRORS)\s*={3,}") +_SUMMARY_END_RE = re.compile( + r"={3,}.*\b(passed|failed|error|errors|skipped|deselected|" + r"xfailed|xpassed|warning|warnings|no tests ran)\b.*={3,}", + re.IGNORECASE, +) +_FAILED_LINE_RE = re.compile(r"^(FAILED|ERROR)\s+\S") +_E_LINE_RE = re.compile(r"^E(?:\s|$)") + +# Matrix legs encode context in the job name (py version, runner OS). +_PY_RE = re.compile(r"(?:py[ -]?)?3[.\-](\d{1,2})\b", re.IGNORECASE) +_OS_RE = re.compile( + r"(ubuntu-latest|ubuntu-\d[\w.]*|ubuntu|macos-[\w.]+|macos|" + r"windows-[\w.]+|windows|self-hosted)", + re.IGNORECASE, +) + + +def clean_log_line(raw: str) -> Optional[str]: + r"""Strip GitHub-Actions scaffolding from one raw log line. + + Removes the optional ``\t\t`` prefix that + ``gh run view --log-failed`` prepends, the ISO-8601 runner timestamp, + and the UTF-8 BOM. ``##[group]`` / ``##[endgroup]`` fold markers are + dropped entirely (return ``None``). Everything else — including + ``##[error]`` — is returned as bare content. + """ + line = raw.rstrip("\n").replace(_BOM, "") + m = _TS_RE.search(line) + content = line[m.end() :] if m else line + stripped = content.strip() + if stripped.startswith("##[group]") or stripped.startswith("##[endgroup]"): + return None + return content + + +def split_log_by_job(log_text: str) -> "OrderedDict[str, str]": + r"""Group ``gh run view --log-failed`` lines by job name (first column). + + ``--log-failed`` prefixes each line ``\t\t ``. + Lines with no such prefix (a plain single-job log, as in a fixture) + group under the empty-string key. + """ + groups: "OrderedDict[str, list[str]]" = OrderedDict() + for raw in log_text.splitlines(): + line = raw.replace(_BOM, "") + parts = line.split("\t", 2) + if len(parts) == 3 and _TS_RE.match(parts[2]): + job = parts[0] + else: + job = "" + groups.setdefault(job, []).append(raw) + return OrderedDict((job, "\n".join(lines)) for job, lines in groups.items()) + + +def parse_job_context(job_name: str) -> tuple[Optional[str], Optional[str]]: + """Best-effort ``(python_version, runner_os)`` from a job name. + + Matrix legs encode context in the name, e.g. + ``pytest-matrix-on-ubuntu-py3.11`` -> ('3.11', 'ubuntu'), + ``import-smoke-on-ubuntu-py3-12`` -> ('3.12', 'ubuntu'), + ``...guard-on-self-hosted`` -> (None, 'self-hosted'). + """ + py = None + m = _PY_RE.search(job_name) + if m: + py = f"3.{m.group(1)}" + os_ = None + mo = _OS_RE.search(job_name) + if mo: + os_ = mo.group(1).lower() + return py, os_ + + +def parse_failed_log( + log_text: str, + *, + job_name: str = "", + url: str = "", + tail_lines: int = 8, +) -> JobFailure: + """Parse ONE job's ``--log-failed`` text into a :class:`JobFailure`. + + Priority of signals: (1) the ``short test summary info`` ``FAILED`` + lines; (2) the ``FAILURES`` block ``E`` assertion lines; (3) + ``##[error]`` annotations (setup/infra failures); (4) fallback to the + last ``tail_lines`` non-blank cleaned lines. + """ + py, os_ = parse_job_context(job_name) + fail = JobFailure(job=job_name, py=py, os=os_, url=url) + + clean: list[str] = [] + for raw in log_text.splitlines(): + c = clean_log_line(raw) + if c is None: + continue + clean.append(c) + if c.lstrip().startswith(_ERROR_ANNOT): + annot = c.lstrip()[len(_ERROR_ANNOT) :].strip() + if annot: + fail.errors.append(annot) + + # (1) pytest short test summary — the cheapest, richest signal. + in_summary = False + for c in clean: + s = c.strip() + if _SUMMARY_RE.search(s): + in_summary = True + continue + if in_summary: + if _SUMMARY_END_RE.search(s): + break + if _FAILED_LINE_RE.match(s): + fail.failed_tests.append(s) + + # (2) assertion detail from the FAILURES / ERRORS block. + in_failures = False + for c in clean: + st = c.strip() + if _FAILURES_HDR_RE.search(st): + in_failures = True + continue + if in_failures: + if _SUMMARY_RE.search(st): + break + if _E_LINE_RE.match(c): + fail.assertions.append(st) + + # (4) fallback tail when nothing structured was found. + if not (fail.failed_tests or fail.assertions or fail.errors): + nonblank = [c for c in clean if c.strip()] + fail.tail = nonblank[-tail_lines:] + return fail + + +__all__ = [ + "clean_log_line", + "split_log_by_job", + "parse_job_context", + "parse_failed_log", +] diff --git a/src/scitex_dev/ci/why/_resolve.py b/src/scitex_dev/ci/why/_resolve.py new file mode 100755 index 00000000..25f42023 --- /dev/null +++ b/src/scitex_dev/ci/why/_resolve.py @@ -0,0 +1,254 @@ +"""Run-level orchestration: resolve a target to run id(s), fetch, distil. + +The only network-touching layer. :func:`run_gh` is the default ``gh`` +seam; every resolver accepts ``run_gh=None`` and falls back to it, so a +test injects a plain callable and stays offline. :func:`explain_ci_run` +is the public entry a consumer's thin verb calls: a PR number / run id / +branch / nothing in, the distilled failing run(s) out. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from typing import Optional + +from ._model import CIWhyError, GhRunner, RunFailures +from ._parse import parse_failed_log, split_log_by_job + +# --log-failed can be a few MB; give the fetch room but stay bounded. +_GH_TIMEOUT_S = 90 + +# A PR number is small; a run id is a 10+ digit database id. Anything at +# or above this magnitude is treated as a run id, below it as a PR number. +_RUN_ID_MIN = 10_000_000 + +_FAILED_JOB_CONCLUSIONS = {"failure", "cancelled", "timed_out", "startup_failure"} +_RUN_ID_IN_LINK = re.compile(r"/actions/runs/(\d+)") + + +def run_gh(args: list[str], *, _run=subprocess.run) -> str: + """Default ``gh`` seam: run ``gh `` and return stdout text. + + The one place the network is touched. Returns stdout even on a + non-zero exit *when stdout is non-empty* — ``gh pr checks`` exits + non-zero precisely when checks fail, yet still prints the JSON we + want. A non-zero exit with no output (bad run id, auth failure, gh + missing) raises :class:`CIWhyError`. + """ + try: + proc = _run( + ["gh", *args], + capture_output=True, + text=True, + timeout=_GH_TIMEOUT_S, + check=False, + ) + except FileNotFoundError as exc: + raise CIWhyError( + "gh CLI not found on PATH — install GitHub CLI: https://cli.github.com" + ) from exc + except (OSError, subprocess.SubprocessError) as exc: + raise CIWhyError(f"gh {' '.join(args)} failed to run: {exc}") from exc + out = proc.stdout or "" + if proc.returncode != 0 and not out.strip(): + err = (proc.stderr or "").strip() or f"exited {proc.returncode}" + raise CIWhyError(f"gh {' '.join(args)}: {err}") + return out + + +def _resolve_gh(gh: Optional[GhRunner]) -> GhRunner: + """Fall back to the default :func:`run_gh` seam when none is injected.""" + return gh if gh is not None else run_gh + + +def _repo_args(repo: Optional[str]) -> list[str]: + return ["-R", repo] if repo else [] + + +def _current_branch(_run=subprocess.run) -> str: + try: + proc = _run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + raise CIWhyError(f"could not determine current branch: {exc}") from exc + branch = (proc.stdout or "").strip() + if proc.returncode != 0 or not branch: + raise CIWhyError( + "not in a git checkout (or detached HEAD) — pass a PR number, " + "run id, or branch explicitly" + ) + return branch + + +def _pr_failing_run_ids(pr: str, *, gh: GhRunner, repo: Optional[str]) -> list[str]: + raw = gh( + ["pr", "checks", pr, *_repo_args(repo), "--json", "name,state,link,bucket"] + ) + try: + checks = json.loads(raw or "[]") + except ValueError as exc: + raise CIWhyError(f"could not parse gh pr checks JSON for PR {pr}") from exc + run_ids: list[str] = [] + for c in checks or []: + failed = c.get("bucket") == "fail" or str(c.get("state", "")).upper() in { + "FAILURE", + "ERROR", + "CANCELLED", + "TIMED_OUT", + } + if not failed: + continue + m = _RUN_ID_IN_LINK.search(str(c.get("link", ""))) + if m and m.group(1) not in run_ids: + run_ids.append(m.group(1)) + return run_ids + + +def _branch_latest_run_id( + branch: str, *, gh: GhRunner, repo: Optional[str] +) -> list[str]: + raw = gh( + [ + "run", + "list", + *_repo_args(repo), + "-b", + branch, + "-L", + "1", + "--json", + "databaseId", + ] + ) + try: + runs = json.loads(raw or "[]") + except ValueError as exc: + raise CIWhyError( + f"could not parse gh run list JSON for branch {branch}" + ) from exc + if not runs: + raise CIWhyError(f"no workflow runs found for branch '{branch}'") + return [str(runs[0]["databaseId"])] + + +def resolve_run_ids( + target: Optional[str], + *, + run_gh: Optional[GhRunner] = None, + repo: Optional[str] = None, +) -> list[str]: + """Resolve a PR number / run id / branch / nothing to failing run id(s). + + * empty -> the latest run for the current git branch; + * a run id (>= 8 digits) -> itself; + * a PR number -> the run id(s) behind its failing checks; + * anything else -> the latest run for that branch name. + """ + gh = _resolve_gh(run_gh) + target = (target or "").strip() + if not target: + return _branch_latest_run_id(_current_branch(), gh=gh, repo=repo) + if target.isdigit(): + if int(target) >= _RUN_ID_MIN: + return [target] + return _pr_failing_run_ids(target, gh=gh, repo=repo) + return _branch_latest_run_id(target, gh=gh, repo=repo) + + +def explain_run( + run_id: str, *, run_gh: Optional[GhRunner] = None, repo: Optional[str] = None +) -> RunFailures: + """Fetch + distil every failing job of a single run.""" + gh = _resolve_gh(run_gh) + meta_raw = gh( + [ + "run", + "view", + str(run_id), + *_repo_args(repo), + "--json", + "jobs,displayTitle,headBranch,conclusion,url,workflowName", + ] + ) + try: + meta = json.loads(meta_raw or "{}") + except ValueError as exc: + raise CIWhyError(f"could not parse gh run view JSON for run {run_id}") from exc + run = RunFailures( + run_id=str(run_id), + workflow=meta.get("workflowName", ""), + title=meta.get("displayTitle", ""), + branch=meta.get("headBranch", ""), + url=meta.get("url", ""), + ) + failing = [ + j + for j in (meta.get("jobs") or []) + if str(j.get("conclusion", "")).lower() in _FAILED_JOB_CONCLUSIONS + ] + if not failing: + return run + + by_job = split_log_by_job( + gh(["run", "view", str(run_id), *_repo_args(repo), "--log-failed"]) + ) + for j in failing: + name = j.get("name", "") + jf = parse_failed_log(by_job.get(name, ""), job_name=name, url=j.get("url", "")) + if jf.signal == "none": + steps = [ + s.get("name", "?") + for s in (j.get("steps") or []) + if str(s.get("conclusion", "")).lower() == "failure" + ] + if steps: + jf.errors.append("failed step: " + ", ".join(steps)) + run.failures.append(jf) + return run + + +def explain_ci_run( + target: Optional[str], + *, + run_gh: Optional[GhRunner] = None, + repo: Optional[str] = None, +) -> list[RunFailures]: + """Resolve ``target`` and distil the failing run(s) behind it. + + A PR number can front more than one failing run (one per matrix + workflow), so this returns a list; a bare run id yields a one-element + list. This is the SSOT entry a consumer's thin ``ci why`` verb calls. + """ + gh = _resolve_gh(run_gh) + return [ + explain_run(rid, run_gh=gh, repo=repo) + for rid in resolve_run_ids(target, run_gh=gh, repo=repo) + ] + + +def render_text(run: RunFailures) -> str: + """Render one run's failures as compact human text (per-job blocks).""" + if not run.failures: + return "no failures" + out: list[str] = [] + for jf in run.failures: + out.append(f"{jf.job}{jf.context()}") + out.extend(f" {line}" for line in jf.primary_lines()) + if jf.url: + out.append(f" -> {jf.url}") + return "\n".join(out) + + +__all__ = [ + "run_gh", + "resolve_run_ids", + "explain_run", + "explain_ci_run", + "render_text", +] diff --git a/tests/scitex_dev/ci/__init__.py b/tests/scitex_dev/ci/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/scitex_dev/ci/test__why_cli.py b/tests/scitex_dev/ci/test__why_cli.py new file mode 100755 index 00000000..b899c359 --- /dev/null +++ b/tests/scitex_dev/ci/test__why_cli.py @@ -0,0 +1,161 @@ +"""Tests for the ``scitex-dev ci why`` CLI verb (``scitex_dev.ci._why_cli``). + +No mocks. The verb is attached to the SAME top-level ``ci`` group that +holds ``ci runner``; the tests build that group and confirm both verbs +coexist. The end-to-end path is driven against a REAL (fake) ``gh`` +executable placed on ``PATH`` — a tiny script emitting canned ``gh run +view`` JSON and ``--log-failed`` output — so the actual ``run_gh`` +subprocess seam is exercised without a network. AAA, one logical +assertion per test. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +pytest.importorskip("click") + +import click +from click.testing import CliRunner + +from scitex_dev.ci._why_cli import register_ci_why_command +from scitex_dev.ci.runner import register_ci_runner_commands + +_JOB = "pytest-matrix-on-ubuntu-py3.11" + +# A fake `gh`: emit run-view JSON on --json, job-prefixed --log-failed log. +_FAKE_GH = f'''#!/usr/bin/env python3 +import sys, json +argv = sys.argv[1:] +JOB = "{_JOB}" +if "--log-failed" in argv: + rows = [ + JOB + "\\tRun pytest\\t2026-07-15T10:00:02.0Z " + "=========== short test summary info ===========", + JOB + "\\tRun pytest\\t2026-07-15T10:00:02.1Z " + "FAILED tests/test_x.py::test_y - AssertionError: assert 1 == 2", + JOB + "\\tRun pytest\\t2026-07-15T10:00:02.2Z " + "=========== 1 failed in 0.01s ===========", + ] + sys.stdout.write("\\n".join(rows) + "\\n") +elif "--json" in argv: + sys.stdout.write(json.dumps({{ + "workflowName": "tests", "displayTitle": "t", "headBranch": "b", + "url": "https://github.com/o/r/actions/runs/10000042", + "jobs": [{{"name": JOB, "conclusion": "failure", "url": "https://x/1"}}], + }})) +sys.exit(0) +''' + +# A broken `gh`: always non-zero with empty stdout — an UNKNOWN, not green. +_BROKEN_GH = """#!/usr/bin/env python3 +import sys +sys.stderr.write("gh: could not resolve\\n") +sys.exit(1) +""" + + +def _build_main(): + @click.group() + def main(): + pass + + ci = register_ci_runner_commands(main) + register_ci_why_command(ci) + return main + + +def _put_gh_on_path(tmp_path, script: str): + """Write an executable fake ``gh`` and prepend its dir to PATH. + + Returns a ``restore()`` callable that puts PATH back — no pytest + env-patching fixture (PA-306 forbids it), just real env save/restore. + """ + gh = tmp_path / "gh" + gh.write_text(script) + gh.chmod(0o755) + saved = os.environ.get("PATH", "") + os.environ["PATH"] = f"{tmp_path}{os.pathsep}{saved}" + return lambda: os.environ.__setitem__("PATH", saved) + + +def test_ci_why_registered_on_top_level_ci_group(): + # Arrange + main = _build_main() + # Act + result = CliRunner().invoke(main, ["ci", "why", "--help"]) + # Assert + assert result.exit_code == 0 + + +def test_ci_runner_still_resolves_alongside_why(): + # Arrange — why must not clobber the sibling runner group. + main = _build_main() + # Act + result = CliRunner().invoke(main, ["ci", "runner", "--help"]) + # Assert + assert result.exit_code == 0 + + +def test_ci_why_help_documents_target(): + # Arrange + main = _build_main() + # Act + result = CliRunner().invoke(main, ["ci", "why", "--help"]) + # Assert + assert "TARGET" in result.output + + +def test_ci_why_extracts_failing_test_from_fake_gh(tmp_path): + # Arrange + restore = _put_gh_on_path(tmp_path, _FAKE_GH) + main = _build_main() + # Act + try: + result = CliRunner().invoke(main, ["ci", "why", "10000042"]) + finally: + restore() + # Assert + assert "tests/test_x.py::test_y" in result.output + + +def test_ci_why_exits_1_when_failures_found(tmp_path): + # Arrange + restore = _put_gh_on_path(tmp_path, _FAKE_GH) + main = _build_main() + # Act + try: + result = CliRunner().invoke(main, ["ci", "why", "10000042"]) + finally: + restore() + # Assert + assert result.exit_code == 1 + + +def test_ci_why_json_emits_failed_tests(tmp_path): + # Arrange + restore = _put_gh_on_path(tmp_path, _FAKE_GH) + main = _build_main() + # Act + try: + result = CliRunner().invoke(main, ["ci", "why", "10000042", "--json"]) + finally: + restore() + # Assert + assert json.loads(result.output)[0]["failures"][0]["failed_tests"] + + +def test_ci_why_exits_2_when_gh_cannot_read(tmp_path): + # Arrange — a broken gh is UNKNOWN; it must never read as green (exit 0). + restore = _put_gh_on_path(tmp_path, _BROKEN_GH) + main = _build_main() + # Act + try: + result = CliRunner().invoke(main, ["ci", "why", "10000042"]) + finally: + restore() + # Assert + assert result.exit_code == 2 diff --git a/tests/scitex_dev/ci/why/__init__.py b/tests/scitex_dev/ci/why/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/scitex_dev/ci/why/test__parse.py b/tests/scitex_dev/ci/why/test__parse.py new file mode 100755 index 00000000..a31c0a50 --- /dev/null +++ b/tests/scitex_dev/ci/why/test__parse.py @@ -0,0 +1,221 @@ +"""Tests for the pure CI-log parser (``scitex_dev.ci.why._parse``). + +No mocks. The parser is exercised on real-shaped GitHub-Actions log +STRINGS — timestamp prefixes, ``##[group]`` noise, a ``FAILURES`` block, +a ``short test summary info`` block, and a setup ``##[error]`` log. AAA, +one logical assertion per test. +""" + +from __future__ import annotations + +from scitex_dev.ci.why import ( + clean_log_line, + parse_failed_log, + parse_job_context, + split_log_by_job, +) + +# gh --log-failed prefixes each line "\t\tZ ". +_PJ = "pytest-matrix-on-ubuntu-py3.11" +_PS = "Run pytest" + + +def _line(job: str, step: str, ts: str, content: str) -> str: + return f"{job}\t{step}\t{ts}Z {content}" + + +PYTEST_LOG = "\n".join( + _line(_PJ, _PS, ts, content) + for ts, content in [ + ("2026-07-15T10:00:00.1000000", "##[group]Run python -m pytest"), + ("2026-07-15T10:00:00.2000000", "python -m pytest -v"), + ("2026-07-15T10:00:00.3000000", "##[endgroup]"), + ("2026-07-15T10:00:01.0000000", "=============== FAILURES ==============="), + ("2026-07-15T10:00:01.1000000", "_______________ test_math _______________"), + ("2026-07-15T10:00:01.3000000", " def test_math():"), + ("2026-07-15T10:00:01.4000000", "> assert 3 == 4"), + ("2026-07-15T10:00:01.5000000", "E assert 3 == 4"), + ("2026-07-15T10:00:01.7000000", "tests/test_math.py:5: AssertionError"), + ( + "2026-07-15T10:00:02.0000000", + "=========== short test summary info ===========", + ), + ( + "2026-07-15T10:00:02.1000000", + "FAILED tests/test_math.py::test_math - AssertionError: assert 3 == 4", + ), + ( + "2026-07-15T10:00:02.2000000", + "=========== 1 failed, 4 passed in 0.12s ===========", + ), + ("2026-07-15T10:00:02.3000000", "##[error]Process completed with exit code 1."), + ] +) + +_SJ = "no-hosted-runners-guard-on-self-hosted" +_SS = "Run astral-sh/setup-uv@v3" + +SETUP_LOG = "\n".join( + _line(_SJ, _SS, ts, content) + for ts, content in [ + ("2026-07-15T15:59:17.6765113", "##[group]Run astral-sh/setup-uv@v3"), + ("2026-07-15T15:59:17.6769300", "##[endgroup]"), + ( + "2026-07-15T15:59:18.4499226", + "ENOENT: no such file or directory, open '/data/_temp/99eb7246'", + ), + ( + "2026-07-15T15:59:48.7185987", + "##[error]ENOENT: no such file or directory, open '/data/_temp/99eb7246'", + ), + ] +) + +TAIL_LOG = "\n".join( + _line("some-job-on-ubuntu-latest", "Build", ts, content) + for ts, content in [ + ("2026-07-15T10:00:00.1000000", "##[group]Build"), + ("2026-07-15T10:00:00.2000000", "compiling module foo"), + ("2026-07-15T10:00:00.3000000", "##[endgroup]"), + ("2026-07-15T10:00:00.4000000", "linker: undefined reference to bar"), + ] +) + + +def test_clean_log_line_strips_job_prefix_and_timestamp(): + # Arrange + raw = _line(_PJ, _PS, "2026-07-15T10:00:02.1000000", "FAILED tests/t.py::t - X") + # Act + cleaned = clean_log_line(raw) + # Assert + assert cleaned == "FAILED tests/t.py::t - X" + + +def test_clean_log_line_drops_group_markers(): + # Arrange + raw = _line(_PJ, _PS, "2026-07-15T10:00:00.1000000", "##[group]Run x") + # Act + cleaned = clean_log_line(raw) + # Assert + assert cleaned is None + + +def test_clean_log_line_keeps_error_annotation(): + # Arrange + raw = _line(_SJ, _SS, "2026-07-15T15:59:48.7185987", "##[error]ENOENT: boom") + # Act + cleaned = clean_log_line(raw) + # Assert + assert cleaned == "##[error]ENOENT: boom" + + +def test_split_log_by_job_separates_two_jobs(): + # Arrange + text = "\n".join( + [ + _line("job-a", "s", "2026-07-15T10:00:00.1000000", "a1"), + _line("job-b", "s", "2026-07-15T10:00:00.2000000", "b1"), + _line("job-a", "s", "2026-07-15T10:00:00.3000000", "a2"), + ] + ) + # Act + groups = split_log_by_job(text) + # Assert + assert list(groups) == ["job-a", "job-b"] + + +def test_parse_job_context_dotted_pyversion_and_os(): + # Arrange + name = "pytest-matrix-on-ubuntu-py3.11" + # Act + py, os_ = parse_job_context(name) + # Assert + assert (py, os_) == ("3.11", "ubuntu") + + +def test_parse_job_context_dashed_pyversion(): + # Arrange + name = "import-smoke-on-ubuntu-py3-12" + # Act + py, _os = parse_job_context(name) + # Assert + assert py == "3.12" + + +def test_parse_job_context_self_hosted_has_no_python(): + # Arrange + name = "no-hosted-runners-guard-on-self-hosted" + # Act + py, os_ = parse_job_context(name) + # Assert + assert (py, os_) == (None, "self-hosted") + + +def test_parse_failed_log_extracts_failing_test_id(): + # Arrange + log = PYTEST_LOG + # Act + fail = parse_failed_log(log, job_name=_PJ) + # Assert + assert fail.failed_tests == [ + "FAILED tests/test_math.py::test_math - AssertionError: assert 3 == 4" + ] + + +def test_parse_failed_log_extracts_assertion_line(): + # Arrange + log = PYTEST_LOG + # Act + fail = parse_failed_log(log, job_name=_PJ) + # Assert + assert any("assert 3 == 4" in a for a in fail.assertions) + + +def test_parse_failed_log_strips_timestamp_and_group_noise(): + # Arrange + log = PYTEST_LOG + # Act + fail = parse_failed_log(log, job_name=_PJ) + # Assert + leaked = [ + line + for line in fail.failed_tests + fail.assertions + if "2026-" in line or "##[group]" in line + ] + assert leaked == [] + + +def test_parse_failed_log_carries_matrix_context(): + # Arrange + log = PYTEST_LOG + # Act + fail = parse_failed_log(log, job_name=_PJ) + # Assert + assert (fail.py, fail.os) == ("3.11", "ubuntu") + + +def test_parse_failed_log_setup_failure_signal_is_annotation(): + # Arrange + log = SETUP_LOG + # Act + fail = parse_failed_log(log, job_name=_SJ) + # Assert + assert fail.signal == "annotation" + + +def test_parse_failed_log_setup_failure_surfaces_enoent(): + # Arrange + log = SETUP_LOG + # Act + fail = parse_failed_log(log, job_name=_SJ) + # Assert + assert any("ENOENT" in e for e in fail.errors) + + +def test_parse_failed_log_falls_back_to_tail_when_no_signal(): + # Arrange + log = TAIL_LOG + # Act + fail = parse_failed_log(log, job_name="some-job-on-ubuntu-latest") + # Assert + assert "linker: undefined reference to bar" in fail.tail diff --git a/tests/scitex_dev/ci/why/test__resolve.py b/tests/scitex_dev/ci/why/test__resolve.py new file mode 100755 index 00000000..ab7b5772 --- /dev/null +++ b/tests/scitex_dev/ci/why/test__resolve.py @@ -0,0 +1,198 @@ +"""Tests for the run-level resolver (``scitex_dev.ci.why._resolve``). + +No mocks. The default ``run_gh`` seam is exercised through its ``_run`` +injection point (a plain callable returning a ``CompletedProcess``); the +resolver + ``explain_run`` + ``explain_ci_run`` through an injected +``run_gh`` callable returning canned gh output. Nothing here touches the +network. AAA, one logical assertion per test. +""" + +from __future__ import annotations + +import json +import subprocess + +from scitex_dev.ci.why import ( + CIWhyError, + RunFailures, + explain_ci_run, + explain_run, + render_text, + resolve_run_ids, + run_gh, +) +from tests.scitex_dev.ci.why.test__parse import PYTEST_LOG, _PJ + + +# --------------------------------------------------------------------------- +# run_gh seam — honest failure, and gh-pr-checks non-zero semantics. +# --------------------------------------------------------------------------- + + +def test_run_gh_raises_when_gh_missing(): + # Arrange + def _no_gh(*_a, **_kw): + raise FileNotFoundError("gh") + + captured = None + # Act + try: + run_gh(["run", "list"], _run=_no_gh) + except CIWhyError as exc: + captured = exc + # Assert + assert captured is not None + + +def test_run_gh_returns_stdout_on_nonzero_with_output(): + # Arrange — gh pr checks exits non-zero when checks fail, yet prints JSON. + def _fake(argv, **_kw): + return subprocess.CompletedProcess(argv, 1, stdout='[{"a":1}]', stderr="") + + # Act + out = run_gh(["pr", "checks", "712"], _run=_fake) + # Assert + assert out == '[{"a":1}]' + + +def test_run_gh_raises_on_nonzero_with_empty_output(): + # Arrange — a bad run id: non-zero AND nothing on stdout is a real error. + def _fake(argv, **_kw): + return subprocess.CompletedProcess(argv, 1, stdout="", stderr="not found") + + captured = None + # Act + try: + run_gh(["run", "view", "0"], _run=_fake) + except CIWhyError as exc: + captured = exc + # Assert + assert captured is not None + + +# --------------------------------------------------------------------------- +# resolve_run_ids — target → failing run id(s), through the injected seam. +# --------------------------------------------------------------------------- + + +def test_resolve_run_ids_treats_large_number_as_run_id_without_calling_gh(): + # Arrange — a run id must not cost a gh round-trip. + def _boom(_args): + raise RuntimeError("gh should not be called for a bare run id") + + # Act + ids = resolve_run_ids("29446283736", run_gh=_boom) + # Assert + assert ids == ["29446283736"] + + +def test_resolve_run_ids_pr_number_extracts_failing_run_id(): + # Arrange — one failing check, its link carries the run id. + checks = [ + {"bucket": "pass", "state": "SUCCESS", "link": ".../actions/runs/111/job/1"}, + { + "bucket": "fail", + "state": "FAILURE", + "link": "https://github.com/o/r/actions/runs/29446283736/job/9", + }, + ] + + def _fake(_args): + return json.dumps(checks) + + # Act + ids = resolve_run_ids("712", run_gh=_fake) + # Assert + assert ids == ["29446283736"] + + +# --------------------------------------------------------------------------- +# explain_run / explain_ci_run — through the injected run_gh seam. +# --------------------------------------------------------------------------- + + +def _gh_router(jobs: list, log_text: str): + """A canned run_gh: run-view JSON on --json, the log on --log-failed.""" + + def _router(args: list) -> str: + if "--log-failed" in args: + return log_text + if "--json" in args: + return json.dumps( + { + "workflowName": "tests", + "displayTitle": "t", + "headBranch": "b", + "jobs": jobs, + } + ) + raise RuntimeError(f"unexpected gh args: {args}") + + return _router + + +def test_explain_run_green_has_no_failures(): + # Arrange + router = _gh_router([{"name": "x", "conclusion": "success"}], "") + # Act + run = explain_run("10000001", run_gh=router) + # Assert + assert run.failures == [] + + +def test_explain_run_parses_the_failing_job(): + # Arrange + jobs = [{"name": _PJ, "conclusion": "failure"}] + router = _gh_router(jobs, PYTEST_LOG) + # Act + run = explain_run("10000002", run_gh=router) + # Assert + assert ( + run.failures[0] + .failed_tests[0] + .startswith("FAILED tests/test_math.py::test_math") + ) + + +def test_explain_ci_run_returns_a_list_of_run_failures(): + # Arrange + jobs = [{"name": _PJ, "conclusion": "failure"}] + router = _gh_router(jobs, PYTEST_LOG) + # Act + runs = explain_ci_run("10000003", run_gh=router) + # Assert + assert len(runs) == 1 and isinstance(runs[0], RunFailures) + + +# --------------------------------------------------------------------------- +# render_text — compact human output. +# --------------------------------------------------------------------------- + + +def test_render_text_says_no_failures_when_empty(): + # Arrange + run = RunFailures(run_id="1") + # Act + rendered = render_text(run) + # Assert + assert rendered == "no failures" + + +def test_render_text_is_far_smaller_than_the_raw_log(): + # Arrange — the whole point: the reason costs a fraction of the log. + jobs = [{"name": _PJ, "conclusion": "failure"}] + run = explain_run("1", run_gh=_gh_router(jobs, PYTEST_LOG)) + # Act + rendered = render_text(run) + # Assert + assert len(rendered) < len(PYTEST_LOG) // 2 + + +def test_render_text_includes_the_failing_test_id(): + # Arrange + jobs = [{"name": _PJ, "conclusion": "failure"}] + run = explain_run("1", run_gh=_gh_router(jobs, PYTEST_LOG)) + # Act + rendered = render_text(run) + # Assert + assert "tests/test_math.py::test_math" in rendered