From 15b1449e57e748a3d20d7ee70b786fc1e0912014 Mon Sep 17 00:00:00 2001 From: Yusuke Watanabe Date: Tue, 14 Jul 2026 15:49:08 +0900 Subject: [PATCH] =?UTF-8?q?feat(audit):=20PS-169=20=E2=80=94=20GitHub-host?= =?UTF-8?q?ed=20runners=20are=20forbidden,=20no=20exceptions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator mandate (2026-07-14): 「PR用のテストとgithub側のランナーというのは 本当にもう一切使わないでください」「強制です、例外なしです」— never use a GitHub-hosted runner, including for PR tests; any package still using one must be a hard ERROR. This rule is the ONLY enforcement that exists. Blocking hosted runners at the org level is a GitHub Enterprise Cloud policy and `scitex-ai` is on the Free plan, so there is no backstop behind this check. Hence severity E. Why it is not a grep for `ubuntu-latest` ---------------------------------------- The runner is routinely INDIRECT, and only the first of these is greppable on the `runs-on:` line: runs-on: ubuntu-latest # direct runs-on: [ubuntu-22.04] # list form runs-on: ${{ matrix.os }} # hides in strategy.matrix.os runs-on: ${{ inputs.runner }} # hides in a workflow_call input default PS-169 resolves the EFFECTIVE runner — it follows `matrix.*` and `inputs.*` back to their declared values before judging. A literal line-scan would pass a workflow that runs every job on ubuntu-latest via a matrix, which is exactly how the violation hides. An UNRESOLVABLE runner is also a violation: it cannot be proven Spartan-only, and absence of evidence is not evidence of compliance. Consuming the shared workflow is not, by itself, compliance: a repo can be a thin caller of scitex-ai/.github and still run hosted if the CALLEE is dirty. That is closed structurally — the SSOT's own `runs-on` is audited by this same rule in its own repo, so a dirty shared workflow fires at source. Callers of reusable workflows from owners outside `scitex-ai` are flagged, since a third party's runner cannot be vouched for. Proof it fires (not merely declared): - scitex-ai/.github @ main -> 3 violations (the real dirty SSOT) - scitex-ai/.github @ fixed branch -> 0 - scitex-dev 4 | figrecipe 6 | newb 7 | scitex-agent-container 10 - 12/12 unit tests green, covering every runner form incl. matrix indirection, workflow_call input defaults, unparseable YAML, and the compliant Spartan + trusted-caller cases (which must stay silent). Registered in _extra_rules.py (the sanctioned sidecar) rather than _registry.py, which is already over the 512-line cap. --- src/scitex_dev/_cli/audit/_project/_audit.py | 4 + .../audit/_project/_check_hosted_runners.py | 273 ++++++++++++++++++ .../_cli/audit/_project/_extra_rules.py | 24 ++ .../_project/test__check_hosted_runners.py | 205 +++++++++++++ 4 files changed, 506 insertions(+) create mode 100755 src/scitex_dev/_cli/audit/_project/_check_hosted_runners.py create mode 100755 tests/scitex_dev/_cli/audit/_project/test__check_hosted_runners.py diff --git a/src/scitex_dev/_cli/audit/_project/_audit.py b/src/scitex_dev/_cli/audit/_project/_audit.py index d4bbee89..62913e53 100755 --- a/src/scitex_dev/_cli/audit/_project/_audit.py +++ b/src/scitex_dev/_cli/audit/_project/_audit.py @@ -235,6 +235,10 @@ def audit_project( check_ps164_workflow_naming(repo_root, Violation, violations) # hook-bypass: line-limit + from ._check_hosted_runners import check_ps169_hosted_runners + + check_ps169_hosted_runners(repo_root, Violation, violations) + # hook-bypass: line-limit from ._check_secret_env_prefix import check_ps168_secret_env_prefix check_ps168_secret_env_prefix(repo_root, distribution, Violation, violations) diff --git a/src/scitex_dev/_cli/audit/_project/_check_hosted_runners.py b/src/scitex_dev/_cli/audit/_project/_check_hosted_runners.py new file mode 100755 index 00000000..578cc27a --- /dev/null +++ b/src/scitex_dev/_cli/audit/_project/_check_hosted_runners.py @@ -0,0 +1,273 @@ +"""PS-169 — GitHub-hosted runners are forbidden. No exceptions. + +Operator mandate (2026-07-14): + + 「PR用のテストとgithub側のランナーというのは本当にもう一切使わないでください」 + — never use GitHub-hosted runners, at all, including PR tests. + 「もし使っているパッケージがあれば…リンター、フックでエラーにしてください。 + 強制です、例外なしです」 + — any package still using one must be a hard ERROR. Mandatory, no exceptions. + +Every SciTeX job runs on the self-hosted Spartan runners. If Spartan cannot +run something, we fix Spartan — falling back to a GitHub-hosted runner is +forbidden. + +**This rule is the only enforcement that exists.** GitHub cannot help us +here: blocking hosted runners at the org level is an Enterprise Cloud +policy, and the `scitex-ai` org is on the Free plan. There is no backstop +behind this check, which is why it is severity **E** and why it is wired +into both the pre-commit hook and CI. + +Why this cannot be a `grep ubuntu-latest` +----------------------------------------- +The runner is frequently *indirect*. All of these are violations, and only +the first is greppable on the ``runs-on:`` line:: + + runs-on: ubuntu-latest # (1) direct scalar + runs-on: [ubuntu-latest] # (2) list form + runs-on: ${{ matrix.os }} # (3) resolves via strategy.matrix.os + strategy: {matrix: {os: [ubuntu-latest, macos-14]}} + runs-on: ${{ inputs.runner }} # (4) resolves via a workflow_call input default + +So the check resolves the **effective** runner: it follows ``matrix.*`` and +``inputs.*`` expressions back to their declared values before judging. A rule +that only pattern-matched the literal line would pass a workflow that runs +every job on ``ubuntu-latest`` via a matrix — which is precisely how the +violation hides. + +Consuming the shared workflow is not, by itself, compliance +----------------------------------------------------------- +A repo can be "unified" (a thin caller of ``scitex-ai/.github``) and still +effectively run on a hosted runner if the *callee* is dirty. That is closed +structurally rather than by network lookup: the SSOT repo's own workflows +declare their ``runs-on`` inline, so this same rule, run against +``scitex-ai/.github``, fires on a dirty shared workflow at its source. The +callee is audited where it lives; the caller inherits a runner that is +already known-good. + +A caller of a reusable workflow from an **unrecognised** owner is flagged +(``unverifiable``): we cannot vouch for a third party's runner, and silently +trusting it would reopen the hole. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +import yaml + +#: Runner-image prefixes GitHub hosts. Anything matching these is a violation. +#: Covers `ubuntu-latest`, `ubuntu-24.04`, `macos-14`, `windows-2022`, and the +#: large/arm variants (`ubuntu-22.04-arm`, `macos-13-xlarge`, ...). +_HOSTED_RE = re.compile(r"^(ubuntu|macos|windows)-", re.IGNORECASE) + +#: Owners whose reusable workflows we vouch for. Their `runs-on` is audited by +#: this same rule in their own repo, so a caller inherits a verified runner. +_TRUSTED_REUSABLE_OWNERS = frozenset({"scitex-ai"}) + +#: `${{ matrix.os }}` / `${{ inputs.runner }}` / `${{ vars.RUNNER }}` +_RE_EXPR = re.compile(r"\$\{\{\s*([a-zA-Z_][\w.-]*)\s*\}\}") + +#: Fallback scan when YAML will not parse — better a crude finding than none. +_RE_RUNS_ON_LINE = re.compile(r"^\s*runs-on:\s*(.+?)\s*$", re.MULTILINE) + + +def _is_hosted(label: object) -> bool: + """True iff ``label`` names a GitHub-hosted runner image.""" + return isinstance(label, str) and bool(_HOSTED_RE.match(label.strip())) + + +def _as_labels(value: Any) -> list[str]: + """Flatten a ``runs-on`` value into a list of candidate label strings. + + Handles the scalar (``ubuntu-latest``), sequence (``[self-hosted, X64]``) + and ``group:``/``labels:`` mapping forms. + """ + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [v for v in value if isinstance(v, str)] + if isinstance(value, dict): + out: list[str] = [] + for key in ("labels", "group"): + got = value.get(key) + if isinstance(got, str): + out.append(got) + elif isinstance(got, list): + out.extend(v for v in got if isinstance(v, str)) + return out + return [] + + +def _resolve_expr(expr_path: str, job: dict, workflow: dict) -> list[str]: + """Resolve a `${{ ... }}` reference to the values it can take. + + Follows the two indirections that actually occur in our workflows: + + * ``matrix.`` -> ``job.strategy.matrix.`` (a list of images) + * ``inputs.`` -> ``on.workflow_call.inputs..default`` + + Anything else resolves to ``[]`` (unknown), which is reported separately + rather than assumed innocent — an unresolvable runner is not a pass. + """ + parts = expr_path.split(".") + if len(parts) != 2: + return [] + scope, key = parts + + if scope == "matrix": + matrix = (job.get("strategy") or {}).get("matrix") or {} + if isinstance(matrix, dict): + got = matrix.get(key) + if isinstance(got, list): + return [v for v in got if isinstance(v, str)] + if isinstance(got, str): + return [got] + # `matrix.include` entries can also carry the key + include = matrix.get("include") + if isinstance(include, list): + return [ + e[key] + for e in include + if isinstance(e, dict) and isinstance(e.get(key), str) + ] + return [] + + if scope == "inputs": + # `on:` is parsed by YAML as the boolean True — check both spellings. + on_block = workflow.get("on") or workflow.get(True) or {} + if isinstance(on_block, dict): + call = on_block.get("workflow_call") or {} + if isinstance(call, dict): + inputs = call.get("inputs") or {} + if isinstance(inputs, dict): + spec = inputs.get(key) or {} + if isinstance(spec, dict) and isinstance(spec.get("default"), str): + return [spec["default"]] + return [] + + return [] + + +def _effective_labels( + runs_on: Any, job: dict, workflow: dict +) -> tuple[list[str], list[str]]: + """Return ``(concrete_labels, unresolved_exprs)`` for one job's ``runs-on``.""" + concrete: list[str] = [] + unresolved: list[str] = [] + for label in _as_labels(runs_on): + m = _RE_EXPR.search(label) + if not m: + concrete.append(label) + continue + resolved = _resolve_expr(m.group(1), job, workflow) + if resolved: + concrete.extend(resolved) + else: + unresolved.append(label) + return concrete, unresolved + + +def _reusable_owner(uses: str) -> str | None: + """Owner of a reusable-workflow ``uses:`` ref, or None if it is local.""" + if not isinstance(uses, str) or uses.startswith("./"): + return None + return uses.split("/", 1)[0] if "/" in uses else None + + +def check_ps169_hosted_runners(repo: Path, violation_cls: type, out: list) -> None: + """Append PS-169 violations for any GitHub-hosted runner under `.github/`.""" + wf_dir = repo / ".github" / "workflows" + if not wf_dir.is_dir(): + return + + for path in sorted(wf_dir.iterdir()): + if not path.is_file() or path.suffix not in {".yml", ".yaml"}: + continue + rel = str(path.relative_to(repo)) + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + + try: + doc = yaml.safe_load(text) + except yaml.YAMLError: + doc = None + + # Fallback: unparseable YAML still gets a literal scan, so a broken + # file cannot smuggle a hosted runner past the gate. + if not isinstance(doc, dict): + for raw in _RE_RUNS_ON_LINE.findall(text): + if _HOSTED_RE.search(raw.strip().lstrip("[").strip()): + out.append( + violation_cls( + "PS-169", + rel, + f"GitHub-hosted runner `{raw.strip()}` (file does not " + "parse as YAML; scanned literally). Use the Spartan " + "pool: `runs-on: [self-hosted, Linux, X64, spartan-cpu]`.", + ) + ) + continue + + jobs = doc.get("jobs") or {} + if not isinstance(jobs, dict): + continue + + for job_id, job in jobs.items(): + if not isinstance(job, dict): + continue + + # A caller stub has no `runs-on` — its runner comes from the callee. + uses = job.get("uses") + if uses and "runs-on" not in job: + owner = _reusable_owner(uses) + if owner is not None and owner not in _TRUSTED_REUSABLE_OWNERS: + out.append( + violation_cls( + "PS-169", + rel, + f"job `{job_id}` calls reusable workflow `{uses}` from " + f"untrusted owner `{owner}` — its effective runner " + "cannot be verified. Call a `scitex-ai/.github` " + "reusable workflow, or declare a Spartan `runs-on`.", + ) + ) + continue + + if "runs-on" not in job: + continue + + concrete, unresolved = _effective_labels(job["runs-on"], job, doc) + + for label in concrete: + if _is_hosted(label): + via = "" + if _RE_EXPR.search(str(job["runs-on"])): + via = f" (via `{job['runs-on']}`)" + out.append( + violation_cls( + "PS-169", + rel, + f"job `{job_id}` runs on GitHub-hosted runner " + f"`{label}`{via} — forbidden without exception. Use " + "the Spartan pool: `runs-on: [self-hosted, Linux, " + "X64, spartan-cpu]`. If Spartan cannot run this " + "job, fix Spartan (scitex-hpc) — never fall back.", + ) + ) + + for expr in unresolved: + out.append( + violation_cls( + "PS-169", + rel, + f"job `{job_id}` has an unresolvable runner `{expr}` — " + "its effective runner cannot be verified, so it cannot " + "be proven Spartan-only. Declare `runs-on: [self-hosted, " + "Linux, X64, spartan-cpu]` explicitly.", + ) + ) diff --git a/src/scitex_dev/_cli/audit/_project/_extra_rules.py b/src/scitex_dev/_cli/audit/_project/_extra_rules.py index 1caf5f85..7b617a55 100755 --- a/src/scitex_dev/_cli/audit/_project/_extra_rules.py +++ b/src/scitex_dev/_cli/audit/_project/_extra_rules.py @@ -105,6 +105,30 @@ "E", "secret-env-prefix-missing", ), + ( + "PS-169", + "§1", + ( + "GitHub-hosted runner in `.github/workflows/` — FORBIDDEN, no " + "exceptions (operator mandate 2026-07-14). Every SciTeX job runs " + "on the self-hosted Spartan pool: `runs-on: [self-hosted, Linux, " + "X64, spartan-cpu]`. If Spartan cannot run the job, fix Spartan " + "(scitex-hpc) — never fall back to a hosted runner. Resolves the " + "EFFECTIVE runner, so a hosted image reached via `runs-on: " + "${{ matrix.os }}` or a `workflow_call` input default is caught " + "too; an UNRESOLVABLE runner is also a violation (it cannot be " + "proven Spartan-only). Callers of reusable workflows from owners " + "outside `scitex-ai` are flagged — their runner cannot be " + "vouched for. Consuming the shared workflow is not by itself " + "compliance: the SSOT's own `runs-on` is audited by this rule in " + "`scitex-ai/.github`, so a dirty shared workflow fires at source. " + "Severity E — this is the ONLY enforcement that exists, since " + "blocking hosted runners at the org level requires GitHub " + "Enterprise and `scitex-ai` is on the Free plan." + ), + "E", + "hosted-runner-forbidden", + ), ( "PS-148", "§3", diff --git a/tests/scitex_dev/_cli/audit/_project/test__check_hosted_runners.py b/tests/scitex_dev/_cli/audit/_project/test__check_hosted_runners.py new file mode 100755 index 00000000..bf780f26 --- /dev/null +++ b/tests/scitex_dev/_cli/audit/_project/test__check_hosted_runners.py @@ -0,0 +1,205 @@ +"""PS-169 — GitHub-hosted runners are forbidden (operator mandate 2026-07-14). + +The rule is the ONLY enforcement that exists (GitHub cannot block hosted +runners below the Enterprise plan), so its coverage is load-bearing: every +form the runner can take is exercised here, including the indirect ones a +naive `grep ubuntu-latest` would miss. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scitex_dev._cli.audit._project._check_hosted_runners import ( + check_ps169_hosted_runners, +) + + +class _Violation: + """Stand-in for the auditor's Violation record (real class, not a mock).""" + + def __init__(self, code: str, where: str, message: str) -> None: + self.code = code + self.where = where + self.message = message + + +def _run_rule_on(repo: Path) -> list[_Violation]: + """Run PS-169 against a repo root and return its violations.""" + out: list[_Violation] = [] + check_ps169_hosted_runners(repo, _Violation, out) + return out + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + """An empty repo skeleton with a `.github/workflows/` directory.""" + (tmp_path / ".github" / "workflows").mkdir(parents=True) + return tmp_path + + +def _write_workflow(repo: Path, name: str, body: str) -> None: + (repo / ".github" / "workflows" / name).write_text(body, encoding="utf-8") + + +def test_direct_hosted_runner_is_reported(repo: Path) -> None: + # Arrange + _write_workflow( + repo, + "direct.yml", + "name: d\non: [push]\njobs:\n tests:\n runs-on: ubuntu-latest\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 1 + + +def test_hosted_runner_in_list_form_is_reported(repo: Path) -> None: + # Arrange + _write_workflow( + repo, + "listform.yml", + "name: l\non: [push]\njobs:\n b:\n runs-on: [ubuntu-22.04]\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 1 + + +def test_hosted_runner_reached_via_matrix_expression_is_reported(repo: Path) -> None: + # Arrange — the case a literal `runs-on:` grep cannot see: the line says + # `${{ matrix.os }}`, and the hosted image hides in strategy.matrix. + _write_workflow( + repo, + "matrix.yml", + "name: m\non: [push]\njobs:\n tests:\n" + " strategy:\n matrix:\n os: [ubuntu-latest, macos-14]\n" + " runs-on: ${{ matrix.os }}\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 2 + + +def test_hosted_runner_from_workflow_call_input_default_is_reported(repo: Path) -> None: + # Arrange + _write_workflow( + repo, + "called.yml", + "name: c\non:\n workflow_call:\n inputs:\n runner:\n" + " type: string\n default: ubuntu-latest\n" + "jobs:\n tests:\n runs-on: ${{ inputs.runner }}\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 1 + + +def test_unresolvable_runner_expression_is_reported(repo: Path) -> None: + # Arrange — an unverifiable runner cannot be proven Spartan-only, so it is + # a violation rather than an assumed pass. + _write_workflow( + repo, + "unknown.yml", + "name: u\non: [push]\njobs:\n t:\n runs-on: ${{ vars.RUNNER }}\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 1 + + +def test_spartan_self_hosted_runner_is_not_reported(repo: Path) -> None: + # Arrange + _write_workflow( + repo, + "clean.yml", + "name: c\non: [push]\njobs:\n t:\n" + " runs-on: [self-hosted, Linux, X64, spartan-cpu]\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert found == [] + + +def test_caller_of_trusted_scitex_reusable_workflow_is_not_reported(repo: Path) -> None: + # Arrange — the callee's own runs-on is audited by this rule in + # scitex-ai/.github, so the caller inherits a verified runner. + _write_workflow( + repo, + "caller.yml", + "name: c\non: [push]\njobs:\n call:\n" + " uses: scitex-ai/.github/.github/workflows/pytest-matrix.yml@main\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert found == [] + + +def test_caller_of_untrusted_reusable_workflow_is_reported(repo: Path) -> None: + # Arrange — a third party's effective runner cannot be vouched for. + _write_workflow( + repo, + "untrusted.yml", + "name: u\non: [push]\njobs:\n call:\n" + " uses: some-vendor/ci/.github/workflows/build.yml@v1\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 1 + + +def test_hosted_runner_in_unparseable_yaml_is_still_reported(repo: Path) -> None: + # Arrange — a broken file must not smuggle a hosted runner past the gate. + _write_workflow( + repo, + "broken.yml", + "name: b\non: [push\njobs:\n t:\n runs-on: ubuntu-latest\n : : :\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert len(found) == 1 + + +def test_repo_without_workflows_directory_is_not_reported(tmp_path: Path) -> None: + # Arrange — no .github/workflows/ at all. + # Act + found = _run_rule_on(tmp_path) + # Assert + assert found == [] + + +def test_violation_message_names_the_spartan_remedy(repo: Path) -> None: + # Arrange + _write_workflow( + repo, + "direct.yml", + "name: d\non: [push]\njobs:\n tests:\n runs-on: ubuntu-latest\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert "spartan-cpu" in found[0].message + + +def test_violation_is_tagged_with_the_ps169_code(repo: Path) -> None: + # Arrange + _write_workflow( + repo, + "direct.yml", + "name: d\non: [push]\njobs:\n tests:\n runs-on: ubuntu-latest\n", + ) + # Act + found = _run_rule_on(repo) + # Assert + assert found[0].code == "PS-169"