Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 68 additions & 4 deletions .github/workflows/opencode-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,69 @@ jobs:
rm -f "$log_file"
}

run_and_capture_python_coverage() {
local label="$1"
local project_dir="$2"
shift 2
local log_file classification_file
log_file="$(mktemp)"
classification_file="$(mktemp)"
append "### ${label}"
append ""
append '```text'
append_command "$@"
set +e
timeout --kill-after=20 900 setpriv \
--reuid "$OPENCODE_SANDBOX_UID" \
--regid "$OPENCODE_SANDBOX_GID" \
--clear-groups \
env \
-u ACTIONS_ID_TOKEN_REQUEST_TOKEN \
-u ACTIONS_ID_TOKEN_REQUEST_URL \
-u ACTIONS_RUNTIME_TOKEN \
-u GH_TOKEN \
-u GITHUB_TOKEN \
GITHUB_ENV=/dev/null \
GITHUB_PATH=/dev/null \
GITHUB_OUTPUT=/dev/null \
GITHUB_STEP_SUMMARY=/dev/null \
BASH_ENV=/dev/null \
UV_NO_BUILD=1 \
HOME=/work/.opencode-sandbox-home \
XDG_CACHE_HOME=/work/.opencode-sandbox-cache \
CARGO_HOME=/work/.opencode-sandbox-home/.cargo \
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
"$@" >"$log_file" 2>&1
local rc=$?
set -e
emit_captured_log "$log_file"
append '```'
append ""
if [ "$rc" -eq 0 ]; then
append "- Result: PASS"
elif env -i \
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
HOME=/tmp \
GIT_CONFIG_NOSYSTEM=1 \
GIT_CONFIG_GLOBAL=/dev/null \
python3 -I "${GITHUB_WORKSPACE}/scripts/ci/python_coverage_dependency_guard.py" \
--repo-root "$COVERAGE_SOURCE_WORKDIR" \
--base-sha "$PR_BASE_SHA" \
--project-dir "$project_dir" \
--pytest-exit "$rc" \
--log-file "$log_file" >"$classification_file" 2>&1; then
cat "$classification_file" >>"$summary_file"
append "- Result: DEFERRED"
append "- Reason: the protected base branch declares the unavailable dependency; repository-native required checks remain mandatory before merge."
else
cat "$classification_file" >>"$summary_file"
append "- Result: FAIL (exit ${rc})"
failures=$((failures + 1))
fi
append ""
rm -f "$log_file" "$classification_file"
}

run_and_capture_advisory() {
local label="$1"
shift
Expand Down Expand Up @@ -886,23 +949,24 @@ jobs:
if [ -n "$configured_commands_json" ]; then
while IFS= read -r configured_command_json; do
[ -n "$configured_command_json" ] || continue
run_and_capture "Python configured CI test suite (${project_dir})" \
run_and_capture_python_coverage "Python configured CI test suite (${project_dir})" "$project_dir" \
python3 "${GITHUB_WORKSPACE}/scripts/ci/safe_pytest_command.py" execute \
--project-dir "$project_dir" \
--command-json "$configured_command_json"
done <<<"$configured_commands_json"
else
run_and_capture "Python coverage with missing-line report (${project_dir})" \
run_and_capture_python_coverage "Python coverage with missing-line report (${project_dir})" "$project_dir" \
bash -c 'cd "$1" && PYTHONPATH=. python3 -m coverage run -m pytest tests && python3 -m coverage report --show-missing' bash "$project_dir"
fi
done < <(tracked_python_projects_with_tests)

if [ "$measured_projects" -eq 0 ]; then
if has_tracked_files '*.py'; then
run_and_capture "Python coverage with missing-line report" \
run_and_capture_python_coverage "Python coverage with missing-line report" "." \
bash -c 'PYTHONPATH=. python3 -m coverage run -m pytest && python3 -m coverage report --show-missing'
elif python3 -I -c 'import pytest_cov' >/dev/null 2>&1; then
run_and_capture "Python pytest-cov coverage" python3 -m pytest --cov=. --cov-report=term-missing
run_and_capture_python_coverage "Python pytest-cov coverage" "." \
python3 -m pytest --cov=. --cov-report=term-missing
else
append "### Python test suite"
append ""
Expand Down
259 changes: 259 additions & 0 deletions scripts/ci/python_coverage_dependency_guard.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
"""Classify networkless pytest collection failures against trusted base metadata.

The central coverage sandbox deliberately does not resolve pull-request-selected
Python dependency manifests. A repository test suite can therefore stop at
collection time when a dependency that is already declared on the protected
base branch is absent from the small central tool image. This helper permits
that one state to be reported as deferred to repository-native required checks.
All undeclared imports and every non-collection test failure remain blocking.
"""

from __future__ import annotations

import argparse
import os
from pathlib import Path, PurePosixPath
import re
import subprocess
import tempfile
import tomllib


SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$")
MISSING_MODULE_RE = re.compile(
r"(?m)^E\s+ModuleNotFoundError:\s+No module named ['\"]([A-Za-z0-9_.-]+)['\"]\s*$"
)
TERMINAL_EXCEPTION_RE = re.compile(
r"(?m)^E\s+([A-Za-z_][A-Za-z0-9_.]*(?:Error|Exception|Exit|Interrupt)):\s*"
)
REQUIREMENT_NAME_RE = re.compile(r"^([A-Za-z0-9][A-Za-z0-9._-]*)")
MAX_LOG_BYTES = 10 * 1024 * 1024

# Import roots and distribution names are not always identical. Keep this
# intentionally small and explicit; an unknown mapping stays fail-closed.
IMPORT_DISTRIBUTION_ALIASES = {
"bs4": "beautifulsoup4",
"cv2": "opencv-python",
"dateutil": "python-dateutil",
"jwt": "pyjwt",
"pil": "pillow",
"sklearn": "scikit-learn",
"yaml": "pyyaml",
}


def normalize_distribution(name: str) -> str:
"""Return a PEP 503-style normalized distribution name."""
return re.sub(r"[-_.]+", "-", name.strip().lower())


def validate_project_dir(value: str) -> str:
"""Return a safe repository-relative project directory."""
if value in {"", "."}:
return "."
path = PurePosixPath(value)
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
raise ValueError("project directory must be a safe repository-relative path")
return path.as_posix()


def git_show(repo_root: Path, base_sha: str, relative_path: str) -> str | None:
"""Read one file from the validated base commit without invoking hooks."""
completed = subprocess.run(
[
"git",
"-C",
str(repo_root),
"-c",
f"safe.directory={repo_root}",
"-c",
"core.hooksPath=/dev/null",
"-c",
"core.fsmonitor=false",
"show",
f"{base_sha}:{relative_path}",
],
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
env={
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
"HOME": tempfile.gettempdir(),
"GIT_CONFIG_NOSYSTEM": "1",
"GIT_CONFIG_GLOBAL": "/dev/null",
},
)
if completed.returncode != 0:
return None
return completed.stdout


def requirement_names(text: str) -> set[str]:
"""Extract direct distribution names from a requirements-style file."""
names: set[str] = set()
for raw_line in text.splitlines():
line = raw_line.strip()
if not line or line.startswith(("#", "-")):
continue
match = REQUIREMENT_NAME_RE.match(line)
if match:
names.add(normalize_distribution(match.group(1)))
return names


def pyproject_names(text: str) -> set[str]:
"""Extract declared distributions from supported pyproject tables."""
try:
document = tomllib.loads(text)
except tomllib.TOMLDecodeError:
return set()

names: set[str] = set()
project = document.get("project") or {}
for dependency in project.get("dependencies") or []:
match = REQUIREMENT_NAME_RE.match(str(dependency).strip())
if match:
names.add(normalize_distribution(match.group(1)))
for dependencies in (project.get("optional-dependencies") or {}).values():
for dependency in dependencies or []:
match = REQUIREMENT_NAME_RE.match(str(dependency).strip())
if match:
names.add(normalize_distribution(match.group(1)))

poetry_dependencies = ((document.get("tool") or {}).get("poetry") or {}).get(
"dependencies"
) or {}
names.update(
normalize_distribution(name)
for name in poetry_dependencies
if normalize_distribution(name) != "python"
)
for dependencies in (document.get("dependency-groups") or {}).values():
for dependency in dependencies or []:
match = REQUIREMENT_NAME_RE.match(str(dependency).strip())
if match:
names.add(normalize_distribution(match.group(1)))
return names


def base_declared_distributions(
repo_root: Path, base_sha: str, project_dir: str
) -> tuple[set[str], list[str]]:
"""Return distributions and manifests read only from the base commit."""
directories = [project_dir]
if project_dir != ".":
directories.append(".")
declared: set[str] = set()
manifests: list[str] = []
for directory in directories:
for filename in (
"requirements.txt",
"requirements-hashes.txt",
"pyproject.toml",
):
relative_path = filename if directory == "." else f"{directory}/{filename}"
text = git_show(repo_root, base_sha, relative_path)
if text is None:
continue
manifests.append(relative_path)
if filename == "pyproject.toml":
declared.update(pyproject_names(text))
else:
declared.update(requirement_names(text))
return declared, manifests


def missing_distributions(log_text: str) -> set[str]:
"""Return normalized missing distributions named by pytest collection."""
distributions: set[str] = set()
for module in MISSING_MODULE_RE.findall(log_text):
import_root = normalize_distribution(module.split(".", 1)[0])
distributions.add(IMPORT_DISTRIBUTION_ALIASES.get(import_root, import_root))
return distributions


def classify(
repo_root: Path,
base_sha: str,
project_dir: str,
pytest_exit: int,
log_file: Path,
) -> tuple[bool, str]:
"""Classify a failure, returning whether repository-native CI may own it."""
if pytest_exit != 4:
return False, f"pytest exit {pytest_exit} is not a collection failure"
if log_file.is_symlink() or not log_file.is_file():
return False, "pytest log is not a regular file"
if log_file.stat().st_size > MAX_LOG_BYTES:
return False, "pytest log exceeds the classifier size limit"

log_text = log_file.read_text(encoding="utf-8", errors="replace")
missing = missing_distributions(log_text)
if not missing:
return False, "collection failure did not name a missing Python module"
terminal_exceptions = {
exception.rsplit(".", 1)[-1]
for exception in TERMINAL_EXCEPTION_RE.findall(log_text)
}
unexpected_exceptions = sorted(terminal_exceptions - {"ModuleNotFoundError"})
if unexpected_exceptions:
return False, "collection failure includes other exceptions: " + ", ".join(
unexpected_exceptions
)
declared, manifests = base_declared_distributions(repo_root, base_sha, project_dir)
undeclared = sorted(missing - declared)
if undeclared:
return (
False,
"missing modules are not base-declared dependencies: "
+ ", ".join(undeclared),
)
return True, (
"networkless central coverage lacks base-declared dependencies "
f"{', '.join(sorted(missing))}; trusted manifests: {', '.join(manifests)}; "
"repository-native required checks remain authoritative"
)


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
"""Parse and validate command-line arguments."""
parser = argparse.ArgumentParser()
parser.add_argument("--repo-root", type=Path, required=True)
parser.add_argument("--base-sha", required=True)
parser.add_argument("--project-dir", required=True)
parser.add_argument("--pytest-exit", type=int, required=True)
parser.add_argument("--log-file", type=Path, required=True)
args = parser.parse_args(argv)
if not SHA_RE.fullmatch(args.base_sha):
parser.error("--base-sha must be a 40-character git SHA")
try:
args.project_dir = validate_project_dir(args.project_dir)
except ValueError as exc:
parser.error(str(exc))
args.repo_root = args.repo_root.resolve()
if not (args.repo_root / ".git").exists():
parser.error("--repo-root must name a Git worktree")
# Preserve the final path component so classify() can reject a symlink
# rather than silently following it to an attacker-selected file.
args.log_file = Path(os.path.abspath(args.log_file))
return args


def main(argv: list[str] | None = None) -> int:
"""Print a bounded classification and return zero only for safe deferral."""
args = parse_args(argv)
deferred, reason = classify(
args.repo_root,
args.base_sha,
args.project_dir,
args.pytest_exit,
args.log_file,
)
status = "DEFERRED" if deferred else "BLOCKING"
print(f"Python coverage dependency classification: {status}: {reason}")
return 0 if deferred else 1


if __name__ == "__main__":
raise SystemExit(main())
3 changes: 3 additions & 0 deletions scripts/ci/test_strix_quick_gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,9 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
assert_file_contains "$workflow_file" 'python3 -m coverage report --show-missing' "opencode coverage preserves the missing-line report with the trusted toolchain"
assert_file_contains "$workflow_file" 'cd "$1" && PYTHONPATH=. python3 -m pytest tests/test_docstrings.py' "opencode docstring tests use the trusted preinstalled pytest"
assert_file_contains "$workflow_file" "missing project imports fail in pytest" "unavailable project dependencies fail closed with their import error"
assert_file_contains "$workflow_file" "python_coverage_dependency_guard.py" "coverage defers only missing dependencies declared by the protected base"
assert_file_contains "$workflow_file" '--pytest-exit "$rc"' "coverage dependency deferral is limited to pytest collection failures"
assert_file_contains "$workflow_file" "repository-native required checks remain mandatory before merge" "coverage dependency deferral preserves repository-native merge gates"
assert_file_contains "$workflow_file" "JavaScript/TypeScript dependencies (npm ci, lifecycle hooks disabled)" "opencode coverage evidence installs npm workspace dependencies without lifecycle hooks before JS coverage"
assert_file_contains "$workflow_file" "coverage/coverage-summary.json" "opencode coverage evidence reads JS coverage summaries instead of trusting test exit codes"
assert_file_contains "$workflow_file" "coverage/coverage-final.json" "opencode coverage evidence supports Vitest Istanbul final coverage files"
Expand Down
16 changes: 11 additions & 5 deletions tests/test_opencode_agent_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,12 @@ def test_opencode_python_coverage_never_resolves_pr_dependency_manifests():
assert "python3 -m coverage run -m pytest tests" in measure
assert "python3 -m coverage report --show-missing" in measure
assert "python3 -m pytest tests/test_docstrings.py" in measure
assert "run_and_capture_python_coverage()" in measure
assert "scripts/ci/python_coverage_dependency_guard.py" in measure
assert '--base-sha "$PR_BASE_SHA"' in measure
assert '--pytest-exit "$rc"' in measure
assert "repository-native required checks remain mandatory before merge" in measure
assert "- Result: DEFERRED" in measure


def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm():
Expand Down Expand Up @@ -1572,11 +1578,11 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed():
assert syntax_step < measure_step
assert "\n - name:" not in measure.split("\n run: |", 1)[1]
assert 'UV_NO_BUILD: "1"' in measure
assert measure.count("GITHUB_ENV=/dev/null") == 2
assert measure.count("GITHUB_PATH=/dev/null") == 2
assert measure.count("GITHUB_OUTPUT=/dev/null") == 2
assert measure.count("GITHUB_STEP_SUMMARY=/dev/null") == 2
assert measure.count("BASH_ENV=/dev/null") == 2
assert measure.count("GITHUB_ENV=/dev/null") == 3
assert measure.count("GITHUB_PATH=/dev/null") == 3
assert measure.count("GITHUB_OUTPUT=/dev/null") == 3
assert measure.count("GITHUB_STEP_SUMMARY=/dev/null") == 3
assert measure.count("BASH_ENV=/dev/null") == 3
assert "uv sync --project" not in measure
assert "uv run --no-project" not in measure
assert "uv run --no-build" not in measure
Expand Down
Loading
Loading