From bf51e3f3fcdb53d2a10167af61fefdb657e34f7e Mon Sep 17 00:00:00 2001 From: Michael Kantor <6068672+kantorcodes@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:41:33 -0400 Subject: [PATCH 1/2] ci: enforce Codex plugin contribution scanner gate Signed-off-by: Michael Kantor <6068672+kantorcodes@users.noreply.github.com> --- .github/workflows/sweep-open-prs.yml | 153 +++++ .github/workflows/validate-contribution.yml | 70 ++ CONTRIBUTING.md | 15 +- scripts/publish-open-pr-checks.py | 331 ++++++++++ scripts/validate-contribution.py | 673 ++++++++++++++++++++ 5 files changed, 1240 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/sweep-open-prs.yml create mode 100644 .github/workflows/validate-contribution.yml create mode 100644 scripts/publish-open-pr-checks.py create mode 100644 scripts/validate-contribution.py diff --git a/.github/workflows/sweep-open-prs.yml b/.github/workflows/sweep-open-prs.yml new file mode 100644 index 000000000..051595597 --- /dev/null +++ b/.github/workflows/sweep-open-prs.yml @@ -0,0 +1,153 @@ +name: Sweep Open Codex Plugin Contributions + +on: + pull_request_target: + types: + - opened + - synchronize + - reopened + - ready_for_review + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + pr_number: + description: "Optional open PR number; omit to sweep every open PR" + required: false + type: string + +permissions: + contents: read + pull-requests: read + +jobs: + discover: + name: Validate open contribution requirements + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.export.outputs.matrix }} + has_failures: ${{ steps.export.outputs.has_failures }} + results: ${{ steps.export.outputs.results }} + steps: + - name: Check out validator + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install validator dependencies + run: python3 -m pip install --disable-pip-version-check --no-input "PyYAML==6.0.2" + - name: Validate open PRs + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pr_number }} + MATRIX_FILE: ${{ runner.temp }}/open-pr-matrix.json + REPORT_FILE: ${{ runner.temp }}/open-pr-report.md + STATUS_FILE: ${{ runner.temp }}/open-pr-status.json + run: | + validator_args=( + --open-prs + --repository "$GITHUB_REPOSITORY" + --matrix-output "$MATRIX_FILE" + --report-output "$REPORT_FILE" + --status-output "$STATUS_FILE" + ) + if [[ -n "$PR_NUMBER" ]]; then + validator_args+=(--pr-number "$PR_NUMBER") + fi + python3 scripts/validate-contribution.py "${validator_args[@]}" + - name: Export validation results + id: export + env: + MATRIX_FILE: ${{ runner.temp }}/open-pr-matrix.json + STATUS_FILE: ${{ runner.temp }}/open-pr-status.json + run: | + echo "matrix=$(cat \"$MATRIX_FILE\")" >> "$GITHUB_OUTPUT" + echo "has_failures=$(jq -r '.has_failures' \"$STATUS_FILE\")" >> "$GITHUB_OUTPUT" + echo "results=$(jq -c '.results' \"$STATUS_FILE\")" >> "$GITHUB_OUTPUT" + + scan: + name: Scan PR #${{ matrix.contribution.pr_number }} source + needs: discover + if: needs.discover.outputs.matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + contribution: ${{ fromJSON(needs.discover.outputs.matrix) }} + steps: + - name: Check out contributed repository + env: + CONTRIBUTION_URL: https://github.com/${{ matrix.contribution.owner }}/${{ matrix.contribution.repo }}.git + run: git clone --depth 1 "$CONTRIBUTION_URL" "$RUNNER_TEMP/contributed" + - name: Run HOL AI Plugin Scanner + uses: hashgraph-online/ai-plugin-scanner-action@55616c962cf86368423f7673b2ecdfdbe613d1af # v1.2.515 + with: + plugin_dir: ${{ runner.temp }}/contributed + mode: scan + min_score: 80 + fail_on_severity: high + pr_comment: off + + gate: + name: Open Codex contribution gate + needs: + - discover + - scan + if: always() + runs-on: ubuntu-latest + steps: + - name: Report sweep result + env: + DISCOVER_RESULT: ${{ needs.discover.result }} + HAS_VALIDATION_FAILURES: ${{ needs.discover.outputs.has_failures }} + SCAN_RESULT: ${{ needs.scan.result }} + run: | + if [[ "$DISCOVER_RESULT" != "success" ]]; then + echo "Open-PR discovery failed." + exit 1 + fi + if [[ "$HAS_VALIDATION_FAILURES" == "true" ]]; then + echo "One or more open contributions are missing required scanner CI." + exit 1 + fi + if [[ "$SCAN_RESULT" != "success" && "$SCAN_RESULT" != "skipped" ]]; then + echo "One or more source-repository scanner jobs failed." + exit 1 + fi + echo "All checked open contributions satisfy the scanner gate." + + publish: + name: Publish per-PR contribution checks + needs: + - discover + - scan + - gate + if: always() + runs-on: ubuntu-latest + permissions: + contents: read + checks: write + issues: write + pull-requests: write + steps: + - name: Check out validator + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 1 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Publish checks on PR heads + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} + OPEN_PR_RESULTS: ${{ needs.discover.outputs.results || '[]' }} + SWEEP_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: python3 scripts/publish-open-pr-checks.py diff --git a/.github/workflows/validate-contribution.yml b/.github/workflows/validate-contribution.yml new file mode 100644 index 000000000..0f3308e7b --- /dev/null +++ b/.github/workflows/validate-contribution.yml @@ -0,0 +1,70 @@ +name: Validate Codex Plugin Contributions + +on: + pull_request: + paths: + - "README.md" + - "CONTRIBUTING.md" + - "SCANNER_GUIDE.md" + - "scripts/check-alphabetical.py" + - "scripts/validate-contribution.py" + - ".github/workflows/validate-contribution.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: Check contribution requirements + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.entries.outputs.matrix }} + steps: + - name: Check out catalog + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install validator dependencies + run: python3 -m pip install --disable-pip-version-check --no-input "PyYAML==6.0.2" + - name: Validate changed entries and source scanner CI + env: + BASE_REF: ${{ github.event_name == 'pull_request' && format('origin/{0}', github.base_ref) || 'origin/main' }} + GITHUB_TOKEN: ${{ github.token }} + run: | + python3 scripts/check-alphabetical.py README.md + python3 scripts/validate-contribution.py \ + --base-ref "$BASE_REF" \ + --matrix-output "$RUNNER_TEMP/contributions.json" + - name: Export scanner matrix + id: entries + env: + MATRIX_FILE: ${{ runner.temp }}/contributions.json + run: echo "matrix=$(cat \"$MATRIX_FILE\")" >> "$GITHUB_OUTPUT" + + scan: + name: Scan ${{ matrix.contribution.owner }}/${{ matrix.contribution.repo }} + needs: validate + if: needs.validate.outputs.matrix != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + contribution: ${{ fromJSON(needs.validate.outputs.matrix) }} + steps: + - name: Check out contributed repository + env: + CONTRIBUTION_URL: https://github.com/${{ matrix.contribution.owner }}/${{ matrix.contribution.repo }}.git + run: git clone --depth 1 "$CONTRIBUTION_URL" "$RUNNER_TEMP/contributed" + - name: Run HOL AI Plugin Scanner + uses: hashgraph-online/ai-plugin-scanner-action@55616c962cf86368423f7673b2ecdfdbe613d1af # v1.2.515 + with: + plugin_dir: ${{ runner.temp }}/contributed + mode: scan + min_score: 80 + fail_on_severity: high + pr_comment: off diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a66e054a..804466d25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -354,13 +354,24 @@ Before submitting, verify: ## CI Checks -All PRs to this repo are automatically validated. The CI will check: +All PRs to this repo are automatically validated. The contribution gate runs +on the PR target event, so fork PRs do not wait for first-time workflow +approval. It checks the source repository and publishes one status check on +the PR head. When a requirement is missing, the gate updates one idempotent +comment, tags the PR author, and includes the exact scanner workflow and +remediation steps. The check is re-run on every push, reopen, and daily sweep. + +The CI will check: 1. **Alphabetical order** - README entries must be sorted within each section 2. **Plugin manifest** - For new README entries, the generator fetches your source repo and validates `plugin.json`, required fields, and icon presence -3. **Scanner verification** - PR description must include scanner score or CI link +3. **Scanner verification** - The source repo must invoke `hashgraph-online/ai-plugin-scanner-action` on `push` or `pull_request`, and the gate runs the scanner at the documented score/severity thresholds 4. **Markdown links** - All URLs in README must be reachable +If the gate comments on your PR, fix the linked source repository first, push +the change there, then update the README PR if needed. The status check and +comment will refresh automatically; do not post duplicate remediation comments. + ## Getting Help - Scanner docs: [HOL Guard](https://github.com/hashgraph-online/hol-guard) diff --git a/scripts/publish-open-pr-checks.py b/scripts/publish-open-pr-checks.py new file mode 100644 index 000000000..d933d16e3 --- /dev/null +++ b/scripts/publish-open-pr-checks.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Publish the result of an open-PR sweep on each pull request head commit.""" + +from __future__ import annotations + +import json +import os +import re +import sys +from datetime import datetime, timezone +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +API_ROOT = "https://api.github.com" +CHECK_NAME = "Open Codex Plugin Contribution Gate" +COMMENT_MARKER = "" +USER_AGENT = "awesome-codex-plugins-open-pr-sweep" +REQUEST_TIMEOUT_SECONDS = 30 +SCAN_JOB_RE = re.compile(r"Scan PR (?:#(\d+) source|\((\d+),)") +GITHUB_LOGIN_RE = re.compile(r"^[A-Za-z0-9-]{1,39}$") + + +def github_api(repository: str, path: str, token: str, *, method: str = "GET", payload: object | None = None) -> object: + """Call the GitHub REST API with a bounded, JSON-only request.""" + + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "User-Agent": USER_AGENT, + } + data = None + if payload is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(payload, separators=(",", ":")).encode("utf-8") + request = Request( + f"{API_ROOT}/repos/{repository}{path}", + headers=headers, + method=method, + data=data, + ) + try: + with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + return json.loads(response.read().decode("utf-8")) + except HTTPError as error: + detail = error.read().decode("utf-8", errors="replace").strip() + suffix = f": {detail}" if detail else "" + raise RuntimeError(f"GitHub API {method} {path} failed: HTTP {error.code}{suffix}") from error + except (URLError, TimeoutError, OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"GitHub API {method} {path} failed: {error}") from error + + +def scan_conclusions(repository: str, run_id: str, token: str) -> dict[int, list[str]]: + """Return scanner job conclusions grouped by pull request number.""" + + conclusions: dict[int, list[str]] = {} + page = 1 + while True: + payload = github_api(repository, f"/actions/runs/{run_id}/jobs?per_page=100&page={page}", token) + if not isinstance(payload, dict): + raise RuntimeError("GitHub returned an invalid jobs response") + jobs = payload.get("jobs") + if not isinstance(jobs, list): + raise RuntimeError("GitHub returned no jobs list") + for job in jobs: + if not isinstance(job, dict): + continue + name = job.get("name") + conclusion = job.get("conclusion") + if not isinstance(name, str) or not isinstance(conclusion, str): + continue + match = SCAN_JOB_RE.search(name) + if match: + number = match.group(1) or match.group(2) + conclusions.setdefault(int(number), []).append(conclusion) + if len(jobs) < 100: + return conclusions + page += 1 + + +def check_summary(result: dict[str, object], scanner_jobs: dict[int, list[str]]) -> tuple[str, str, str]: + """Map validator/scan output to a check conclusion, title, and summary.""" + + number = result.get("pr_number") + state = result.get("state") + reasons = result.get("failure_reasons") + if not isinstance(number, int) or not isinstance(state, str): + raise RuntimeError("validator returned an invalid PR result") + + if state == "success": + return ( + "success", + "Contribution requirements passed", + "No new Community Plugins entries require validation in this pull request.", + ) + + if state == "failure": + failure_lines = reasons if isinstance(reasons, list) else [] + details = "\n".join(f"- {item}" for item in failure_lines if isinstance(item, str)) + return ( + "failure", + "Contribution requirements failed", + "Required scanner CI is missing or could not be validated.\n\n" + details, + ) + + if state == "scan": + jobs = scanner_jobs.get(number, []) + if jobs and all(conclusion == "success" for conclusion in jobs): + return ( + "success", + "Contribution scan passed", + f"All {len(jobs)} source-repository scanner job(s) passed the contribution gate.", + ) + job_details = ", ".join(jobs) if jobs else "no scanner job was recorded" + return ( + "failure", + "Contribution scan failed", + f"One or more source-repository scanner jobs did not pass: {job_details}.", + ) + + raise RuntimeError(f"unknown validator result state: {state}") + + +def list_issue_comments(repository: str, number: int, token: str) -> list[dict[str, object]]: + """Return all issue comments for a pull request.""" + + comments: list[dict[str, object]] = [] + page = 1 + while True: + payload = github_api( + repository, + f"/issues/{number}/comments?per_page=100&page={page}", + token, + ) + if not isinstance(payload, list): + raise RuntimeError(f"GitHub returned an invalid comments response for PR #{number}") + page_comments = [item for item in payload if isinstance(item, dict)] + comments.extend(page_comments) + if len(payload) < 100: + return comments + page += 1 + + +def remediation_comment( + result: dict[str, object], + conclusion: str, + check_title: str, + summary: str, + run_url: str, +) -> str: + """Build an idempotent contributor-facing remediation comment.""" + + author_login = result.get("author_login") + mention = f"@{author_login}" if isinstance(author_login, str) and GITHUB_LOGIN_RE.fullmatch(author_login) else "the contributor" + if conclusion == "success": + return ( + f"{COMMENT_MARKER}\n\n" + f"✅ **Contribution gate passed.** {mention}, no action is required.\n\n" + f"The previous contribution-gate failure is resolved. " + f"[View the latest sweep]({run_url})." + ) + + if result.get("state") == "failure": + guidance = ( + "1. Add a workflow under `.github/workflows/` in the linked source repository.\n" + "2. Trigger it on both `push` and `pull_request`, and invoke " + "`hashgraph-online/ai-plugin-scanner-action`.\n" + "3. Configure `plugin_dir: \".\"`, `mode: scan`, `min_score: 80`, and " + "`fail_on_severity: high`.\n" + "4. Keep `.codex-plugin/plugin.json`, `README.md`, `SECURITY.md`, " + "`LICENSE`, and a dependency lockfile in the source repository." + ) + else: + guidance = ( + "1. Run `pipx run plugin-scanner lint .`.\n" + "2. Run `pipx run plugin-scanner verify . --format text`.\n" + "3. Fix all critical/high findings and reach a score of at least 80.\n" + "4. Confirm the source repository workflow passes, then push the fixes " + "and rerun the workflow." + ) + + return f"""{COMMENT_MARKER} + +{mention} — this pull request needs updates before it can be merged. + +### {check_title} +{summary} + +### How to fix it +{guidance} + +See the repository's [contribution requirements](https://github.com/hashgraph-online/awesome-codex-plugins/blob/main/CONTRIBUTING.md) and [scanner guide](https://github.com/hashgraph-online/awesome-codex-plugins/blob/main/SCANNER_GUIDE.md). + +After pushing the changes, this check and comment will update automatically: {run_url} +""" + + +def upsert_remediation_comment( + repository: str, + result: dict[str, object], + conclusion: str, + check_title: str, + summary: str, + run_url: str, + token: str, +) -> None: + """Create or update the single contribution-gate comment for a PR.""" + + number = result.get("pr_number") + if not isinstance(number, int): + raise RuntimeError("validator returned an invalid PR number") + comments = list_issue_comments(repository, number, token) + existing = next( + ( + comment + for comment in comments + if isinstance(comment.get("body"), str) and COMMENT_MARKER in comment["body"] + ), + None, + ) + if conclusion == "success" and existing is None: + return + + body = remediation_comment(result, conclusion, check_title, summary, run_url) + if existing is not None and isinstance(existing.get("id"), int): + github_api( + repository, + f"/issues/comments/{existing['id']}", + token, + method="PATCH", + payload={"body": body}, + ) + print(f"Updated contribution-gate comment for PR #{number}") + return + + github_api( + repository, + f"/issues/{number}/comments", + token, + method="POST", + payload={"body": body}, + ) + print(f"Posted contribution-gate comment for PR #{number}") + + +def publish_check( + repository: str, + result: dict[str, object], + scanner_jobs: dict[int, list[str]], + run_url: str, + token: str, +) -> None: + """Create or update the check run for one pull request head commit.""" + + head_sha = result.get("head_sha") + number = result.get("pr_number") + title = result.get("title") + if not isinstance(head_sha, str) or not re.fullmatch(r"[0-9a-fA-F]{40}", head_sha): + raise RuntimeError(f"PR #{number} has an invalid head SHA") + if not isinstance(number, int) or not isinstance(title, str): + raise RuntimeError("validator returned an invalid PR identity") + + conclusion, check_title, summary = check_summary(result, scanner_jobs) + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + payload = { + "name": CHECK_NAME, + "head_sha": head_sha, + "status": "completed", + "conclusion": conclusion, + "started_at": now, + "completed_at": now, + "details_url": run_url, + "output": { + "title": check_title, + "summary": f"PR #{number} — {title}\n\n{summary}", + }, + } + update_payload = {key: value for key, value in payload.items() if key != "head_sha"} + + existing = github_api( + repository, + f"/commits/{quote(head_sha, safe='')}/check-runs?check_name={quote(CHECK_NAME)}&per_page=100", + token, + ) + check_runs = existing.get("check_runs", []) if isinstance(existing, dict) else [] + matching = [ + item for item in check_runs if isinstance(item, dict) and item.get("name") == CHECK_NAME + ] + if matching and isinstance(matching[-1].get("id"), int): + github_api( + repository, + f"/check-runs/{matching[-1]['id']}", + token, + method="PATCH", + payload=update_payload, + ) + else: + github_api(repository, "/check-runs", token, method="POST", payload=payload) + print(f"Published {CHECK_NAME} for PR #{number}: {conclusion}") + upsert_remediation_comment(repository, result, conclusion, check_title, summary, run_url, token) + + +def main() -> int: + repository = os.environ.get("GITHUB_REPOSITORY", "") + token = os.environ.get("GITHUB_TOKEN", "").strip() + run_id = os.environ.get("GITHUB_RUN_ID", "").strip() + run_url = os.environ.get("SWEEP_RUN_URL", "").strip() + results_json = os.environ.get("OPEN_PR_RESULTS", "[]") + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository): + print("ERROR: GITHUB_REPOSITORY must be an owner/repository pair", file=sys.stderr) + return 1 + if not token or not run_id or not run_url: + print("ERROR: GITHUB_TOKEN, GITHUB_RUN_ID, and SWEEP_RUN_URL are required", file=sys.stderr) + return 1 + try: + results = json.loads(results_json) + if not isinstance(results, list): + raise RuntimeError("OPEN_PR_RESULTS must be a JSON array") + scanner_jobs = scan_conclusions(repository, run_id, token) + for result in results: + if not isinstance(result, dict): + raise RuntimeError("validator returned a non-object PR result") + publish_check(repository, result, scanner_jobs, run_url, token) + except (RuntimeError, json.JSONDecodeError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate-contribution.py b/scripts/validate-contribution.py new file mode 100644 index 000000000..d14614ec7 --- /dev/null +++ b/scripts/validate-contribution.py @@ -0,0 +1,673 @@ +#!/usr/bin/env python3 +"""Validate new Awesome Codex Plugins README contributions. + +The catalog stores links to source repositories rather than plugin code. This +validator therefore checks the contribution at the same boundary a maintainer +reviews it: + +* only newly added Community Plugins entries are considered; +* each source repository is public and exposes workflow-based scanner CI; and +* the workflow is triggered by a push or pull request and invokes the HOL AI + Plugin Scanner action. + +The workflow that calls this script emits a matrix of source repositories. A +follow-up job scans each repository with the same 80-point/high-severity gate +documented in CONTRIBUTING.md. +""" + +from __future__ import annotations + +import argparse +import base64 +import difflib +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + +try: + import yaml +except ModuleNotFoundError: # pragma: no cover - exercised in minimal runners + yaml = None + +REPO_ROOT = Path(__file__).resolve().parent.parent +README_PATH = REPO_ROOT / "README.md" +REQUEST_TIMEOUT_SECONDS = 30 +MAX_WORKFLOW_BYTES = 512 * 1024 +USER_AGENT = "awesome-codex-plugins-contribution-validator" + +README_ENTRY_RE = re.compile( + r"^- \[([^\]]+)\]\((https://github\.com/" + r"([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)(?:[?#][^)]*)?)\)\s*[-\u2013\u2014]\s*(.+)$", + re.MULTILINE, +) + + +@dataclass(frozen=True) +class Contribution: + display_name: str + url: str + owner: str + repo: str + description: str + + +@dataclass(frozen=True) +class OpenPullRequest: + number: int + title: str + head_repository: str + head_ref: str + head_sha: str + base_ref: str + base_sha: str + author_login: str = "" + + +class ValidationError(Exception): + """A user-facing contribution validation error.""" + + +def workflow_has_ci_trigger(document: object) -> bool: + """Return whether a parsed workflow runs on push or pull_request.""" + + if not isinstance(document, dict): + return False + + # PyYAML's YAML 1.1 resolver can load the key ``on`` as True. + trigger = document.get("on", document.get(True)) + if isinstance(trigger, str): + return trigger in {"push", "pull_request"} + if isinstance(trigger, list): + return any(item in {"push", "pull_request"} for item in trigger) + if isinstance(trigger, dict): + return any(key in {"push", "pull_request"} for key in trigger) + return False + + +def scanner_steps(document: object) -> list[str]: + """Return scanner action references from parsed workflow steps.""" + + if not isinstance(document, dict): + return [] + jobs = document.get("jobs") + if not isinstance(jobs, dict): + return [] + + references: list[str] = [] + for job in jobs.values(): + if not isinstance(job, dict): + continue + steps = job.get("steps") + if not isinstance(steps, list): + continue + for step in steps: + if not isinstance(step, dict): + continue + uses = step.get("uses") + if not isinstance(uses, str): + continue + normalized = uses.strip() + if normalized.lower().startswith("hashgraph-online/ai-plugin-scanner-action@"): + references.append(normalized) + return references + + +def parse_workflow_document(name: str, text: str) -> object: + """Parse workflow YAML with a safe loader and a clear dependency error.""" + + if yaml is None: + raise ValidationError( + "PyYAML is required to inspect source workflows; install PyYAML before running validation" + ) + try: + return yaml.safe_load(text) + except yaml.YAMLError as error: + raise ValidationError(f"{name} is not valid workflow YAML: {error}") from error + + +def git(*args: str) -> str: + """Run a read-only git command in the repository and return stdout.""" + + result = subprocess.run( + ["git", "-C", str(REPO_ROOT), *args], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return "" + return result.stdout.strip() + + +def normalize_url(url: str) -> str: + """Normalize a GitHub repository URL for change-set comparison.""" + + return url.rstrip("/").removesuffix(".git").lower() + + +def current_readme_section(readme_lines: list[str], line_number: int) -> str: + """Return the nearest level-two heading before a 1-based line number.""" + + heading = "" + for index, line in enumerate(readme_lines, start=1): + if index > line_number: + break + match = re.match(r"^##\s+(.+?)\s*$", line) + if match: + heading = match.group(1).strip() + return heading + + +def get_new_readme_entries_from_diff(diff: str, base_readme: str, head_readme: str) -> list[Contribution]: + """Find newly added Community Plugins entries in a README diff.""" + + if not diff: + return [] + + base_urls = { + normalize_url(match.group(2)) + for match in README_ENTRY_RE.finditer(base_readme) + } + readme_lines = head_readme.splitlines() + + entries: list[Contribution] = [] + seen_urls: set[str] = set() + added_line_number = 0 + for line in diff.splitlines(): + if line.startswith("@@"): + hunk = re.search(r"\+(\d+)", line) + added_line_number = int(hunk.group(1)) if hunk else 0 + continue + if line.startswith("+") and not line.startswith("+++"): + content = line[1:] + match = README_ENTRY_RE.match(content.strip()) + if match and current_readme_section(readme_lines, added_line_number) == "Community Plugins": + url = normalize_url(match.group(2)) + if url not in base_urls and url not in seen_urls: + seen_urls.add(url) + entries.append( + Contribution( + display_name=match.group(1).strip(), + url=match.group(2).strip(), + owner=match.group(3), + repo=match.group(4), + description=match.group(5).strip(), + ) + ) + added_line_number += 1 + continue + if not line.startswith("-"): + added_line_number += 1 + + return entries + + +def get_new_readme_entries(base_ref: str) -> list[Contribution]: + """Find newly added Community Plugins entries in the local README diff.""" + + diff = git("diff", base_ref, "--", "README.md") + if not diff or not README_PATH.exists(): + return [] + + base_readme = git("show", f"{base_ref}:README.md") + head_readme = README_PATH.read_text(encoding="utf-8") + return get_new_readme_entries_from_diff(diff, base_readme, head_readme) + + +def request_bytes(url: str, *, max_bytes: int = MAX_WORKFLOW_BYTES) -> bytes: + """Fetch a bounded GitHub API/raw response.""" + + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": USER_AGENT, + } + token = os.environ.get("GITHUB_TOKEN", "").strip() + if token: + headers["Authorization"] = f"Bearer {token}" + + request = Request(url, headers=headers) + try: + with urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > max_bytes: + raise ValidationError(f"response is larger than {max_bytes} bytes") + payload = response.read(max_bytes + 1) + except HTTPError as error: + if error.code == 404: + raise ValidationError("source repository or workflow directory was not found") from error + raise ValidationError(f"GitHub returned HTTP {error.code}") from error + except (URLError, TimeoutError, OSError) as error: + raise ValidationError(f"could not fetch GitHub metadata: {error}") from error + + if len(payload) > max_bytes: + raise ValidationError(f"response is larger than {max_bytes} bytes") + return payload + + +def github_json(url: str) -> object: + """Fetch a bounded GitHub API JSON response.""" + + try: + return json.loads(request_bytes(url, max_bytes=4 * 1024 * 1024).decode("utf-8")) + except json.JSONDecodeError as error: + raise ValidationError(f"GitHub returned invalid JSON for {url}") from error + + +def github_api_list(url: str) -> list[dict[str, object]]: + """Fetch all pages from a GitHub API list endpoint.""" + + values: list[dict[str, object]] = [] + page = 1 + while True: + separator = "&" if "?" in url else "?" + payload = github_json(f"{url}{separator}{urlencode({'per_page': 100, 'page': page})}") + if not isinstance(payload, list): + raise ValidationError(f"GitHub returned a non-list response for {url}") + page_values = [item for item in payload if isinstance(item, dict)] + values.extend(page_values) + if len(payload) < 100: + return values + page += 1 + + +def content_file(repository: str, path: str, ref: str) -> str: + """Read a UTF-8 file from a public repository at an exact ref.""" + + url = f"https://api.github.com/repos/{repository}/contents/{path}?{urlencode({'ref': ref})}" + payload = github_json(url) + if not isinstance(payload, dict): + raise ValidationError(f"{repository}/{path} is not a file") + encoded = payload.get("content") + if not isinstance(encoded, str): + raise ValidationError(f"GitHub did not return content for {repository}/{path}") + try: + return base64.b64decode(encoded, validate=False).decode("utf-8") + except (ValueError, UnicodeDecodeError) as error: + raise ValidationError(f"Could not decode {repository}/{path}") from error + + +def workflow_files(owner: str, repo: str) -> list[tuple[str, str]]: + """Return (filename, text) pairs for a source repository's workflows.""" + + api_url = f"https://api.github.com/repos/{owner}/{repo}/contents/.github/workflows" + payload = json.loads(request_bytes(api_url).decode("utf-8")) + if not isinstance(payload, list): + raise ValidationError(".github/workflows is not a directory") + + files: list[tuple[str, str]] = [] + for item in payload: + if not isinstance(item, dict) or item.get("type") != "file": + continue + name = str(item.get("name", "")) + if not name.lower().endswith((".yml", ".yaml")): + continue + download_url = item.get("download_url") + if not isinstance(download_url, str) or not download_url: + continue + text = request_bytes(download_url).decode("utf-8", errors="replace") + files.append((name, text)) + return files + + +def validate_scanner_ci(contribution: Contribution) -> None: + """Require a push/PR workflow that invokes the HOL scanner action.""" + + try: + files = workflow_files(contribution.owner, contribution.repo) + except (ValidationError, json.JSONDecodeError) as error: + raise ValidationError( + f"{contribution.url} does not expose readable GitHub Actions workflows: {error}" + ) from error + + scanner_workflows: list[tuple[str, object, list[str]]] = [] + for name, text in files: + document = parse_workflow_document(name, text) + references = scanner_steps(document) + if references: + scanner_workflows.append((name, document, references)) + + if not scanner_workflows: + raise ValidationError( + f"{contribution.url} must invoke " + "hashgraph-online/ai-plugin-scanner-action in .github/workflows" + ) + + if not any(workflow_has_ci_trigger(document) for _, document, _ in scanner_workflows): + names = ", ".join(name for name, _, _ in scanner_workflows) + raise ValidationError( + f"{contribution.url} scanner workflow ({names}) must run on push or pull_request" + ) + + +def list_open_pull_requests(repository: str, pull_request_number: int | None) -> list[OpenPullRequest]: + """Return open pull requests without checking out untrusted fork code.""" + + if pull_request_number is not None: + url = f"https://api.github.com/repos/{repository}/pulls/{pull_request_number}" + payload = github_json(url) + payloads = [payload] + else: + url = f"https://api.github.com/repos/{repository}/pulls?state=open" + payloads = github_api_list(url) + + pull_requests: list[OpenPullRequest] = [] + for payload in payloads: + if not isinstance(payload, dict): + continue + number = payload.get("number") + title = payload.get("title") + head = payload.get("head") + base = payload.get("base") + if not isinstance(number, int) or not isinstance(title, str): + continue + if not isinstance(head, dict) or not isinstance(base, dict): + continue + head_repository_payload = head.get("repo") + user_payload = payload.get("user") + head_ref = head.get("ref") + head_sha = head.get("sha") + base_ref = base.get("ref") + base_sha = base.get("sha") + if not isinstance(head_repository_payload, dict): + continue + head_repository = head_repository_payload.get("full_name") + if not isinstance(head_repository, str): + continue + if not isinstance(head_ref, str) or not isinstance(base_ref, str): + continue + if not isinstance(head_sha, str) or not isinstance(base_sha, str): + continue + author_login = user_payload.get("login", "") if isinstance(user_payload, dict) else "" + if not isinstance(author_login, str): + author_login = "" + pull_requests.append( + OpenPullRequest( + number=number, + title=title, + head_repository=head_repository, + head_ref=head_ref, + head_sha=head_sha, + base_ref=base_ref, + base_sha=base_sha, + author_login=author_login, + ) + ) + return pull_requests + + +def entries_for_open_pull_request(repository: str, pull_request: OpenPullRequest) -> list[Contribution]: + """Extract new Community Plugin entries from a PR's exact base/head refs.""" + + head_owner = pull_request.head_repository.split("/", maxsplit=1)[0] + compare_ref = quote( + f"{pull_request.base_ref}...{head_owner}:{pull_request.head_ref}", + safe=".../:", + ) + compare_url = f"https://api.github.com/repos/{repository}/compare/{compare_ref}" + compare_payload = github_json(compare_url) + if not isinstance(compare_payload, dict): + raise ValidationError("GitHub returned an invalid merge-base comparison") + merge_base_commit = compare_payload.get("merge_base_commit") + if not isinstance(merge_base_commit, dict) or not isinstance(merge_base_commit.get("sha"), str): + raise ValidationError("GitHub did not return a merge base for this pull request") + merge_base_sha = merge_base_commit["sha"] + + base_readme = content_file(repository, "README.md", merge_base_sha) + head_readme = content_file(pull_request.head_repository, "README.md", pull_request.head_sha) + diff = "".join( + difflib.unified_diff( + base_readme.splitlines(keepends=True), + head_readme.splitlines(keepends=True), + fromfile="README.md", + tofile="README.md", + ) + ) + return get_new_readme_entries_from_diff(diff, base_readme, head_readme) + + +def scan_open_pull_requests( + repository: str, + pull_request_number: int | None, + matrix_output: Path | None, + report_output: Path | None, + status_output: Path | None, +) -> int: + """Validate open PR source repositories and emit a scanner matrix/report.""" + + pull_requests = list_open_pull_requests(repository, pull_request_number) + matrix: list[dict[str, object]] = [] + report_lines = [ + "## Open contribution sweep", + "", + f"Repository: `{repository}`", + f"Open pull requests checked: {len(pull_requests)}", + "", + ] + failures: list[dict[str, object]] = [] + results: list[dict[str, object]] = [] + + for pull_request in pull_requests: + prefix = f"PR #{pull_request.number} — {pull_request.title}" + try: + entries = entries_for_open_pull_request(repository, pull_request) + except ValidationError as error: + failures.append({"pr_number": pull_request.number, "error": str(error)}) + results.append( + { + "pr_number": pull_request.number, + "title": pull_request.title, + "head_sha": pull_request.head_sha, + "author_login": pull_request.author_login, + "state": "failure", + "contributions": [], + "failure_reasons": [str(error)], + } + ) + report_lines.append(f"- **{prefix}: FAIL** — {error}") + continue + + if not entries: + results.append( + { + "pr_number": pull_request.number, + "title": pull_request.title, + "head_sha": pull_request.head_sha, + "author_login": pull_request.author_login, + "state": "success", + "contributions": [], + "failure_reasons": [], + } + ) + report_lines.append(f"- **{prefix}: PASS** — no new Community Plugins entries") + continue + + report_lines.append(f"- **{prefix}**") + scanner_contributions: list[dict[str, str]] = [] + scanner_failures: list[str] = [] + for entry in entries: + contribution = f"{entry.owner}/{entry.repo}" + try: + validate_scanner_ci(entry) + except ValidationError as error: + failures.append( + { + "pr_number": pull_request.number, + "repository": contribution, + "error": str(error), + } + ) + scanner_failures.append(f"{contribution}: {error}") + report_lines.append(f" - `{contribution}`: **FAIL** — {error}") + continue + + scanner_contributions.append({"owner": entry.owner, "repo": entry.repo}) + matrix.append( + { + "pr_number": pull_request.number, + "owner": entry.owner, + "repo": entry.repo, + } + ) + report_lines.append(f" - `{contribution}`: scanner CI present; queued for score scan") + + results.append( + { + "pr_number": pull_request.number, + "title": pull_request.title, + "head_sha": pull_request.head_sha, + "author_login": pull_request.author_login, + "state": "failure" if scanner_failures else "scan", + "contributions": scanner_contributions, + "failure_reasons": scanner_failures, + } + ) + + if not pull_requests: + report_lines.append("No open pull requests found.") + elif failures: + report_lines.extend( + [ + "", + f"Scanner CI validation failures: {len(failures)}", + "Source repositories must add the HOL AI Plugin Scanner workflow before merge.", + ] + ) + else: + report_lines.extend(["", "All open contribution entries passed scanner CI validation."]) + + report = "\n".join(report_lines) + "\n" + if matrix_output: + matrix_output.parent.mkdir(parents=True, exist_ok=True) + matrix_output.write_text(json.dumps(matrix, separators=(",", ":")), encoding="utf-8") + if report_output: + report_output.parent.mkdir(parents=True, exist_ok=True) + report_output.write_text(report, encoding="utf-8") + if status_output: + status_output.parent.mkdir(parents=True, exist_ok=True) + status_output.write_text( + json.dumps( + { + "has_failures": bool(failures), + "failures": failures, + "results": results, + }, + separators=(",", ":"), + ), + encoding="utf-8", + ) + + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write(report) + + print(report, end="") + return 0 + + +def write_matrix(path: Path, entries: list[Contribution]) -> None: + """Write the scanner job matrix as compact JSON.""" + + path.parent.mkdir(parents=True, exist_ok=True) + matrix = [ + {"owner": entry.owner, "repo": entry.repo} + for entry in entries + ] + path.write_text(json.dumps(matrix, separators=(",", ":")), encoding="utf-8") + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--open-prs", + action="store_true", + help="Validate open pull requests through the GitHub API without checking out fork code", + ) + parser.add_argument( + "--repository", + default=os.environ.get("GITHUB_REPOSITORY", ""), + help="owner/repository to inspect in --open-prs mode", + ) + parser.add_argument( + "--pr-number", + type=int, + help="Limit --open-prs mode to one pull request", + ) + parser.add_argument( + "--base-ref", + default=os.environ.get("GITHUB_BASE_REF", "origin/main"), + help="Git ref to compare against (default: GITHUB_BASE_REF or origin/main)", + ) + parser.add_argument( + "--matrix-output", + type=Path, + help="Write the scanner job matrix JSON to this path", + ) + parser.add_argument( + "--report-output", + type=Path, + help="Write the open-PR Markdown report to this path", + ) + parser.add_argument( + "--status-output", + type=Path, + help="Write open-PR validation status JSON to this path", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.open_prs: + if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", args.repository): + print("ERROR: --repository must be an owner/repository pair", file=sys.stderr) + return 1 + return scan_open_pull_requests( + args.repository, + args.pr_number, + args.matrix_output, + args.report_output, + args.status_output, + ) + + if not git("rev-parse", "--verify", args.base_ref): + print(f"ERROR: base ref '{args.base_ref}' is not available", file=sys.stderr) + return 1 + + entries = get_new_readme_entries(args.base_ref) + if not entries: + print("No new Community Plugins entries found; contribution checks are complete.") + if args.matrix_output: + write_matrix(args.matrix_output, []) + return 0 + + failures = 0 + for entry in entries: + print(f"Checking {entry.display_name} ({entry.owner}/{entry.repo})...") + try: + validate_scanner_ci(entry) + except ValidationError as error: + failures += 1 + print(f" FAIL: {error}", file=sys.stderr) + else: + print(" PASS: scanner CI is present and push/PR-triggered") + + if failures: + print(f"\nContribution validation failed for {failures} entr{'y' if failures == 1 else 'ies'}.", file=sys.stderr) + return 1 + + print(f"\nAll {len(entries)} contribution entr{'y' if len(entries) == 1 else 'ies'} passed.") + if args.matrix_output: + write_matrix(args.matrix_output, entries) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 1959442d8a2023c826384577274dcbcba988bff2 Mon Sep 17 00:00:00 2001 From: Michael Kantor <6068672+kantorcodes@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:50:44 -0400 Subject: [PATCH 2/2] fix(ci): harden contribution workflow detection Signed-off-by: Michael Kantor <6068672+kantorcodes@users.noreply.github.com> --- scripts/publish-open-pr-checks.py | 15 +++++- scripts/validate-contribution.py | 88 +++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/scripts/publish-open-pr-checks.py b/scripts/publish-open-pr-checks.py index d933d16e3..4d0304016 100644 --- a/scripts/publish-open-pr-checks.py +++ b/scripts/publish-open-pr-checks.py @@ -160,7 +160,20 @@ def remediation_comment( f"[View the latest sweep]({run_url})." ) - if result.get("state") == "failure": + failure_reasons = result.get("failure_reasons") + malformed_readme = any( + isinstance(reason, str) and "Community Plugins entries" in reason + for reason in (failure_reasons if isinstance(failure_reasons, list) else []) + ) + if malformed_readme: + guidance = ( + "1. Format each Community Plugins entry as " + "`- [Plugin Name](https://github.com//) - Description`.\n" + "2. Keep the link at the repository root, use one sentence for the description, " + "and keep the entry in its category's alphabetical order.\n" + "3. Push the README correction and rerun the check." + ) + elif result.get("state") == "failure": guidance = ( "1. Add a workflow under `.github/workflows/` in the linked source repository.\n" "2. Trigger it on both `push` and `pull_request`, and invoke " diff --git a/scripts/validate-contribution.py b/scripts/validate-contribution.py index d14614ec7..ef6a18940 100644 --- a/scripts/validate-contribution.py +++ b/scripts/validate-contribution.py @@ -44,7 +44,7 @@ README_ENTRY_RE = re.compile( r"^- \[([^\]]+)\]\((https://github\.com/" - r"([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)(?:[?#][^)]*)?)\)\s*[-\u2013\u2014]\s*(.+)$", + r"([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)(?:/)?(?:[?#][^)]*)?)\)\s*[-\u2013\u2014]\s*(.+)$", re.MULTILINE, ) @@ -119,6 +119,55 @@ def scanner_steps(document: object) -> list[str]: return references +def reusable_workflow_references(document: object) -> list[str]: + """Return local reusable workflow filenames referenced by jobs.""" + + if not isinstance(document, dict): + return [] + jobs = document.get("jobs") + if not isinstance(jobs, dict): + return [] + + references: list[str] = [] + for job in jobs.values(): + if not isinstance(job, dict): + continue + uses = job.get("uses") + if not isinstance(uses, str): + continue + workflow_ref = uses.split("@", 1)[0].strip().replace("\\", "/") + if not workflow_ref.startswith(".github/workflows/") and not workflow_ref.startswith( + "./.github/workflows/" + ): + continue + references.append(workflow_ref.rsplit("/", 1)[-1]) + return references + + +def workflow_reaches_scanner( + name: str, + documents: dict[str, object], + visiting: set[str] | None = None, +) -> bool: + """Return whether a workflow directly or indirectly invokes the scanner.""" + + document = documents.get(name) + if document is None: + return False + if scanner_steps(document): + return True + + active = set() if visiting is None else set(visiting) + if name in active: + return False + active.add(name) + names_by_lower = {workflow_name.lower(): workflow_name for workflow_name in documents} + return any( + workflow_reaches_scanner(names_by_lower.get(reference.lower(), ""), documents, active) + for reference in reusable_workflow_references(document) + ) + + def parse_workflow_document(name: str, text: str) -> object: """Parse workflow YAML with a safe loader and a clear dependency error.""" @@ -178,6 +227,7 @@ def get_new_readme_entries_from_diff(diff: str, base_readme: str, head_readme: s readme_lines = head_readme.splitlines() entries: list[Contribution] = [] + invalid_entries: list[str] = [] seen_urls: set[str] = set() added_line_number = 0 for line in diff.splitlines(): @@ -188,7 +238,12 @@ def get_new_readme_entries_from_diff(diff: str, base_readme: str, head_readme: s if line.startswith("+") and not line.startswith("+++"): content = line[1:] match = README_ENTRY_RE.match(content.strip()) - if match and current_readme_section(readme_lines, added_line_number) == "Community Plugins": + in_community_plugins = ( + current_readme_section(readme_lines, added_line_number) == "Community Plugins" + ) + if in_community_plugins and content.strip().startswith("- ") and not match: + invalid_entries.append(f"line {added_line_number}: {content.strip()}") + if match and in_community_plugins: url = normalize_url(match.group(2)) if url not in base_urls and url not in seen_urls: seen_urls.add(url) @@ -206,6 +261,15 @@ def get_new_readme_entries_from_diff(diff: str, base_readme: str, head_readme: s if not line.startswith("-"): added_line_number += 1 + if invalid_entries: + details = "; ".join(invalid_entries[:3]) + suffix = "" if len(invalid_entries) <= 3 else f"; and {len(invalid_entries) - 3} more" + raise ValidationError( + "Community Plugins entries must use " + "`- [Plugin Name](https://github.com//) - Description`; " + f"could not parse {details}{suffix}" + ) + return entries @@ -326,9 +390,11 @@ def validate_scanner_ci(contribution: Contribution) -> None: f"{contribution.url} does not expose readable GitHub Actions workflows: {error}" ) from error + documents: dict[str, object] = {} scanner_workflows: list[tuple[str, object, list[str]]] = [] for name, text in files: document = parse_workflow_document(name, text) + documents[name] = document references = scanner_steps(document) if references: scanner_workflows.append((name, document, references)) @@ -339,10 +405,16 @@ def validate_scanner_ci(contribution: Contribution) -> None: "hashgraph-online/ai-plugin-scanner-action in .github/workflows" ) - if not any(workflow_has_ci_trigger(document) for _, document, _ in scanner_workflows): + triggered_scanner_workflows = [ + name + for name, document in documents.items() + if workflow_has_ci_trigger(document) and workflow_reaches_scanner(name, documents) + ] + if not triggered_scanner_workflows: names = ", ".join(name for name, _, _ in scanner_workflows) raise ValidationError( - f"{contribution.url} scanner workflow ({names}) must run on push or pull_request" + f"{contribution.url} scanner workflow ({names}) must run on push or pull_request " + "or be called by a push/pull_request workflow" ) @@ -641,7 +713,13 @@ def main() -> int: print(f"ERROR: base ref '{args.base_ref}' is not available", file=sys.stderr) return 1 - entries = get_new_readme_entries(args.base_ref) + try: + entries = get_new_readme_entries(args.base_ref) + except ValidationError as error: + print(f"Contribution validation failed: {error}", file=sys.stderr) + if args.matrix_output: + write_matrix(args.matrix_output, []) + return 1 if not entries: print("No new Community Plugins entries found; contribution checks are complete.") if args.matrix_output: