diff --git a/.github/workflows/eval-pr-run.yml b/.github/workflows/eval-pr-run.yml new file mode 100644 index 000000000..a4df18df7 --- /dev/null +++ b/.github/workflows/eval-pr-run.yml @@ -0,0 +1,309 @@ +# Runs evals for PRs via workflow_run dispatch, enabling secret access +# for fork PRs. Triggered by eval-pr.yml completion (path-filtered gate). +# +# Trust model: +# - Collaborators with write/admin permission: evals run automatically +# - External contributors: require manual approval via eval-protected environment +# +# All PR identity and skill discovery is derived from the GitHub API and +# the checked-out PR merge commit — no data from the triggering workflow +# is trusted. +# +# See docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md for full design. + +name: Eval PR Run + +on: + workflow_run: + workflows: ["Eval PR"] + types: [completed] + +permissions: + contents: read + pull-requests: write + actions: read + statuses: write + +jobs: + discover: + name: Discover PR Context + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + skills: ${{ steps.discover.outputs.skills }} + pr_number: ${{ steps.pr.outputs.pr_number }} + author: ${{ steps.pr.outputs.author }} + trusted: ${{ steps.pr.outputs.trusted }} + steps: + - name: Set pending commit status + uses: actions/github-script@v7 + with: + script: | + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.workflow_run.head_sha, + state: 'pending', + description: 'Eval run in progress...', + context: 'Eval PR Run', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + }); + + - name: Resolve PR identity and check trust + id: pr + uses: actions/github-script@v7 + with: + script: | + // listPullRequestsAssociatedWithCommit returns empty for fork PRs + // because the commit lives in the fork's repo, not the base repo's + // commit graph. Use pulls.list filtered by base branch instead. + const headSha = context.payload.workflow_run.head_sha; + const prs = await github.paginate(github.rest.pulls.list, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + base: 'main', + per_page: 100 + }); + + const pr = prs.find(p => p.head.sha === headSha); + if (!pr) { + core.setFailed(`No open PR targeting main found for commit ${headSha}`); + return; + } + core.setOutput('pr_number', pr.number.toString()); + core.setOutput('base_sha', pr.base.sha); + core.setOutput('author', pr.user.login); + + let trusted = false; + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: pr.user.login + }); + trusted = ['admin', 'write'].includes(data.permission); + console.log(`Author ${pr.user.login}: permission=${data.permission}, trusted=${trusted}`); + } catch (e) { + console.log(`Author ${pr.user.login}: not a collaborator (${e.status}), trusted=false`); + } + + core.setOutput('trusted', trusted.toString()); + + - name: Checkout PR merge commit + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ steps.pr.outputs.pr_number }}/merge + fetch-depth: 0 + + - name: Discover changed skills + id: discover + env: + BASE_SHA: ${{ steps.pr.outputs.base_sha }} + run: | + changed_files=$(git diff --name-only "$BASE_SHA"...HEAD) + echo "Changed files:" + echo "$changed_files" + + skills="" + for file in $changed_files; do + skill="" + if [[ "$file" =~ ^plugins/sdlc-workflow/skills/([^/]+)/ ]]; then + skill="${BASH_REMATCH[1]}" + elif [[ "$file" =~ ^evals/([^/]+)/ ]]; then + skill="${BASH_REMATCH[1]}" + fi + + if [ -n "$skill" ] && [ -f "evals/${skill}/evals.json" ]; then + if [[ ! ",$skills," == *",$skill,"* ]]; then + skills="${skills:+${skills},}${skill}" + fi + fi + done + + echo "skills=${skills}" >> "$GITHUB_OUTPUT" + echo "Discovered changed skills with evals: ${skills:-none}" + + - name: Update status for approval gate + if: steps.pr.outputs.trusted != 'true' && steps.pr.outputs.pr_number != '' + uses: actions/github-script@v7 + with: + script: | + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.workflow_run.head_sha, + state: 'pending', + description: 'Waiting for approval in eval-protected environment', + context: 'Eval PR Run', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + }); + + gate: + name: Approval Gate + needs: discover + if: >- + needs.discover.result == 'success' && + needs.discover.outputs.trusted != 'true' && + needs.discover.outputs.skills != '' + runs-on: ubuntu-latest + environment: eval-protected + steps: + - run: echo "Approved by reviewer" + + run-evals: + name: Run PR Evals + needs: [discover, gate] + if: >- + !cancelled() && + needs.discover.result == 'success' && + needs.discover.outputs.skills != '' && + (needs.discover.outputs.trusted == 'true' || needs.gate.result == 'success') + runs-on: ubuntu-latest + steps: + - name: Checkout PR merge commit + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ needs.discover.outputs.pr_number }}/merge + fetch-depth: 0 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Install Claude Code + run: curl -fsSL https://claude.ai/install.sh | bash + + - name: Run PR evals + env: + SKILLS_CSV: ${{ needs.discover.outputs.skills }} + CLAUDE_CODE_USE_VERTEX: "1" + CLOUD_ML_REGION: ${{ secrets.GCP_CLOUD_ML_REGION }} + ANTHROPIC_DEFAULT_SONNET_MODEL: "claude-sonnet-4-6" + ANTHROPIC_DEFAULT_OPUS_MODEL: "claude-opus-4-6" + ANTHROPIC_MODEL: "claude-opus-4-6" + run: | + IFS=',' read -ra SKILLS <<< "$SKILLS_CSV" + + for skill in "${SKILLS[@]}"; do + eval_count=$(jq '.evals | length' "evals/${skill}/evals.json") + workspace="/tmp/${skill}-eval-pr" + mkdir -p "${workspace}" + + echo "=== Running PR evals for ${skill} (${eval_count} cases) ===" + echo "Workspace: ${workspace}" + + claude -p "$(cat <&1 || { + echo "::warning::PR eval run failed for ${skill} (exit $?)" + continue + } + + echo "--- Workspace contents ---" + find "${workspace}" -type f 2>/dev/null | head -80 || true + done + + - name: Post eval results review + env: + SKILLS_CSV: ${{ needs.discover.outputs.skills }} + PR_NUMBER: ${{ needs.discover.outputs.pr_number }} + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const skills = process.env.SKILLS_CSV.split(',').filter(Boolean); + const prNumber = parseInt(process.env.PR_NUMBER); + + let body = '## Eval Results\n\n'; + for (const skill of skills) { + const summaryPath = `/tmp/${skill}-eval-pr/summary.md`; + if (fs.existsSync(summaryPath)) { + body += fs.readFileSync(summaryPath, 'utf8'); + } else { + body += `### ${skill}\n\n> No results produced. See workflow logs.\n\n`; + } + } + + const pluginJson = JSON.parse(fs.readFileSync('plugins/sdlc-workflow/.claude-plugin/plugin.json', 'utf8')); + const version = pluginJson.version; + + body += '---\n'; + body += `*Generated by [sdlc-workflow/run-evals](plugins/sdlc-workflow/skills/run-evals) v${version}*\n`; + + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + const marker = '## Eval Results'; + const existing = reviews.find(r => + r.user?.login === 'github-actions[bot]' && r.body?.startsWith(marker) + ); + + if (existing) { + await github.rest.pulls.updateReview({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + review_id: existing.id, + body + }); + } else { + await github.rest.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + event: 'COMMENT', + body + }); + } + + report-status: + name: Report Status + needs: [discover, gate, run-evals] + if: always() + runs-on: ubuntu-latest + steps: + - name: Set final commit status + env: + DISCOVER_RESULT: ${{ needs.discover.result }} + EVALS_RESULT: ${{ needs.run-evals.result }} + GATE_RESULT: ${{ needs.gate.result }} + uses: actions/github-script@v7 + with: + script: | + const discoverResult = process.env.DISCOVER_RESULT; + const evalsResult = process.env.EVALS_RESULT; + const gateResult = process.env.GATE_RESULT; + + let state, description; + if (evalsResult === 'success') { + state = 'success'; + description = 'Eval run completed — results posted as PR review'; + } else if (discoverResult !== 'success') { + state = 'failure'; + description = 'Eval run failed — could not resolve PR context'; + } else if (gateResult === 'failure') { + state = 'failure'; + description = 'Approval was rejected in eval-protected environment'; + } else { + state = 'failure'; + description = 'Eval run failed — check workflow logs'; + } + + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.workflow_run.head_sha, + state, + description, + context: 'Eval PR Run', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + }); diff --git a/.github/workflows/eval-pr.yml b/.github/workflows/eval-pr.yml index 989941a27..cc676c384 100644 --- a/.github/workflows/eval-pr.yml +++ b/.github/workflows/eval-pr.yml @@ -1,7 +1,8 @@ -# Runs skill evals on PRs that modify skill or eval files, compares results -# against the stored baseline, and posts a benchmark delta as a PR review. +# Path-filtered trigger for the eval dispatch workflow (eval-pr-run.yml). +# This workflow does no work itself — its completion signals eval-pr-run.yml +# to run via workflow_run. # -# See docs/specs/2026-04-21-eval-skills-ci-workflow-design.md for full design. +# See docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md for full design. name: Eval PR @@ -14,139 +15,10 @@ on: - '.github/workflows/eval-pr.yml' jobs: - eval-pr: - name: Eval PR Changes + trigger: + name: Trigger Eval Dispatch runs-on: ubuntu-latest permissions: contents: read - pull-requests: write steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v3 - with: - credentials_json: ${{ secrets.GCP_SA_KEY }} - - - name: Install Claude Code - run: curl -fsSL https://claude.ai/install.sh | bash - - - name: Discover changed skills - id: discover - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - changed_files=$(git diff --name-only "$BASE_SHA"...HEAD) - echo "Changed files:" - echo "$changed_files" - - skills="" - for file in $changed_files; do - skill="" - if [[ "$file" =~ ^plugins/sdlc-workflow/skills/([^/]+)/ ]]; then - skill="${BASH_REMATCH[1]}" - elif [[ "$file" =~ ^evals/([^/]+)/ ]]; then - skill="${BASH_REMATCH[1]}" - fi - - if [ -n "$skill" ] && [ -f "evals/${skill}/evals.json" ]; then - if [[ ! ",$skills," == *",$skill,"* ]]; then - skills="${skills:+${skills},}${skill}" - fi - fi - done - - echo "skills=${skills}" >> "$GITHUB_OUTPUT" - echo "Discovered changed skills with evals: ${skills:-none}" - - - name: Run PR evals - if: steps.discover.outputs.skills != '' - env: - CLAUDE_CODE_USE_VERTEX: "1" - CLOUD_ML_REGION: ${{ secrets.GCP_CLOUD_ML_REGION }} - ANTHROPIC_DEFAULT_SONNET_MODEL: "claude-sonnet-4-6" - ANTHROPIC_DEFAULT_OPUS_MODEL: "claude-opus-4-6" - ANTHROPIC_MODEL: "claude-opus-4-6" - run: | - IFS=',' read -ra SKILLS <<< "${{ steps.discover.outputs.skills }}" - - for skill in "${SKILLS[@]}"; do - eval_count=$(jq '.evals | length' "evals/${skill}/evals.json") - workspace="/tmp/${skill}-eval-pr" - mkdir -p "${workspace}" - - echo "=== Running PR evals for ${skill} (${eval_count} cases) ===" - echo "Workspace: ${workspace}" - - claude -p "$(cat <&1 || { - echo "::warning::PR eval run failed for ${skill} (exit $?)" - continue - } - - echo "--- Workspace contents ---" - find "${workspace}" -type f 2>/dev/null | head -80 || true - done - - - name: Post eval results review - if: steps.discover.outputs.skills != '' - env: - SKILLS_CSV: ${{ steps.discover.outputs.skills }} - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - const skills = process.env.SKILLS_CSV.split(',').filter(Boolean); - - let body = ''; - for (const skill of skills) { - const summaryPath = `/tmp/${skill}-eval-pr/summary.md`; - if (fs.existsSync(summaryPath)) { - body += fs.readFileSync(summaryPath, 'utf8'); - } else { - body += `### ${skill}\n\n> No results produced. See workflow logs.\n\n`; - } - } - - const pluginJson = JSON.parse(fs.readFileSync('plugins/sdlc-workflow/.claude-plugin/plugin.json', 'utf8')); - const version = pluginJson.version; - - body += '---\n'; - body += `*Generated by [sdlc-workflow/run-evals](plugins/sdlc-workflow/skills/run-evals) v${version}*\n`; - - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number - }); - const marker = '## Eval Results'; - const existing = reviews.find(r => - r.user?.login === 'github-actions[bot]' && r.body?.startsWith(marker) - ); - - if (existing) { - await github.rest.pulls.updateReview({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - review_id: existing.id, - body - }); - } else { - await github.rest.pulls.createReview({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - event: 'COMMENT', - body - }); - } + - run: echo "PR touches eval paths — dispatching eval run" diff --git a/docs/specs/2026-04-21-eval-skills-ci-workflow-design.md b/docs/specs/2026-04-21-eval-skills-ci-workflow-design.md index ff0771609..136b1a26f 100644 --- a/docs/specs/2026-04-21-eval-skills-ci-workflow-design.md +++ b/docs/specs/2026-04-21-eval-skills-ci-workflow-design.md @@ -166,6 +166,7 @@ The prompt describes the task in natural language (slash commands are interactiv - No secrets are exposed beyond `ANTHROPIC_API_KEY` - This is a feasibility proof, not a production gate - Future hardening: migrate to `--permission-mode dontAsk` with `--allowedTools` once tool requirements are known from initial runs. +- Fork PR secret access is handled via `workflow_run` dispatch — see [Eval PR Fork Dispatch](2026-05-07-eval-pr-fork-dispatch-design.md). ## Out of Scope diff --git a/docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md b/docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md new file mode 100644 index 000000000..2e96c4a84 --- /dev/null +++ b/docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md @@ -0,0 +1,285 @@ +# Eval PR Fork Dispatch + +**Date**: 2026-05-07 +**Scope**: Secure eval execution for fork PRs via `workflow_run` dispatch +**Relates to**: [Eval Skills CI Workflow](2026-04-21-eval-skills-ci-workflow-design.md) + +## Problem + +The `eval-pr.yml` workflow requires repository secrets (`GCP_SA_KEY`, `CLOUD_ML_REGION`) to authenticate to Google Cloud for running Claude Code via Vertex AI. GitHub does not pass secrets to workflows triggered from forked repositories: + +> "With the exception of `GITHUB_TOKEN`, secrets are not passed to the runner when a workflow is triggered from a forked repository." +> — [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions) + +This means fork PRs that modify skills or evals cannot produce eval results. + +## Solution + +Split `eval-pr.yml` into two workflows using the `workflow_run` event, which runs in the base repository context with full secret access: + +> "The workflow started by the `workflow_run` event is able to access secrets and write tokens, even if the previous workflow was not." +> — [Events that trigger workflows: workflow_run](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run) + +A trust check auto-approves eval runs for repository collaborators with write access, while external contributors require manual approval via a GitHub protected environment. + +## Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Dispatch mechanism | `workflow_run` | Only mechanism that works for fork PRs — `workflow_dispatch`/`repository_dispatch` require write tokens that fork PR `GITHUB_TOKEN` lacks | +| Trust signal | Collaborator permission API | `GET /repos/{owner}/{repo}/collaborators/{username}/permission` — uses GitHub's actual permission model, handles teams automatically, no file parsing | +| Trust threshold | `admin` or `write` | Collaborators with write access can already push to the repo directly — granting them secret access in CI adds no new attack surface | +| Untrusted gate | Protected environment | GitHub's built-in approval mechanism — reviewers click "Approve" in the Actions UI | +| PR coverage | All PRs (fork and non-fork) | Single code path avoids duplicate logic and ensures consistent behavior | +| Data flow | No artifact — Stage 2 derives everything | Artifact is attacker-controlled (fork PR can modify Stage 1 workflow). Stage 2 derives PR identity from GitHub API and skills from git diff on the checked-out merge commit | + +## Architecture + +``` +Fork PR opened + │ + ▼ +┌──────────────────────────────┐ +│ eval-pr.yml (pull_request) │ Path-filtered gate only +│ No work — just triggers │ +│ workflow_run on completion │ +└──────────────┬───────────────┘ + │ workflow_run (completed) + ▼ +┌──────────────────────────────────────┐ +│ eval-pr-run.yml (workflow_run) │ Base repo context — secrets available +│ │ +│ Job 1: discover │ +│ - Set pending commit status on PR │ +│ - Resolve PR identity via GitHub API│ +│ - Checkout PR merge commit │ +│ - Discover skills via git diff │ +│ - Check collaborator permission │ +│ - Update status if awaiting approval│ +│ - Output: trusted, skills, PR# │ +│ │ +│ Job 2: gate │ +│ - Runs ONLY if trusted != true │ +│ - environment: eval-protected │ +│ - Blocks until reviewer approves │ +│ │ +│ Job 3: run-evals │ +│ - Checkout PR merge commit │ +│ - Authenticate GCP │ +│ - Run evals │ +│ - Post PR review │ +│ │ +│ Job 4: report-status │ +│ - Set final commit status (pass/fail)│ +└──────────────────────────────────────┘ +``` + +## Workflow 1: eval-pr.yml (modified) + +### Changes from Current + +The workflow is reduced to a minimal path-filtered trigger. All logic is removed — no checkout, no discovery, no artifact. Its sole purpose is to act as a gate: GitHub's path filter ensures it only runs when skill or eval files change, and its completion triggers Stage 2 via `workflow_run`. + +### Trigger + +Unchanged: + +```yaml +on: + pull_request: + branches: [main] + paths: + - 'plugins/sdlc-workflow/skills/**/*.md' + - 'evals/**/evals.json' + - '.github/workflows/eval-pr.yml' +``` + +### Steps + +Single no-op step. The workflow just needs to complete successfully to trigger Stage 2. + +## Workflow 2: eval-pr-run.yml (new) + +### Trigger + +```yaml +on: + workflow_run: + workflows: ["Eval PR"] + types: [completed] +``` + +No path filtering — `workflow_run` does not support it. The `completed` type fires on both success and failure, so the `discover` job checks `github.event.workflow_run.conclusion == 'success'` before proceeding. + +### Permissions + +```yaml +permissions: + contents: read + pull-requests: write +``` + +### Job 1: discover + +Resolves PR identity from the GitHub API, checks out the PR merge commit, discovers changed skills via git diff, and determines trust level. Skips early if the triggering workflow failed. No data from the triggering workflow is used — everything is derived independently. + +**Steps:** + +1. **Resolve PR identity and check trust** via `actions/github-script@v7`: + - Find PR via `pulls.list` filtered by `base: 'main'`, matched by `head.sha == workflow_run.head_sha` + - Check collaborator permission level for the PR author + - Output `pr_number`, `base_sha`, `author`, `trusted` + +Note: `listPullRequestsAssociatedWithCommit` does not work for fork PRs because the commit lives in the fork's repository, not the base repo's commit graph. `pulls.list` with SHA matching is the reliable alternative. + +```javascript +const headSha = context.payload.workflow_run.head_sha; +const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, repo: context.repo.repo, + state: 'open', base: 'main', per_page: 100 +}); +const pr = prs.find(p => p.head.sha === headSha); +const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, repo: context.repo.repo, username: pr.user.login +}); +const trusted = ['admin', 'write'].includes(data.permission); +``` + +2. **Checkout PR merge commit** — `actions/checkout@v4` with `ref: refs/pull//merge` and `fetch-depth: 0` + +3. **Discover changed skills** — git diff against `base_sha`, same logic as the original `eval-pr.yml` discovery step (map changed files to eval suites) + +4. **Set outputs**: `skills`, `pr_number`, `author`, `trusted` + +### Job 2: gate + +```yaml +gate: + needs: discover + if: >- + needs.discover.result == 'success' && + needs.discover.outputs.trusted != 'true' && + needs.discover.outputs.skills != '' + runs-on: ubuntu-latest + environment: eval-protected + steps: + - run: echo "Approved by reviewer" +``` + +Skipped entirely for trusted authors and when there are no skills to evaluate. For untrusted authors, the job is created but paused until a designated reviewer approves it in the Actions UI. + +### Job 3: run-evals + +```yaml +run-evals: + needs: [discover, gate] + if: >- + !cancelled() && + needs.discover.result == 'success' && + needs.discover.outputs.skills != '' && + (needs.discover.outputs.trusted == 'true' || needs.gate.result == 'success') + runs-on: ubuntu-latest +``` + +**Steps:** + +1. **Checkout PR merge commit**: + +```yaml +- uses: actions/checkout@v4 + with: + ref: refs/pull/${{ needs.discover.outputs.pr_number }}/merge + fetch-depth: 0 +``` + +2. **Authenticate to Google Cloud** — existing step, moved from `eval-pr.yml`: + +```yaml +- uses: google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} +``` + +3. **Install Claude Code** — existing step, moved from `eval-pr.yml` + +4. **Run PR evals** — existing step, moved from `eval-pr.yml`. Uses `needs.discover.outputs.skills` instead of the inline discovery output. + +5. **Post eval results review** — existing step, moved from `eval-pr.yml`. Uses `needs.discover.outputs.pr_number` instead of `context.issue.number` (which is unavailable in `workflow_run` context). + +### Job 4: report-status + +Sets a final commit status on the PR's head SHA via the Commit Status API. Runs unconditionally (`always()`) after all other jobs complete so a pending status is never left unresolved — including when the discover job itself fails. + +This is necessary because `workflow_run` workflows run in the default branch context and don't appear as status checks on the PR page. Without explicit commit statuses, PR authors and reviewers have no visibility that evals are pending, waiting for approval, running, or complete. + +Status transitions: +- **pending** — set immediately when the discover job starts ("Eval run in progress...") +- **pending** — updated if the author is untrusted ("Waiting for approval in eval-protected environment") +- **success** — eval run completed, results posted as PR review +- **failure** — discover failed (could not resolve PR context), approval rejected, or eval run failed + +All statuses use the same `context: 'Eval PR Run'` so GitHub updates the existing status rather than creating duplicates. + +## Trust Model + +### Trusted Authors (auto-approve) + +Repository collaborators with `write` or `admin` permission. Determined by the GitHub REST API: + +``` +GET /repos/{owner}/{repo}/collaborators/{username}/permission +``` + +This returns the highest permission level across all grant sources (direct, teams, org, enterprise). Collaborators with write access can already push directly to the repository, so granting them secret access in CI adds no new attack surface. + +### Untrusted Authors (manual approval) + +All other PR authors. Their eval runs require a reviewer to click "Approve" in the GitHub Actions UI. The protected environment `eval-protected` enforces this gate. + +### Security Properties + +- The `workflow_run` workflow executes on the base repository's default branch — a fork PR cannot modify the workflow definition or trust logic +- No data passes from Stage 1 to Stage 2 — there is no artifact. Stage 2 derives everything independently: PR identity from the GitHub API, skills from git diff on the checked-out merge commit. This eliminates artifact poisoning as an attack vector entirely. +- The collaborator permission check uses GitHub's API, not a file in the repo — it cannot be manipulated by a PR +- Once approved, the workflow does execute skill files from the PR with access to secrets via environment variables — this is inherent to the use case (evaluating skills requires running them) +- The `dontAsk` permission mode with explicit `--allowedTools` limits Claude Code's capabilities, but a malicious skill could still read environment variables via the Bash tool + +### Why Not Other Approaches + +| Approach | Why not | +|----------|---------| +| `workflow_dispatch` / `repository_dispatch` | Cannot be triggered from fork PR workflows — `GITHUB_TOKEN` is read-only for fork PRs | +| `pull_request_target` | Runs in base repo context with secrets, but GitHub warns: "Avoid using this event if you need to build or run code from the pull request" | +| CODEOWNERS parsing | Text file parsing is fragile (teams, globs, multiple entries); collaborator API is authoritative | +| Skip evals for fork PRs | Fork contributors would never see eval results | + +## Setup Requirements + +### GitHub Environment + +Create a protected environment in the repository: + +1. Settings > Environments > New environment: `eval-protected` +2. Add required reviewers (at minimum, repository maintainers) +3. No environment-specific secrets needed — existing repository-level secrets (`GCP_SA_KEY`, `CLOUD_ML_REGION`) are available to all `workflow_run` jobs + +### No Other Infrastructure + +- No CODEOWNERS file needed +- No GitHub App or PAT required +- No changes to `eval-baseline.yml` (runs on push to main, secrets always available) + +## File Changes + +| File | Change | +|------|--------| +| `.github/workflows/eval-pr.yml` | Reduce to minimal path-filtered trigger (no discovery, no artifact) | +| `.github/workflows/eval-pr-run.yml` | New workflow: trust check, gate, eval execution, review posting | +| `docs/specs/2026-04-21-eval-skills-ci-workflow-design.md` | Add reference to this spec | + +## References + +- [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-for-github-actions/security-guides/using-secrets-in-github-actions) — fork PR secret restriction +- [Events that trigger workflows: workflow_run](https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#workflow_run) — secret access in `workflow_run` +- [GitHub Security Lab: Preventing pwn requests](https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/) — two-stage pattern for fork PR processing +- [Repository collaborators API](https://docs.github.com/en/rest/collaborators/collaborators) — permission check endpoint diff --git a/docs/superpowers/plans/2026-05-07-eval-pr-fork-dispatch.md b/docs/superpowers/plans/2026-05-07-eval-pr-fork-dispatch.md new file mode 100644 index 000000000..9e80bb879 --- /dev/null +++ b/docs/superpowers/plans/2026-05-07-eval-pr-fork-dispatch.md @@ -0,0 +1,373 @@ +# Eval PR Fork Dispatch Implementation Plan + +> **Superseded:** The implementation evolved past this plan during security hardening. The artifact was eliminated entirely — Stage 2 now derives all data from the GitHub API and git diff. See [the spec](../specs/2026-05-07-eval-pr-fork-dispatch-design.md) for the current design. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split `eval-pr.yml` into a two-stage `workflow_run` dispatch pattern so fork PRs can run evals with secret access, gated by a collaborator trust check. + +**Architecture:** Stage 1 (`eval-pr.yml`, `pull_request` trigger) discovers changed skills and uploads an artifact with PR metadata. Stage 2 (`eval-pr-run.yml`, `workflow_run` trigger) downloads the artifact, checks if the PR author is a trusted collaborator, gates untrusted authors behind a protected environment, then runs evals and posts the PR review. + +**Tech Stack:** GitHub Actions, `actions/upload-artifact@v4`, `actions/download-artifact@v4`, `actions/github-script@v7`, GitHub REST API (collaborator permissions) + +**Spec:** `docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md` + +--- + +### Task 1: Modify eval-pr.yml to discovery-only with artifact upload + +Strip all secret-dependent steps (GCP auth, Claude Code install, eval execution, review posting) and add metadata write + artifact upload. + +**Files:** +- Modify: `.github/workflows/eval-pr.yml` + +- [ ] **Step 1: Rewrite eval-pr.yml** + +Replace the entire file with the discovery-only version. The job is renamed from `eval-pr` to `discover` to reflect its new purpose. Permissions drop to `contents: read` only. + +```yaml +# Discovers changed skills for PR evals and uploads metadata for the +# dispatch workflow (eval-pr-run.yml) to execute with secret access. +# +# See docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md for full design. + +name: Eval PR + +on: + pull_request: + branches: [main] + paths: + - 'plugins/sdlc-workflow/skills/**/*.md' + - 'evals/**/evals.json' + - '.github/workflows/eval-pr.yml' + +jobs: + discover: + name: Discover Changed Skills + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Discover changed skills + id: discover + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + changed_files=$(git diff --name-only "$BASE_SHA"...HEAD) + echo "Changed files:" + echo "$changed_files" + + skills="" + for file in $changed_files; do + skill="" + if [[ "$file" =~ ^plugins/sdlc-workflow/skills/([^/]+)/ ]]; then + skill="${BASH_REMATCH[1]}" + elif [[ "$file" =~ ^evals/([^/]+)/ ]]; then + skill="${BASH_REMATCH[1]}" + fi + + if [ -n "$skill" ] && [ -f "evals/${skill}/evals.json" ]; then + if [[ ! ",$skills," == *",$skill,"* ]]; then + skills="${skills:+${skills},}${skill}" + fi + fi + done + + echo "skills=${skills}" >> "$GITHUB_OUTPUT" + echo "Discovered changed skills with evals: ${skills:-none}" + + - name: Write PR metadata + env: + SKILLS: ${{ steps.discover.outputs.skills }} + run: | + jq -n \ + --arg skills "$SKILLS" \ + --argjson pr_number ${{ github.event.pull_request.number }} \ + --arg head_sha "${{ github.event.pull_request.head.sha }}" \ + --arg author "${{ github.event.pull_request.user.login }}" \ + '{skills: $skills, pr_number: $pr_number, head_sha: $head_sha, author: $author}' \ + > eval-pr-metadata.json + cat eval-pr-metadata.json + + - name: Upload PR metadata + uses: actions/upload-artifact@v4 + with: + name: eval-pr-metadata + path: eval-pr-metadata.json +``` + +- [ ] **Step 2: Validate YAML syntax** + +Run: `python3 -c "import yaml; yaml.safe_load(open('.github/workflows/eval-pr.yml')); print('Valid YAML')"` + +Expected: `Valid YAML` + +- [ ] **Step 3: Commit** + +```bash +git add .github/workflows/eval-pr.yml +git commit -m "refactor(ci): strip eval-pr.yml to discovery-only with artifact upload + +Stage 1 of the workflow_run dispatch pattern for fork PR secret access. +Eval execution moves to eval-pr-run.yml (next commit)." +``` + +--- + +### Task 2: Create eval-pr-run.yml dispatch workflow + +The new workflow with three jobs: `discover` (download artifact + trust check), `gate` (environment approval for untrusted authors), and `run-evals` (checkout PR, run evals, post review). + +**Files:** +- Create: `.github/workflows/eval-pr-run.yml` + +- [ ] **Step 1: Create eval-pr-run.yml** + +```yaml +# Runs evals for PRs via workflow_run dispatch, enabling secret access +# for fork PRs. Pairs with eval-pr.yml (Stage 1: discovery). +# +# Trust model: +# - Collaborators with write/admin permission: evals run automatically +# - External contributors: require manual approval via eval-protected environment +# +# See docs/specs/2026-05-07-eval-pr-fork-dispatch-design.md for full design. + +name: Eval PR Run + +on: + workflow_run: + workflows: ["Eval PR"] + types: [completed] + +permissions: + contents: read + pull-requests: write + actions: read + +jobs: + discover: + name: Discover PR Context + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + outputs: + skills: ${{ steps.metadata.outputs.skills }} + pr_number: ${{ steps.metadata.outputs.pr_number }} + head_sha: ${{ steps.metadata.outputs.head_sha }} + author: ${{ steps.metadata.outputs.author }} + trusted: ${{ steps.metadata.outputs.trusted }} + steps: + - name: Download PR metadata + uses: actions/download-artifact@v4 + with: + name: eval-pr-metadata + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Parse metadata and check trust + id: metadata + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const metadata = JSON.parse(fs.readFileSync('eval-pr-metadata.json', 'utf8')); + + core.setOutput('skills', metadata.skills); + core.setOutput('pr_number', metadata.pr_number.toString()); + core.setOutput('head_sha', metadata.head_sha); + core.setOutput('author', metadata.author); + + let trusted = false; + try { + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: metadata.author + }); + trusted = ['admin', 'write'].includes(data.permission); + console.log(`Author ${metadata.author}: permission=${data.permission}, trusted=${trusted}`); + } catch (e) { + console.log(`Author ${metadata.author}: not a collaborator (${e.status}), trusted=false`); + } + + core.setOutput('trusted', trusted.toString()); + + gate: + name: Approval Gate + needs: discover + if: needs.discover.outputs.trusted != 'true' && needs.discover.outputs.skills != '' + runs-on: ubuntu-latest + environment: eval-protected + steps: + - run: echo "Approved by reviewer" + + run-evals: + name: Run PR Evals + needs: [discover, gate] + if: >- + !cancelled() && + needs.discover.result == 'success' && + needs.discover.outputs.skills != '' && + (needs.discover.outputs.trusted == 'true' || needs.gate.result == 'success') + runs-on: ubuntu-latest + steps: + - name: Checkout PR merge commit + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ needs.discover.outputs.pr_number }}/merge + fetch-depth: 0 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Install Claude Code + run: curl -fsSL https://claude.ai/install.sh | bash + + - name: Run PR evals + env: + CLAUDE_CODE_USE_VERTEX: "1" + CLOUD_ML_REGION: ${{ secrets.GCP_CLOUD_ML_REGION }} + ANTHROPIC_DEFAULT_SONNET_MODEL: "claude-sonnet-4-6" + ANTHROPIC_DEFAULT_OPUS_MODEL: "claude-opus-4-6" + ANTHROPIC_MODEL: "claude-opus-4-6" + run: | + IFS=',' read -ra SKILLS <<< "${{ needs.discover.outputs.skills }}" + + for skill in "${SKILLS[@]}"; do + eval_count=$(jq '.evals | length' "evals/${skill}/evals.json") + workspace="/tmp/${skill}-eval-pr" + mkdir -p "${workspace}" + + echo "=== Running PR evals for ${skill} (${eval_count} cases) ===" + echo "Workspace: ${workspace}" + + claude -p "$(cat <&1 || { + echo "::warning::PR eval run failed for ${skill} (exit $?)" + continue + } + + echo "--- Workspace contents ---" + find "${workspace}" -type f 2>/dev/null | head -80 || true + done + + - name: Post eval results review + env: + SKILLS_CSV: ${{ needs.discover.outputs.skills }} + PR_NUMBER: ${{ needs.discover.outputs.pr_number }} + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const skills = process.env.SKILLS_CSV.split(',').filter(Boolean); + const prNumber = parseInt(process.env.PR_NUMBER); + + let body = ''; + for (const skill of skills) { + const summaryPath = `/tmp/${skill}-eval-pr/summary.md`; + if (fs.existsSync(summaryPath)) { + body += fs.readFileSync(summaryPath, 'utf8'); + } else { + body += `### ${skill}\n\n> No results produced. See workflow logs.\n\n`; + } + } + + const pluginJson = JSON.parse(fs.readFileSync('plugins/sdlc-workflow/.claude-plugin/plugin.json', 'utf8')); + const version = pluginJson.version; + + body += '---\n'; + body += `*Generated by [sdlc-workflow/run-evals](plugins/sdlc-workflow/skills/run-evals) v${version}*\n`; + + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + const marker = '## Eval Results'; + const existing = reviews.find(r => + r.user?.login === 'github-actions[bot]' && r.body?.startsWith(marker) + ); + + if (existing) { + await github.rest.pulls.updateReview({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + review_id: existing.id, + body + }); + } else { + await github.rest.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + event: 'COMMENT', + body + }); + } +``` + +- [ ] **Step 2: Validate YAML syntax** + +Run: `python3 -c "import yaml; yaml.safe_load(open('.github/workflows/eval-pr-run.yml')); print('Valid YAML')"` + +Expected: `Valid YAML` + +- [ ] **Step 3: Commit** + +```bash +git add .github/workflows/eval-pr-run.yml +git commit -m "feat(ci): add eval-pr-run.yml for fork PR eval dispatch + +Stage 2 of the workflow_run dispatch pattern. Downloads PR metadata +from eval-pr.yml, gates untrusted authors via eval-protected environment, +then runs evals with secret access and posts the PR review." +``` + +--- + +### Task 3: Update existing spec with cross-reference + +Add a reference from the original eval CI spec to the fork dispatch spec. + +**Files:** +- Modify: `docs/specs/2026-04-21-eval-skills-ci-workflow-design.md` + +- [ ] **Step 1: Add fork dispatch reference** + +Add the following line at the end of the "Security Considerations" section (after line 168, before the "Out of Scope" heading): + +```markdown +- Fork PR secret access is handled via `workflow_run` dispatch — see [Eval PR Fork Dispatch](2026-05-07-eval-pr-fork-dispatch-design.md). +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/specs/2026-04-21-eval-skills-ci-workflow-design.md +git commit -m "docs(specs): add fork dispatch cross-reference to eval CI spec" +``` + +--- + +### Post-Implementation: GitHub Environment Setup + +After merging, create the `eval-protected` environment in the repository: + +1. Go to Settings > Environments > New environment +2. Name: `eval-protected` +3. Add required reviewers (repository maintainers) +4. No environment-specific secrets needed — existing repository-level secrets are used