From e1d8eb73c84d76a37831d0d5afe3be10a9ecbc21 Mon Sep 17 00:00:00 2001 From: Nick Nikolakakis Date: Mon, 22 Jun 2026 19:01:14 +0300 Subject: [PATCH 1/2] Grant scoped tool permissions to skills under test (#2) Eval runs invoked claude -p with no tool permissions, so every Bash call was auto-denied with "requires approval". Skills that diagnose by running read-only commands (kubectl get, gh api, gcloud ... list) could never execute their steps and failed spuriously, the eval measured the permission gate rather than the skill. Add allowed-tools and permission-mode action inputs, forwarded to the execute call as claude --allowedTools / --permission-mode. Support per-case allowed_tools (string or list) and permission_mode overrides in eval YAML so a refusal-of-mutation case can widen scope while diagnosis cases stay read-only. Validate the new fields before any API calls and cover normalization plus validation with unit tests. Scoped allow-lists are preferred over a blanket bypass so read-only commands run while mutation-refusal behavior stays testable. --- CHANGELOG.md | 5 +++ README.md | 18 +++++++++++ action.yml | 10 ++++++ scripts/eval.py | 62 +++++++++++++++++++++++++++++++++++--- scripts/test_validation.py | 43 +++++++++++++++++++++++++- 5 files changed, 133 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d87f3b..bf81ce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Add `allowed-tools` and `permission-mode` inputs to grant scoped tool permissions to the skill under test, forwarded to `claude --allowedTools` / `--permission-mode`. Fixes skills that diagnose by running read-only commands failing because every tool call was auto-denied (#2) +- Support per-case `allowed_tools` (string or list) and `permission_mode` overrides in eval YAML, validated before any API calls + ## v1.1.0 - Auto-fix YAML plain scalars containing `: ` (colon-space) — no more parse errors for natural-language criteria diff --git a/README.md b/README.md index 6fc3e96..8fce151 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,8 @@ Within a single skill, eval cases run sequentially to avoid Anthropic API rate l | `anthropic-api-key` | Yes | - | Anthropic API key for the `claude` CLI | | `pass-threshold` | No | `80` | Minimum pass rate (0-100) to succeed | | `timeout` | No | `120` | Timeout per eval case in seconds | +| `allowed-tools` | No | `''` | Tool allow-list granted to the skill under test, forwarded to `claude --allowedTools` (e.g. `Bash(kubectl get:*),Bash(gh api:*),Read`). Per-case `allowed_tools` overrides it | +| `permission-mode` | No | `''` | Permission mode forwarded to `claude --permission-mode` (`default`, `acceptEdits`, `plan`, `bypassPermissions`). Per-case `permission_mode` overrides it | | `post-comment` | No | `true` | Post results as a PR comment | | `github-token` | No | `${{ github.token }}` | Token for PR comments | | `upload-viewer` | No | `true` | Upload eval-viewer HTML as an artifact | @@ -233,10 +235,26 @@ criteria: # success criteria - ALL must pass - "Uses for_each, not count, for multiple resources" expect_skill: true # optional - default true timeout: 120 # optional - default from action input +allowed_tools: # optional - overrides the action `allowed-tools` input + - "Bash(kubectl get:*)" + - "Read" +permission_mode: default # optional - overrides the action `permission-mode` input ``` Include at least one negative trigger case (`expect_skill: false`). +### Granting tool permissions + +Skills that diagnose by running read-only commands (`kubectl get`, `gh api`, `gcloud ... list`) need those commands to actually execute. By default the model runs with no tool permissions, so every `Bash` call is denied and such skills fail spuriously. Grant a scoped allow-list via the `allowed-tools` input (or per-case `allowed_tools`): + +```yaml +# action invocation +with: + allowed-tools: "Bash(kubectl get:*),Bash(gh api:*),Read" +``` + +Prefer scoped allow-lists over `permission-mode: bypassPermissions` so read-only commands run while a skill's refusal-of-mutation behavior can still be tested. Use a per-case `allowed_tools` to widen scope for a single case (e.g. allow `Bash(kubectl delete:*)` only in a case that asserts the skill refuses to run it). + ## PR comment The action posts (or updates) a PR comment with: diff --git a/action.yml b/action.yml index 230f836..fa3339b 100644 --- a/action.yml +++ b/action.yml @@ -24,6 +24,14 @@ inputs: description: 'Timeout per eval case in seconds' required: false default: '120' + allowed-tools: + description: 'Comma/space-separated tool allow-list granted to the skill under test (e.g. "Bash(kubectl get:*),Bash(gh api:*),Read"). Forwarded to claude --allowedTools. Per-case allowed_tools overrides this.' + required: false + default: '' + permission-mode: + description: 'Permission mode for the skill under test (default, acceptEdits, plan, bypassPermissions). Forwarded to claude --permission-mode. Per-case permission_mode overrides this.' + required: false + default: '' post-comment: description: 'Post results as a PR comment' required: false @@ -99,6 +107,8 @@ runs: SKILL_NAME: ${{ inputs.skill-name }} SKILL_PATH: ${{ inputs.skill-path }} EVAL_TIMEOUT: ${{ inputs.timeout }} + ALLOWED_TOOLS: ${{ inputs.allowed-tools }} + PERMISSION_MODE: ${{ inputs.permission-mode }} PASS_THRESHOLD: ${{ inputs.pass-threshold }} ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} WORKSPACE: ${{ runner.temp }}/skill-eval/${{ inputs.skill-name }} diff --git a/scripts/eval.py b/scripts/eval.py index 8742d42..ded4cfb 100644 --- a/scripts/eval.py +++ b/scripts/eval.py @@ -27,7 +27,11 @@ SKILL_PATH = Path(os.environ["SKILL_PATH"]) WORKSPACE = Path(os.environ["WORKSPACE"]) EVAL_TIMEOUT = int(os.environ.get("EVAL_TIMEOUT", "120")) +ALLOWED_TOOLS = os.environ.get("ALLOWED_TOOLS", "").strip() +PERMISSION_MODE = os.environ.get("PERMISSION_MODE", "").strip() PASS_THRESHOLD = float(os.environ.get("PASS_THRESHOLD", "80")) + +VALID_PERMISSION_MODES = {"default", "acceptEdits", "plan", "bypassPermissions"} MAX_RETRIES = int(os.environ.get("MAX_RETRIES", "3")) RETRY_DELAY = int(os.environ.get("RETRY_DELAY", "10")) @@ -186,6 +190,26 @@ def validate_cases(cases: list[dict]) -> list[str]: if to is not None and not isinstance(to, (int, float)): errors.append(f"{prefix}: 'timeout' must be a number, got {type(to).__name__}") + # allowed_tools — optional, string or list of strings + at = case.get("allowed_tools") + if at is not None: + if isinstance(at, list): + for i, t in enumerate(at): + if not isinstance(t, str): + errors.append(f"{prefix}: allowed_tools[{i}] must be a string, got {type(t).__name__}") + elif not isinstance(at, str): + errors.append(f"{prefix}: 'allowed_tools' must be a string or list of strings, got {type(at).__name__}") + + # permission_mode — optional, one of the valid modes + pm = case.get("permission_mode") + if pm is not None: + if not isinstance(pm, str): + errors.append(f"{prefix}: 'permission_mode' must be a string, got {type(pm).__name__}") + elif pm not in VALID_PERMISSION_MODES: + errors.append( + f"{prefix}: 'permission_mode' must be one of {sorted(VALID_PERMISSION_MODES)}, got '{pm}'" + ) + return errors @@ -193,14 +217,37 @@ def validate_cases(cases: list[dict]) -> list[str]: # Execution # --------------------------------------------------------------------------- -def _run_claude(prompt: str, work_dir: Path, timeout: int) -> subprocess.CompletedProcess: - """Run claude -p with retries on timeout/error.""" +def _normalize_allowed_tools(value) -> str: + """Coerce allowed_tools (str or list) into a comma-separated string.""" + if isinstance(value, list): + return ",".join(str(t).strip() for t in value if str(t).strip()) + return str(value).strip() + + +def _run_claude( + prompt: str, + work_dir: Path, + timeout: int, + allowed_tools: str = "", + permission_mode: str = "", +) -> subprocess.CompletedProcess: + """Run claude -p with retries on timeout/error. + + allowed_tools / permission_mode grant scoped tool permissions so skills + that diagnose by running read-only commands can actually execute them. + """ env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + cmd = ["claude", "-p", prompt, "--output-format", "stream-json", "--verbose"] + if allowed_tools: + cmd += ["--allowedTools", allowed_tools] + if permission_mode: + cmd += ["--permission-mode", permission_mode] + for attempt in range(1, MAX_RETRIES + 1): try: result = subprocess.run( - ["claude", "-p", prompt, "--output-format", "stream-json", "--verbose"], + cmd, capture_output=True, text=True, timeout=timeout, cwd=str(work_dir), env=env, ) @@ -286,10 +333,17 @@ def execute_case(case: dict, skill_content: str, case_dir: Path) -> dict: else: prompt = raw_prompt + # Per-case scopes override the action-level defaults + allowed_tools = _normalize_allowed_tools(case.get("allowed_tools", ALLOWED_TOOLS)) + permission_mode = str(case.get("permission_mode", PERMISSION_MODE)).strip() + start = time.time() try: - result = _run_claude(prompt, work_dir, case.get("timeout", EVAL_TIMEOUT)) + result = _run_claude( + prompt, work_dir, case.get("timeout", EVAL_TIMEOUT), + allowed_tools=allowed_tools, permission_mode=permission_mode, + ) elapsed = time.time() - start parsed = _parse_stream_json(result.stdout) diff --git a/scripts/test_validation.py b/scripts/test_validation.py index 74c7a61..26e7276 100644 --- a/scripts/test_validation.py +++ b/scripts/test_validation.py @@ -17,7 +17,12 @@ os.environ.setdefault("SKILL_PATH", ".") os.environ.setdefault("WORKSPACE", "/tmp/skill-eval-test") -from eval import _safe_yaml_load, discover_evals, validate_cases # noqa: E402 +from eval import ( # noqa: E402 + _normalize_allowed_tools, + _safe_yaml_load, + discover_evals, + validate_cases, +) def test_safe_yaml_load() -> int: @@ -67,6 +72,41 @@ def test_safe_yaml_load() -> int: return errors +def test_allowed_tools() -> int: + """Test allowed_tools normalization and permission_mode validation.""" + errors = 0 + + # list → comma-separated string, blanks dropped + if _normalize_allowed_tools(["Read", " Bash(gh api:*) ", ""]) != "Read,Bash(gh api:*)": + print(f" FAIL: list normalize: {_normalize_allowed_tools(['Read', ' Bash(gh api:*) ', ''])}") + errors += 1 + + # string passes through trimmed + if _normalize_allowed_tools(" Read,Bash ") != "Read,Bash": + print(f" FAIL: string normalize: {_normalize_allowed_tools(' Read,Bash ')}") + errors += 1 + + base = {"prompt": "p", "criteria": ["c"]} + + # valid per-case overrides → no errors + ok = validate_cases([{**base, "allowed_tools": ["Read"], "permission_mode": "plan"}]) + if ok: + print(f" FAIL: valid scopes rejected: {ok}") + errors += 1 + + # bad permission_mode → error + if not validate_cases([{**base, "permission_mode": "yolo"}]): + print(" FAIL: invalid permission_mode not caught") + errors += 1 + + # non-string allowed_tools entry → error + if not validate_cases([{**base, "allowed_tools": [123]}]): + print(" FAIL: non-string allowed_tools entry not caught") + errors += 1 + + return errors + + def test_skills_dir(skills_dir: Path) -> int: """Discover and validate all skills in a directory.""" errors = 0 @@ -112,6 +152,7 @@ def test_skills_dir(skills_dir: Path) -> int: def main() -> None: print("=== _safe_yaml_load unit tests ===") unit_errors = test_safe_yaml_load() + unit_errors += test_allowed_tools() if unit_errors: print(f"FAIL: {unit_errors} unit test(s) failed\n") else: From b312c4b54dfba6a248cb881e9fb8dfe69383ecc9 Mon Sep 17 00:00:00 2001 From: Nick Nikolakakis Date: Mon, 22 Jun 2026 19:22:01 +0300 Subject: [PATCH 2/2] Install claude CLI from a committed lockfile CI's zizmor (unpinned, bumped to 1.26.1) now fails the adhoc-packages audit on `npm install -g @anthropic-ai/claude-code`: an ad-hoc install leaves transitive deps unpinned. Version pinning alone does not satisfy the audit, the documented remediation is a committed lockfile installed with `npm ci`. Add package.json + package-lock.json pinning the claude CLI and install it with `npm ci --prefix`. The eval step prepends the local node_modules/.bin to PATH for its own shell instead of writing to GITHUB_PATH (which trips zizmor's github-env audit). Ignore node_modules. --- .gitignore | 1 + action.yml | 10 ++- package-lock.json | 154 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 9 +++ 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 3521dd9..88afb2d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ *.pyc +node_modules/ .ruff_cache/ .venv/ .env diff --git a/action.yml b/action.yml index fa3339b..67c2ef0 100644 --- a/action.yml +++ b/action.yml @@ -93,15 +93,19 @@ runs: - name: Install dependencies shell: bash + env: + GITHUB_ACTION_PATH: ${{ github.action_path }} run: | pip install pyyaml --quiet - npm install -g @anthropic-ai/claude-code --quiet 2>/dev/null || true - claude --version || { echo "::error::Failed to install claude CLI"; exit 1; } + npm ci --prefix "$GITHUB_ACTION_PATH" --silent + "$GITHUB_ACTION_PATH/node_modules/.bin/claude" --version || { echo "::error::Failed to install claude CLI"; exit 1; } - name: Run eval pipeline id: eval shell: bash - run: python "$GITHUB_ACTION_PATH/scripts/eval.py" + run: | + export PATH="$GITHUB_ACTION_PATH/node_modules/.bin:$PATH" + python "$GITHUB_ACTION_PATH/scripts/eval.py" env: GITHUB_ACTION_PATH: ${{ github.action_path }} SKILL_NAME: ${{ inputs.skill-name }} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..d69455d --- /dev/null +++ b/package-lock.json @@ -0,0 +1,154 @@ +{ + "name": "skill-eval-action", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "skill-eval-action", + "version": "0.0.0", + "dependencies": { + "@anthropic-ai/claude-code": "2.1.185" + } + }, + "node_modules/@anthropic-ai/claude-code": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code/-/claude-code-2.1.185.tgz", + "integrity": "sha512-Hdajoh0bqjSyWaO78jb923Sk0hNUmKQ1kMFnysb/y5M0SPA7Nq9rRpL/my88Jcc/+dM+S7wnHrUyNCWA/IdTog==", + "hasInstallScript": true, + "license": "SEE LICENSE IN README.md", + "bin": { + "claude": "bin/claude.exe" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "@anthropic-ai/claude-code-darwin-arm64": "2.1.185", + "@anthropic-ai/claude-code-darwin-x64": "2.1.185", + "@anthropic-ai/claude-code-linux-arm64": "2.1.185", + "@anthropic-ai/claude-code-linux-arm64-musl": "2.1.185", + "@anthropic-ai/claude-code-linux-x64": "2.1.185", + "@anthropic-ai/claude-code-linux-x64-musl": "2.1.185", + "@anthropic-ai/claude-code-win32-arm64": "2.1.185", + "@anthropic-ai/claude-code-win32-x64": "2.1.185" + } + }, + "node_modules/@anthropic-ai/claude-code-darwin-arm64": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-arm64/-/claude-code-darwin-arm64-2.1.185.tgz", + "integrity": "sha512-wmz4Sil/m4iP/N2kYjzxwPMd42iOe6c9N+knpoAeqy9wITAnEL1SnllGwvm3OcVUTwiJ8tOrefUswnMHVXMXsw==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@anthropic-ai/claude-code-darwin-x64": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-darwin-x64/-/claude-code-darwin-x64-2.1.185.tgz", + "integrity": "sha512-0ugRcohzNWj66meFATJxEoHR1/TTYuLvA344IUUdeSl/ZQ4vQ8W3nQamsA2kjLJEaBYdEKDZBWgUTvR13Wkf3Q==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-arm64": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64/-/claude-code-linux-arm64-2.1.185.tgz", + "integrity": "sha512-z/HIA74aQ2PKs0XqgI6G1mykAXrlbuOSun/EIcHRRm0sy1hA75H9fkGCR0d/DvKRq3+P2ehcXzGxlpmHmQCj0A==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-arm64-musl": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-arm64-musl/-/claude-code-linux-arm64-musl-2.1.185.tgz", + "integrity": "sha512-BFznDZQZ9n/jiulHGr/u6e2TLqUaNtcXUhGP5y1zKwaEnqqCLLKKyzwZ4k0eBnTW+8OgeSdIo5RiNVmHgJAOJw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-x64": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64/-/claude-code-linux-x64-2.1.185.tgz", + "integrity": "sha512-5EXH6o7UHthMKMdCHr+RkhmE4lc6+FM+muKpmQJlgNtfD2c5piXZxZ4lYNa2Sso7G8OVTSiBRYhKRykk6ZoK1A==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-linux-x64-musl": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-linux-x64-musl/-/claude-code-linux-x64-musl-2.1.185.tgz", + "integrity": "sha512-b20aVo8kZYaHD20yqkW99ZMRtQ7DBDf4dmoqmSr14POu0FcZ9NQCAEAUG/kUQSpaR8IxLfXq1fLujQSU+HRuwA==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@anthropic-ai/claude-code-win32-arm64": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-arm64/-/claude-code-win32-arm64-2.1.185.tgz", + "integrity": "sha512-6IB067wUAU1z46sBUIeUmFmmwYK0F1aTVqVVGi/KeCps697fRtLsaUBfaeOanRVyAtEIyNay/rhjeRXHDx7mXA==", + "cpu": [ + "arm64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@anthropic-ai/claude-code-win32-x64": { + "version": "2.1.185", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-code-win32-x64/-/claude-code-win32-x64-2.1.185.tgz", + "integrity": "sha512-HDciy8qvlDWeMQ+CXcdAU+XyRvHZjcrcfi5lG/Hzt2wQkDd/U7026feg14tWQg//NHTM3DwT4IlSnO47+dxmPw==", + "cpu": [ + "x64" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "win32" + ] + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..15c8e07 --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "name": "skill-eval-action", + "version": "0.0.0", + "private": true, + "description": "Pins the claude CLI used by the action so installs come from a committed lockfile", + "dependencies": { + "@anthropic-ai/claude-code": "2.1.185" + } +}