Grant scoped tool permissions to skills under test

.gitignore

@@ -1,5 +1,6 @@
 __pycache__/
 *.pyc
+node_modules/
 .ruff_cache/
 .venv/
 .env

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+
+- Add `allowed-tools` and `permission-mode` inputs to grant scoped tool permissions to the skill under test, forwarded to `claude --allowedTools` / `--permission-mode`. Fixes skills that diagnose by running read-only commands failing because every tool call was auto-denied (#2)
+- Support per-case `allowed_tools` (string or list) and `permission_mode` overrides in eval YAML, validated before any API calls
+
 ## v1.1.0
 
 - Auto-fix YAML plain scalars containing `: ` (colon-space) — no more parse errors for natural-language criteria

README.md

@@ -186,6 +186,8 @@ Within a single skill, eval cases run sequentially to avoid Anthropic API rate l
 | `anthropic-api-key` | Yes | - | Anthropic API key for the `claude` CLI |
 | `pass-threshold` | No | `80` | Minimum pass rate (0-100) to succeed |
 | `timeout` | No | `120` | Timeout per eval case in seconds |
+| `allowed-tools` | No | `''` | Tool allow-list granted to the skill under test, forwarded to `claude --allowedTools` (e.g. `Bash(kubectl get:*),Bash(gh api:*),Read`). Per-case `allowed_tools` overrides it |
+| `permission-mode` | No | `''` | Permission mode forwarded to `claude --permission-mode` (`default`, `acceptEdits`, `plan`, `bypassPermissions`). Per-case `permission_mode` overrides it |
 | `post-comment` | No | `true` | Post results as a PR comment |
 | `github-token` | No | `${{ github.token }}` | Token for PR comments |
 | `upload-viewer` | No | `true` | Upload eval-viewer HTML as an artifact |
@@ -233,10 +235,26 @@ criteria:                       # success criteria - ALL must pass
   - "Uses for_each, not count, for multiple resources"
 expect_skill: true              # optional - default true
 timeout: 120                    # optional - default from action input
+allowed_tools:                  # optional - overrides the action `allowed-tools` input
+  - "Bash(kubectl get:*)"
+  - "Read"
+permission_mode: default        # optional - overrides the action `permission-mode` input
 ```
 
 Include at least one negative trigger case (`expect_skill: false`).
 
+### Granting tool permissions
+
+Skills that diagnose by running read-only commands (`kubectl get`, `gh api`, `gcloud ... list`) need those commands to actually execute. By default the model runs with no tool permissions, so every `Bash` call is denied and such skills fail spuriously. Grant a scoped allow-list via the `allowed-tools` input (or per-case `allowed_tools`):
+
+```yaml
+# action invocation
+with:
+  allowed-tools: "Bash(kubectl get:*),Bash(gh api:*),Read"
+```
+
+Prefer scoped allow-lists over `permission-mode: bypassPermissions` so read-only commands run while a skill's refusal-of-mutation behavior can still be tested. Use a per-case `allowed_tools` to widen scope for a single case (e.g. allow `Bash(kubectl delete:*)` only in a case that asserts the skill refuses to run it).
+
 ## PR comment
 
 The action posts (or updates) a PR comment with:

action.yml

@@ -24,6 +24,14 @@ inputs:
     description: 'Timeout per eval case in seconds'
     required: false
     default: '120'
+  allowed-tools:
+    description: 'Comma/space-separated tool allow-list granted to the skill under test (e.g. "Bash(kubectl get:*),Bash(gh api:*),Read"). Forwarded to claude --allowedTools. Per-case allowed_tools overrides this.'
+    required: false
+    default: ''
+  permission-mode:
+    description: 'Permission mode for the skill under test (default, acceptEdits, plan, bypassPermissions). Forwarded to claude --permission-mode. Per-case permission_mode overrides this.'
+    required: false
+    default: ''
   post-comment:
     description: 'Post results as a PR comment'
     required: false
@@ -85,20 +93,26 @@ runs:
 
     - name: Install dependencies
       shell: bash
+      env:
+        GITHUB_ACTION_PATH: ${{ github.action_path }}
       run: |
         pip install pyyaml --quiet
-        npm install -g @anthropic-ai/claude-code --quiet 2>/dev/null || true
-        claude --version || { echo "::error::Failed to install claude CLI"; exit 1; }
+        npm ci --prefix "$GITHUB_ACTION_PATH" --silent
+        "$GITHUB_ACTION_PATH/node_modules/.bin/claude" --version || { echo "::error::Failed to install claude CLI"; exit 1; }
 
     - name: Run eval pipeline
       id: eval
       shell: bash
-      run: python "$GITHUB_ACTION_PATH/scripts/eval.py"
+      run: |
+        export PATH="$GITHUB_ACTION_PATH/node_modules/.bin:$PATH"
+        python "$GITHUB_ACTION_PATH/scripts/eval.py"
       env:
         GITHUB_ACTION_PATH: ${{ github.action_path }}
         SKILL_NAME: ${{ inputs.skill-name }}
         SKILL_PATH: ${{ inputs.skill-path }}
         EVAL_TIMEOUT: ${{ inputs.timeout }}
+        ALLOWED_TOOLS: ${{ inputs.allowed-tools }}
+        PERMISSION_MODE: ${{ inputs.permission-mode }}
         PASS_THRESHOLD: ${{ inputs.pass-threshold }}
         ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
         WORKSPACE: ${{ runner.temp }}/skill-eval/${{ inputs.skill-name }}

package.json

@@ -0,0 +1,9 @@
+{
+  "name": "skill-eval-action",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Pins the claude CLI used by the action so installs come from a committed lockfile",
+  "dependencies": {
+    "@anthropic-ai/claude-code": "2.1.185"
+  }
+}

scripts/eval.py

@@ -28,7 +28,11 @@
 PLUGIN_ROOT = SKILL_PATH.resolve().parent.parent
 WORKSPACE = Path(os.environ["WORKSPACE"])
 EVAL_TIMEOUT = int(os.environ.get("EVAL_TIMEOUT", "120"))
+ALLOWED_TOOLS = os.environ.get("ALLOWED_TOOLS", "").strip()
+PERMISSION_MODE = os.environ.get("PERMISSION_MODE", "").strip()
 PASS_THRESHOLD = float(os.environ.get("PASS_THRESHOLD", "80"))
+
+VALID_PERMISSION_MODES = {"default", "acceptEdits", "plan", "bypassPermissions"}
 MAX_RETRIES = int(os.environ.get("MAX_RETRIES", "3"))
 RETRY_DELAY = int(os.environ.get("RETRY_DELAY", "10"))
 
@@ -187,21 +191,64 @@ def validate_cases(cases: list[dict]) -> list[str]:
         if to is not None and not isinstance(to, (int, float)):
             errors.append(f"{prefix}: 'timeout' must be a number, got {type(to).__name__}")
 
+        # allowed_tools — optional, string or list of strings
+        at = case.get("allowed_tools")
+        if at is not None:
+            if isinstance(at, list):
+                for i, t in enumerate(at):
+                    if not isinstance(t, str):
+                        errors.append(f"{prefix}: allowed_tools[{i}] must be a string, got {type(t).__name__}")
+            elif not isinstance(at, str):
+                errors.append(f"{prefix}: 'allowed_tools' must be a string or list of strings, got {type(at).__name__}")
+
+        # permission_mode — optional, one of the valid modes
+        pm = case.get("permission_mode")
+        if pm is not None:
+            if not isinstance(pm, str):
+                errors.append(f"{prefix}: 'permission_mode' must be a string, got {type(pm).__name__}")
+            elif pm not in VALID_PERMISSION_MODES:
+                errors.append(
+                    f"{prefix}: 'permission_mode' must be one of {sorted(VALID_PERMISSION_MODES)}, got '{pm}'"
+                )
+
     return errors
 
 
 # ---------------------------------------------------------------------------
 # Execution
 # ---------------------------------------------------------------------------
 
-def _run_claude(prompt: str, work_dir: Path, timeout: int) -> subprocess.CompletedProcess:
-    """Run claude -p with retries on timeout/error."""
+def _normalize_allowed_tools(value) -> str:
+    """Coerce allowed_tools (str or list) into a comma-separated string."""
+    if isinstance(value, list):
+        return ",".join(str(t).strip() for t in value if str(t).strip())
+    return str(value).strip()
+
+
+def _run_claude(
+    prompt: str,
+    work_dir: Path,
+    timeout: int,
+    allowed_tools: str = "",
+    permission_mode: str = "",
+) -> subprocess.CompletedProcess:
+    """Run claude -p with retries on timeout/error.
+
+    allowed_tools / permission_mode grant scoped tool permissions so skills
+    that diagnose by running read-only commands can actually execute them.
+    """
     env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"}
 
+    cmd = ["claude", "-p", prompt, "--add-dir", str(PLUGIN_ROOT), "--output-format", "stream-json", "--verbose"]
+    if allowed_tools:
+        cmd += ["--allowedTools", allowed_tools]
+    if permission_mode:
+        cmd += ["--permission-mode", permission_mode]
+
     for attempt in range(1, MAX_RETRIES + 1):
         try:
             result = subprocess.run(
-                ["claude", "-p", prompt, "--add-dir", str(PLUGIN_ROOT), "--output-format", "stream-json", "--verbose"],
+                cmd,
                 capture_output=True, text=True,
                 timeout=timeout, cwd=str(work_dir), env=env,
             )
@@ -287,10 +334,17 @@ def execute_case(case: dict, skill_content: str, case_dir: Path) -> dict:
     else:
         prompt = raw_prompt
 
+    # Per-case scopes override the action-level defaults
+    allowed_tools = _normalize_allowed_tools(case.get("allowed_tools", ALLOWED_TOOLS))
+    permission_mode = str(case.get("permission_mode", PERMISSION_MODE)).strip()
+
     start = time.time()
 
     try:
-        result = _run_claude(prompt, work_dir, case.get("timeout", EVAL_TIMEOUT))
+        result = _run_claude(
+            prompt, work_dir, case.get("timeout", EVAL_TIMEOUT),
+            allowed_tools=allowed_tools, permission_mode=permission_mode,
+        )
         elapsed = time.time() - start
         parsed = _parse_stream_json(result.stdout)