Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ scanner_completed: 2
scanner_failed: 0
scanner_skipped: 0
issues_found: 2
gate: pass
errors: 0
full_results: /tmp/clawscan-csv-summarizer.json
```
Expand Down
17 changes: 16 additions & 1 deletion cmd/clawscan/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,15 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str
fmt.Fprintf(w, "scanner_other: %d\n", summary.ScannerOther)
}
fmt.Fprintf(w, "issues_found: %d\n", summary.IssuesFound)
fmt.Fprintf(w, "gate: %s", summary.Gate)
if len(summary.GateRules) > 0 {
details := make([]string, 0, len(summary.GateRules))
for _, rule := range summary.GateRules {
details = append(details, fmt.Sprintf("%s exit %d -> %s", rule.Scanner, rule.ExitCode, rule.Action))
}
fmt.Fprintf(w, " (%s)", strings.Join(details, ", "))
}
fmt.Fprintln(w)
if summary.HasJudge {
fmt.Fprintf(w, "judge_completed: %d\n", summary.JudgeCompleted)
fmt.Fprintf(w, "judge_failed: %d\n", summary.JudgeFailed)
Expand Down Expand Up @@ -404,6 +413,8 @@ type runSummary struct {
ScannerSkipped int
ScannerOther int
IssuesFound int
Gate string
GateRules []runner.FiredGateRule
HasJudge bool
JudgeCompleted int
JudgeFailed int
Expand All @@ -415,7 +426,7 @@ type runSummary struct {
}

func summarizeRunTargets(result runner.RunTargetsResult) runSummary {
var summary runSummary
summary := runSummary{Gate: "pass"}
if result.Batch != nil {
summary.Profile = result.Batch.Profile
summary.Profiles = result.Batch.Summary.ProfileCount
Expand All @@ -436,6 +447,10 @@ func (summary *runSummary) addArtifact(artifact runner.Artifact) {
summary.Profile = artifact.Profile
}
summary.Targets++
summary.GateRules = append(summary.GateRules, artifact.GateRules...)
if artifact.Gate == "block" || (artifact.Gate == "warn" && summary.Gate == "pass") {
summary.Gate = artifact.Gate
}
for _, result := range artifact.Scanners {
switch result.Status {
case "completed":
Expand Down
83 changes: 83 additions & 0 deletions cmd/clawscan/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ package main

import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"testing"

"github.com/openclaw/clawscan/internal/runner"
)

func TestRunCommandPrintsHelp(t *testing.T) {
Expand Down Expand Up @@ -511,6 +514,86 @@ func TestRunCommandWritesDefaultOutputAndPrintsKeyValueSummary(t *testing.T) {
}
}

func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) {
artifact := runner.Artifact{
Gate: "block",
GateRules: []runner.FiredGateRule{
{Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: 3, Action: "block"},
},
Scanners: map[string]runner.ScannerResult{},
}
var output strings.Builder
printRunSummary(&output, runner.RunTargetsResult{Single: &artifact}, "")
if !strings.Contains(output.String(), "gate: block (my-scanner exit 3 -> block)") {
t.Fatalf("summary missing gate rule:\n%s", output.String())
}
}

func TestPrintRunSummaryKeepsBlockAcrossBatchOrder(t *testing.T) {
for _, runs := range [][]runner.Artifact{
{{Gate: "warn", Scanners: map[string]runner.ScannerResult{}}, {Gate: "block", Scanners: map[string]runner.ScannerResult{}}},
{{Gate: "block", Scanners: map[string]runner.ScannerResult{}}, {Gate: "warn", Scanners: map[string]runner.ScannerResult{}}},
} {
summary := summarizeRunTargets(runner.RunTargetsResult{Batch: &runner.BatchArtifact{Runs: runs}})
if summary.Gate != "block" {
t.Fatalf("gate = %q for runs %#v", summary.Gate, runs)
}
}
}

func TestRunCommandAppliesRecordOnlyExitCodeGate(t *testing.T) {
for _, exitCode := range []int{0, 3} {
t.Run(fmt.Sprintf("exit-%d", exitCode), func(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "skill")
writeSkill(t, target, "# Gate\n")
config := filepath.Join(dir, ".clawscan.yml")
writeFile(t, config, fmt.Sprintf(`version: 1
profiles:
review:
scanners:
- id: demo-scanner
command: |
printf '{"ok":true}\n'
test -d {{target}}
exit %d
gate:
blockOnExitCode: %d
`, exitCode, exitCode))
artifactPath := filepath.Join(dir, "artifact.json")

stdout := captureStdout(t, func() {
if err := run([]string{
target, "--config", config, "--profile", "review",
"--sandbox", "off", "--output", artifactPath,
}, []string{}); err != nil {
t.Fatal(err)
}
})

raw, err := os.ReadFile(artifactPath)
if err != nil {
t.Fatal(err)
}
var artifact runner.Artifact
if err := json.Unmarshal(raw, &artifact); err != nil {
t.Fatal(err)
}
result := artifact.Scanners["demo-scanner"]
if result.Status != "completed" || result.ExitCode == nil || *result.ExitCode != exitCode {
t.Fatalf("scanner result = %#v", result)
}
if artifact.Gate != "block" || len(artifact.GateRules) != 1 {
t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules)
}
wantSummary := fmt.Sprintf("gate: block (demo-scanner exit %d -> block)", exitCode)
if !strings.Contains(stdout, wantSummary) {
t.Fatalf("summary missing %q:\n%s", wantSummary, stdout)
}
})
}
}

func TestRunCommandJSONDoesNotWriteDefaultOutput(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "skill")
Expand Down
1 change: 1 addition & 0 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ scanner_completed: 2
scanner_failed: 0
scanner_skipped: 0
issues_found: 2
gate: pass
errors: 0
full_results: /tmp/clawscan-csv-summarizer.json
```
Expand Down
32 changes: 32 additions & 0 deletions docs/scanners.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ profiles:
targets:
- skill
- plugin
gate:
blockOnExitCode: nonzero
```

String entries select built-in scanners. Object entries define a scanner for
Expand All @@ -47,6 +49,36 @@ that config-backed run and accept these fields:
| `command` | yes | Shell command to execute. Unquoted `{{target}}` is replaced with the safely passed resolved target; do not wrap the placeholder in shell quotes. |
| `env` | no | Required environment variable names. Values stay in the process environment and are never stored in the config or artifact. |
| `targets` | no | Supported target kinds: `skill`, `plugin`, and/or `url`. Defaults to `skill` and `url`. |
| `gate` | no | Exit-code policy with optional `blockOnExitCode` and `warnOnExitCode` rules. |

Each exit-code rule accepts one integer from 0 through 124, a list such as
`[1, 2, 3]`, or the string `nonzero`. The block and warning rules may not
claim the same exit code. Exit codes 125 and above are reserved for shell,
container-runtime, and signal failures, so ClawScan does not treat them as
scanner verdicts. For example:

```yaml
gate:
blockOnExitCode: [2, 3]
warnOnExitCode: 1
```

After every selected scanner finishes, ClawScan records the strongest fired
action as the top-level artifact `gate`: `block` wins over `warn`, and an
artifact with no fired rules records `"gate": "pass"`. Each fired rule is also
listed in `gateRules` with its scanner ID, rule name, exit code, and action.
Gate actions are record-only: `block` does not stop later scanners or the
judge, and it does not change ClawScan's process exit status. For enforcement,
inspect `gate` and `gateRules` on a single run, `runs[].gate` and
`runs[].gateRules` in a batch, or `cases[].run.gate` and
`cases[].run.gateRules` in a benchmark. The human scan summary aggregates the
strongest batch action; the benchmark summary does not aggregate gate actions.

Skipped scanners do not fire gate rules. A scanner result with status `failed`
also does not fire an exit-code rule; a nonzero command that still returned
valid JSON has status `completed` and can fire one. Valid JSON from a timeout,
signal, or reserved infrastructure exit is still preserved, but its omitted
`exitCode` means it cannot fire a gate rule.

The command must write JSON to stdout. ClawScan preserves valid stdout as the
scanner's raw evidence; empty or non-JSON stdout produces a failed scanner
Expand Down
Loading
Loading