Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion cmd/clawscan/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str
if len(summary.GateRules) > 0 {
details := make([]string, 0, len(summary.GateRules))
for _, rule := range summary.GateRules {
details = append(details, fmt.Sprintf("%s exit %d -> %s", rule.Scanner, rule.ExitCode, rule.Action))
details = append(details, gateRuleSummary(rule))
}
fmt.Fprintf(w, " (%s)", strings.Join(details, ", "))
}
Expand All @@ -362,6 +362,19 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str
}
}

func gateRuleSummary(rule runner.FiredGateRule) string {
if rule.ExitCode != nil {
return fmt.Sprintf("%s exit %d -> %s", rule.Scanner, *rule.ExitCode, rule.Action)
}
if rule.FindingCode != "" {
return fmt.Sprintf("%s %s (%s) -> %s", rule.Scanner, rule.FindingCode, rule.FindingSeverity, rule.Action)
}
if rule.Value != "" {
return fmt.Sprintf("%s %s -> %s", rule.Scanner, rule.Value, rule.Action)
}
return fmt.Sprintf("%s %s -> %s", rule.Scanner, rule.Rule, rule.Action)
}

func printBenchmarkSummary(w io.Writer, artifact runner.BenchmarkArtifact, outputPath string) {
fmt.Fprintf(w, "benchmark: %s\n", artifact.Benchmark.ID)
fmt.Fprintf(w, "split: %s\n", artifact.Benchmark.Split)
Expand Down
27 changes: 25 additions & 2 deletions cmd/clawscan/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,8 @@ profiles:
"profiles:",
"clawhub:",
"clawhub-aig:",
"- skillspector",
"- id: skillspector",
"native: true",
"- aig",
} {
if !strings.Contains(stdout, want) {
Expand Down Expand Up @@ -515,10 +516,11 @@ func TestRunCommandWritesDefaultOutputAndPrintsKeyValueSummary(t *testing.T) {
}

func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) {
exitCode := 3
artifact := runner.Artifact{
Gate: "block",
GateRules: []runner.FiredGateRule{
{Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: 3, Action: "block"},
{Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: &exitCode, Action: "block"},
},
Scanners: map[string]runner.ScannerResult{},
}
Expand All @@ -529,6 +531,27 @@ func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) {
}
}

func TestPrintRunSummaryIncludesNativeGateFinding(t *testing.T) {
artifact := runner.Artifact{
Gate: "warn",
GateRules: []runner.FiredGateRule{
{
Scanner: "skillspector",
Rule: "nativeFindingSeverity",
FindingCode: "HIGH-1",
FindingSeverity: "HIGH",
Action: "warn",
},
},
Scanners: map[string]runner.ScannerResult{},
}
var output strings.Builder
printRunSummary(&output, runner.RunTargetsResult{Single: &artifact}, "")
if !strings.Contains(output.String(), "gate: warn (skillspector HIGH-1 (HIGH) -> warn)") {
t.Fatalf("summary missing native gate rule:\n%s", output.String())
}
}

func TestPrintRunSummaryKeepsBlockAcrossBatchOrder(t *testing.T) {
for _, runs := range [][]runner.Artifact{
{{Gate: "warn", Scanners: map[string]runner.ScannerResult{}}, {Gate: "block", Scanners: map[string]runner.ScannerResult{}}},
Expand Down
43 changes: 34 additions & 9 deletions docs/scanners.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ clawscan scanners
clawscan scanners skillspector
```

## User-defined scanners
## Profile scanner configuration

A trusted config can mix built-in scanner IDs with user-defined command
scanners. The config schema uses the existing `profiles.<name>.scanners` list:
Expand All @@ -28,7 +28,12 @@ version: 1
profiles:
review:
scanners:
- clawscan-static
- id: skillspector
gate:
native: true
- id: clawscan-static
gate:
native: true
- id: my-scanner
command: my-scanner --json {{target}}
env:
Expand All @@ -42,8 +47,22 @@ profiles:
blockOnExitCode: nonzero
```

String entries select built-in scanners. Object entries define a scanner for
that config-backed run and accept these fields:
String entries select built-in scanners without gate policy. An object with a
registered built-in `id` and no `command` selects that built-in and can attach
its native gate policy. `native: true` currently supports these fixed policies:

- SkillSpector recommendation `DO_NOT_INSTALL` or any `CRITICAL` finding
blocks; a `HIGH` finding warns.
- Every `clawscan-static` finding warns, regardless of severity, and never
blocks.

Native policy is derived from each scanner's raw JSON after the scanner
completes. It does not modify that raw evidence and does not require a judge.
Other built-in scanners reject `native: true` until they provide a native
policy.

An object with a `command` defines a user-provided scanner for that
config-backed run:

| Field | Required | Meaning |
| --- | --- | --- |
Expand All @@ -66,14 +85,20 @@ gate:
warnOnExitCode: 1
```

Built-in object references can combine `native: true` with exit-code rules.
SkillSpector's gate-eligible process exit code is preserved alongside its raw
JSON, and all fired rules participate in the same strongest-action decision.

After every selected scanner finishes, ClawScan records the strongest fired
action as the top-level artifact `gate`: `block` wins over `warn`, and an
artifact with no fired rules records `"gate": "pass"`. Each fired rule is also
listed in `gateRules` with its scanner ID, rule name, exit code, and action.
Gate actions are record-only: `block` does not stop later scanners or the
judge, and it does not change ClawScan's process exit status. For enforcement,
inspect `gate` and `gateRules` on a single run, `runs[].gate` and
`runs[].gateRules` in a batch, or `cases[].run.gate` and
listed in `gateRules` with its scanner ID, rule name, action, and match
evidence. Exit-code rules include `exitCode`; native recommendation rules
include `value`; native finding rules include `findingCode`, `findingTitle`,
and `findingSeverity`. Gate actions are record-only: `block` does not stop
later scanners or the judge, and it does not change ClawScan's process exit
status. For enforcement, inspect `gate` and `gateRules` on a single run,
`runs[].gate` and `runs[].gateRules` in a batch, or `cases[].run.gate` and
`cases[].run.gateRules` in a benchmark. The human scan summary aggregates the
strongest batch action; the benchmark summary does not aggregate gate actions.

Expand Down
12 changes: 9 additions & 3 deletions internal/profiles/clawhub/clawscan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@ version: 1
profiles:
clawhub:
scanners:
- skillspector
- clawscan-static
- id: skillspector
gate:
native: true
- id: clawscan-static
gate:
native: true
sandbox:
env:
- OPENAI_API_KEY
Expand All @@ -29,7 +33,9 @@ profiles:
- < {{ prompt:prompt.md }}
clawhub-aig:
scanners:
- skillspector
- id: skillspector
gate:
native: true
- aig
sandbox:
env:
Expand Down
45 changes: 38 additions & 7 deletions internal/profiles/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,13 @@ type ProfileScanner struct {
Targets []string
Gate *ProfileScannerGate
custom bool
mapping bool
}

type ProfileScannerGate struct {
BlockOnExitCode *profileExitCodeRule `yaml:"blockOnExitCode,omitempty"`
WarnOnExitCode *profileExitCodeRule `yaml:"warnOnExitCode,omitempty"`
Native bool `yaml:"native,omitempty"`
}

type profileExitCodeRule struct {
Expand Down Expand Up @@ -124,7 +126,7 @@ func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error {
}
for index := 0; index < len(node.Content); index += 2 {
switch node.Content[index].Value {
case "blockOnExitCode", "warnOnExitCode":
case "blockOnExitCode", "warnOnExitCode", "native":
value := resolvedYAMLNode(node.Content[index+1])
if value.Tag == "!!null" {
return fmt.Errorf("scanner gate %s must not be null", node.Content[index].Value)
Expand All @@ -134,7 +136,13 @@ func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error {
}
}
type plainGate ProfileScannerGate
return node.Decode((*plainGate)(gate))
if err := node.Decode((*plainGate)(gate)); err != nil {
return err
}
if gate.BlockOnExitCode == nil && gate.WarnOnExitCode == nil && !gate.Native {
return errors.New("scanner gate must include blockOnExitCode, warnOnExitCode, or native: true")
}
return nil
}

func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error {
Expand All @@ -148,6 +156,9 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error {
for index := 0; index < len(node.Content); index += 2 {
switch node.Content[index].Value {
case "id", "command", "env", "secretEnv", "targets", "gate":
if node.Content[index].Value == "command" {
scanner.custom = true
}
if node.Content[index].Value == "gate" {
gateNode := resolvedYAMLNode(node.Content[index+1])
if gateNode.Kind != yaml.MappingNode {
Expand Down Expand Up @@ -175,7 +186,7 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error {
scanner.SecretEnv = value.SecretEnv
scanner.Targets = value.Targets
scanner.Gate = value.Gate
scanner.custom = true
scanner.mapping = true
return nil
default:
return fmt.Errorf("scanner entry must be a string or object")
Expand All @@ -190,12 +201,12 @@ func resolvedYAMLNode(node *yaml.Node) *yaml.Node {
}

func (scanner ProfileScanner) MarshalYAML() (interface{}, error) {
if !scanner.custom {
if !scanner.mapping && !scanner.custom {
return scanner.ID, nil
}
return struct {
ID string `yaml:"id"`
Command string `yaml:"command"`
Command string `yaml:"command,omitempty"`
Env []string `yaml:"env,omitempty"`
SecretEnv []string `yaml:"secretEnv,omitempty"`
Targets []string `yaml:"targets,omitempty"`
Expand Down Expand Up @@ -243,7 +254,7 @@ func profileGateRules(scanners []ProfileScanner, selectedScannerIDs []string) ma
if scanner.Gate == nil || !selected[scanner.ID] {
continue
}
policy := runner.ScannerGatePolicy{}
policy := runner.ScannerGatePolicy{Native: scanner.Gate.Native}
if scanner.Gate.BlockOnExitCode != nil {
policy.BlockOnExitCode = &runner.ExitCodeRule{
Codes: append([]int(nil), scanner.Gate.BlockOnExitCode.Codes...), Nonzero: scanner.Gate.BlockOnExitCode.Nonzero,
Expand All @@ -254,7 +265,7 @@ func profileGateRules(scanners []ProfileScanner, selectedScannerIDs []string) ma
Codes: append([]int(nil), scanner.Gate.WarnOnExitCode.Codes...), Nonzero: scanner.Gate.WarnOnExitCode.Nonzero,
}
}
if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil {
if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil && !policy.Native {
continue
}
rules[scanner.ID] = policy
Expand Down Expand Up @@ -1087,6 +1098,20 @@ func invalidDeclaredEnvName(env []string) string {
func validateProfile(name string, profile Profile) error {
seen := map[string]bool{}
for _, scanner := range profile.Scanners {
if scanner.mapping && strings.TrimSpace(scanner.ID) == "" {
if scanner.custom {
return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name)
}
return fmt.Errorf("Scanner object in profile %s must include a non-empty id", name)
}
if scanner.mapping && !scanner.custom {
if !runner.DefaultScannerRegistry().Contains(scanner.ID) {
return fmt.Errorf("User-defined scanner %s in profile %s must include a non-empty command", scanner.ID, name)
}
if len(scanner.Env) > 0 || len(scanner.SecretEnv) > 0 || len(scanner.Targets) > 0 {
return fmt.Errorf("Built-in scanner reference %s in profile %s accepts only id and gate", scanner.ID, name)
}
}
if scanner.custom && strings.TrimSpace(scanner.ID) == "" {
return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name)
}
Expand Down Expand Up @@ -1120,6 +1145,12 @@ func validateProfile(name string, profile Profile) error {
return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID)
}
if scanner.Gate != nil {
if scanner.Gate.Native && scanner.custom {
return fmt.Errorf("User-defined scanner %s in profile %s cannot use native gate policy", scanner.ID, name)
}
if scanner.Gate.Native && scanner.ID != "skillspector" && scanner.ID != "clawscan-static" {
return fmt.Errorf("Built-in scanner %s in profile %s does not provide native gate policy", scanner.ID, name)
}
if code, overlaps := overlappingExitCodeRules(scanner.Gate.BlockOnExitCode, scanner.Gate.WarnOnExitCode); overlaps {
return fmt.Errorf("User-defined scanner %s in profile %s gate blockOnExitCode and warnOnExitCode both claim exit code %d", scanner.ID, name, code)
}
Expand Down
Loading
Loading