diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index 34a66f4..1c534fe 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -341,7 +341,7 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str if len(summary.GateRules) > 0 { details := make([]string, 0, len(summary.GateRules)) for _, rule := range summary.GateRules { - details = append(details, fmt.Sprintf("%s exit %d -> %s", rule.Scanner, rule.ExitCode, rule.Action)) + details = append(details, gateRuleSummary(rule)) } fmt.Fprintf(w, " (%s)", strings.Join(details, ", ")) } @@ -362,6 +362,19 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str } } +func gateRuleSummary(rule runner.FiredGateRule) string { + if rule.ExitCode != nil { + return fmt.Sprintf("%s exit %d -> %s", rule.Scanner, *rule.ExitCode, rule.Action) + } + if rule.FindingCode != "" { + return fmt.Sprintf("%s %s (%s) -> %s", rule.Scanner, rule.FindingCode, rule.FindingSeverity, rule.Action) + } + if rule.Value != "" { + return fmt.Sprintf("%s %s -> %s", rule.Scanner, rule.Value, rule.Action) + } + return fmt.Sprintf("%s %s -> %s", rule.Scanner, rule.Rule, rule.Action) +} + func printBenchmarkSummary(w io.Writer, artifact runner.BenchmarkArtifact, outputPath string) { fmt.Fprintf(w, "benchmark: %s\n", artifact.Benchmark.ID) fmt.Fprintf(w, "split: %s\n", artifact.Benchmark.Split) diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 9902ab9..ee9af75 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -221,7 +221,8 @@ profiles: "profiles:", "clawhub:", "clawhub-aig:", - "- skillspector", + "- id: skillspector", + "native: true", "- aig", } { if !strings.Contains(stdout, want) { @@ -515,10 +516,11 @@ func TestRunCommandWritesDefaultOutputAndPrintsKeyValueSummary(t *testing.T) { } func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) { + exitCode := 3 artifact := runner.Artifact{ Gate: "block", GateRules: []runner.FiredGateRule{ - {Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: 3, Action: "block"}, + {Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: &exitCode, Action: "block"}, }, Scanners: map[string]runner.ScannerResult{}, } @@ -529,6 +531,27 @@ func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) { } } +func TestPrintRunSummaryIncludesNativeGateFinding(t *testing.T) { + artifact := runner.Artifact{ + Gate: "warn", + GateRules: []runner.FiredGateRule{ + { + Scanner: "skillspector", + Rule: "nativeFindingSeverity", + FindingCode: "HIGH-1", + FindingSeverity: "HIGH", + Action: "warn", + }, + }, + Scanners: map[string]runner.ScannerResult{}, + } + var output strings.Builder + printRunSummary(&output, runner.RunTargetsResult{Single: &artifact}, "") + if !strings.Contains(output.String(), "gate: warn (skillspector HIGH-1 (HIGH) -> warn)") { + t.Fatalf("summary missing native gate rule:\n%s", output.String()) + } +} + func TestPrintRunSummaryKeepsBlockAcrossBatchOrder(t *testing.T) { for _, runs := range [][]runner.Artifact{ {{Gate: "warn", Scanners: map[string]runner.ScannerResult{}}, {Gate: "block", Scanners: map[string]runner.ScannerResult{}}}, diff --git a/docs/scanners.md b/docs/scanners.md index eef21b8..71099e1 100644 --- a/docs/scanners.md +++ b/docs/scanners.md @@ -17,7 +17,7 @@ clawscan scanners clawscan scanners skillspector ``` -## User-defined scanners +## Profile scanner configuration A trusted config can mix built-in scanner IDs with user-defined command scanners. The config schema uses the existing `profiles..scanners` list: @@ -28,7 +28,12 @@ version: 1 profiles: review: scanners: - - clawscan-static + - id: skillspector + gate: + native: true + - id: clawscan-static + gate: + native: true - id: my-scanner command: my-scanner --json {{target}} env: @@ -42,8 +47,22 @@ profiles: blockOnExitCode: nonzero ``` -String entries select built-in scanners. Object entries define a scanner for -that config-backed run and accept these fields: +String entries select built-in scanners without gate policy. An object with a +registered built-in `id` and no `command` selects that built-in and can attach +its native gate policy. `native: true` currently supports these fixed policies: + +- SkillSpector recommendation `DO_NOT_INSTALL` or any `CRITICAL` finding + blocks; a `HIGH` finding warns. +- Every `clawscan-static` finding warns, regardless of severity, and never + blocks. + +Native policy is derived from each scanner's raw JSON after the scanner +completes. It does not modify that raw evidence and does not require a judge. +Other built-in scanners reject `native: true` until they provide a native +policy. + +An object with a `command` defines a user-provided scanner for that +config-backed run: | Field | Required | Meaning | | --- | --- | --- | @@ -66,14 +85,20 @@ gate: warnOnExitCode: 1 ``` +Built-in object references can combine `native: true` with exit-code rules. +SkillSpector's gate-eligible process exit code is preserved alongside its raw +JSON, and all fired rules participate in the same strongest-action decision. + After every selected scanner finishes, ClawScan records the strongest fired action as the top-level artifact `gate`: `block` wins over `warn`, and an artifact with no fired rules records `"gate": "pass"`. Each fired rule is also -listed in `gateRules` with its scanner ID, rule name, exit code, and action. -Gate actions are record-only: `block` does not stop later scanners or the -judge, and it does not change ClawScan's process exit status. For enforcement, -inspect `gate` and `gateRules` on a single run, `runs[].gate` and -`runs[].gateRules` in a batch, or `cases[].run.gate` and +listed in `gateRules` with its scanner ID, rule name, action, and match +evidence. Exit-code rules include `exitCode`; native recommendation rules +include `value`; native finding rules include `findingCode`, `findingTitle`, +and `findingSeverity`. Gate actions are record-only: `block` does not stop +later scanners or the judge, and it does not change ClawScan's process exit +status. For enforcement, inspect `gate` and `gateRules` on a single run, +`runs[].gate` and `runs[].gateRules` in a batch, or `cases[].run.gate` and `cases[].run.gateRules` in a benchmark. The human scan summary aggregates the strongest batch action; the benchmark summary does not aggregate gate actions. diff --git a/internal/profiles/clawhub/clawscan.yml b/internal/profiles/clawhub/clawscan.yml index 23f1d07..45415ed 100644 --- a/internal/profiles/clawhub/clawscan.yml +++ b/internal/profiles/clawhub/clawscan.yml @@ -3,8 +3,12 @@ version: 1 profiles: clawhub: scanners: - - skillspector - - clawscan-static + - id: skillspector + gate: + native: true + - id: clawscan-static + gate: + native: true sandbox: env: - OPENAI_API_KEY @@ -29,7 +33,9 @@ profiles: - < {{ prompt:prompt.md }} clawhub-aig: scanners: - - skillspector + - id: skillspector + gate: + native: true - aig sandbox: env: diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 49b6512..1ba21f7 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -50,11 +50,13 @@ type ProfileScanner struct { Targets []string Gate *ProfileScannerGate custom bool + mapping bool } type ProfileScannerGate struct { BlockOnExitCode *profileExitCodeRule `yaml:"blockOnExitCode,omitempty"` WarnOnExitCode *profileExitCodeRule `yaml:"warnOnExitCode,omitempty"` + Native bool `yaml:"native,omitempty"` } type profileExitCodeRule struct { @@ -124,7 +126,7 @@ func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error { } for index := 0; index < len(node.Content); index += 2 { switch node.Content[index].Value { - case "blockOnExitCode", "warnOnExitCode": + case "blockOnExitCode", "warnOnExitCode", "native": value := resolvedYAMLNode(node.Content[index+1]) if value.Tag == "!!null" { return fmt.Errorf("scanner gate %s must not be null", node.Content[index].Value) @@ -134,7 +136,13 @@ func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error { } } type plainGate ProfileScannerGate - return node.Decode((*plainGate)(gate)) + if err := node.Decode((*plainGate)(gate)); err != nil { + return err + } + if gate.BlockOnExitCode == nil && gate.WarnOnExitCode == nil && !gate.Native { + return errors.New("scanner gate must include blockOnExitCode, warnOnExitCode, or native: true") + } + return nil } func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { @@ -148,6 +156,9 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { for index := 0; index < len(node.Content); index += 2 { switch node.Content[index].Value { case "id", "command", "env", "secretEnv", "targets", "gate": + if node.Content[index].Value == "command" { + scanner.custom = true + } if node.Content[index].Value == "gate" { gateNode := resolvedYAMLNode(node.Content[index+1]) if gateNode.Kind != yaml.MappingNode { @@ -175,7 +186,7 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { scanner.SecretEnv = value.SecretEnv scanner.Targets = value.Targets scanner.Gate = value.Gate - scanner.custom = true + scanner.mapping = true return nil default: return fmt.Errorf("scanner entry must be a string or object") @@ -190,12 +201,12 @@ func resolvedYAMLNode(node *yaml.Node) *yaml.Node { } func (scanner ProfileScanner) MarshalYAML() (interface{}, error) { - if !scanner.custom { + if !scanner.mapping && !scanner.custom { return scanner.ID, nil } return struct { ID string `yaml:"id"` - Command string `yaml:"command"` + Command string `yaml:"command,omitempty"` Env []string `yaml:"env,omitempty"` SecretEnv []string `yaml:"secretEnv,omitempty"` Targets []string `yaml:"targets,omitempty"` @@ -243,7 +254,7 @@ func profileGateRules(scanners []ProfileScanner, selectedScannerIDs []string) ma if scanner.Gate == nil || !selected[scanner.ID] { continue } - policy := runner.ScannerGatePolicy{} + policy := runner.ScannerGatePolicy{Native: scanner.Gate.Native} if scanner.Gate.BlockOnExitCode != nil { policy.BlockOnExitCode = &runner.ExitCodeRule{ Codes: append([]int(nil), scanner.Gate.BlockOnExitCode.Codes...), Nonzero: scanner.Gate.BlockOnExitCode.Nonzero, @@ -254,7 +265,7 @@ func profileGateRules(scanners []ProfileScanner, selectedScannerIDs []string) ma Codes: append([]int(nil), scanner.Gate.WarnOnExitCode.Codes...), Nonzero: scanner.Gate.WarnOnExitCode.Nonzero, } } - if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil { + if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil && !policy.Native { continue } rules[scanner.ID] = policy @@ -1087,6 +1098,20 @@ func invalidDeclaredEnvName(env []string) string { func validateProfile(name string, profile Profile) error { seen := map[string]bool{} for _, scanner := range profile.Scanners { + if scanner.mapping && strings.TrimSpace(scanner.ID) == "" { + if scanner.custom { + return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name) + } + return fmt.Errorf("Scanner object in profile %s must include a non-empty id", name) + } + if scanner.mapping && !scanner.custom { + if !runner.DefaultScannerRegistry().Contains(scanner.ID) { + return fmt.Errorf("User-defined scanner %s in profile %s must include a non-empty command", scanner.ID, name) + } + if len(scanner.Env) > 0 || len(scanner.SecretEnv) > 0 || len(scanner.Targets) > 0 { + return fmt.Errorf("Built-in scanner reference %s in profile %s accepts only id and gate", scanner.ID, name) + } + } if scanner.custom && strings.TrimSpace(scanner.ID) == "" { return fmt.Errorf("User-defined scanner in profile %s must include a non-empty id", name) } @@ -1120,6 +1145,12 @@ func validateProfile(name string, profile Profile) error { return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID) } if scanner.Gate != nil { + if scanner.Gate.Native && scanner.custom { + return fmt.Errorf("User-defined scanner %s in profile %s cannot use native gate policy", scanner.ID, name) + } + if scanner.Gate.Native && scanner.ID != "skillspector" && scanner.ID != "clawscan-static" { + return fmt.Errorf("Built-in scanner %s in profile %s does not provide native gate policy", scanner.ID, name) + } if code, overlaps := overlappingExitCodeRules(scanner.Gate.BlockOnExitCode, scanner.Gate.WarnOnExitCode); overlaps { return fmt.Errorf("User-defined scanner %s in profile %s gate blockOnExitCode and warnOnExitCode both claim exit code %d", scanner.ID, name, code) } diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index c9bbac5..086e38d 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -32,6 +32,11 @@ func TestResolveArgsUsesEmbeddedClawHubProfile(t *testing.T) { if got := strings.Join(opts.Scanners, ","); got != "skillspector,clawscan-static" { t.Fatalf("scanners = %q", got) } + for _, scanner := range []string{"skillspector", "clawscan-static"} { + if !opts.GateRules[scanner].Native { + t.Fatalf("%s gate policy = %#v", scanner, opts.GateRules[scanner]) + } + } if opts.Judge == nil { t.Fatal("expected embedded clawhub judge") } @@ -79,6 +84,9 @@ func TestResolveArgsUsesEmbeddedClawHubAIGCandidateProfile(t *testing.T) { if got := strings.Join(candidate.Scanners, ","); got != "skillspector,aig" { t.Fatalf("scanners = %q", got) } + if !candidate.GateRules["skillspector"].Native { + t.Fatalf("skillspector gate policy = %#v", candidate.GateRules["skillspector"]) + } if candidate.Judge == nil || clawhub.Judge == nil { t.Fatal("missing embedded ClawHub judge") } @@ -880,6 +888,80 @@ profiles: } } +func TestResolveArgsAttachesNativeGatePolicyToBuiltInScanner(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: skillspector + gate: + native: true + blockOnExitCode: 1 +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(opts.Scanners, ","); got != "skillspector" { + t.Fatalf("scanners = %q", got) + } + if !opts.GateRules["skillspector"].Native { + t.Fatalf("gate policy = %#v", opts.GateRules["skillspector"]) + } + if got := opts.GateRules["skillspector"].BlockOnExitCode.Codes; !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("exit-code policy = %#v", opts.GateRules["skillspector"]) + } + adapter, ok := opts.ScannerRegistry.Adapter("skillspector") + if !ok || adapter.ID() != "skillspector" { + t.Fatalf("built-in scanner adapter = %#v, present = %v", adapter, ok) + } +} + +func TestResolveArgsRejectsInvalidNativeGatePolicyUse(t *testing.T) { + tests := []struct { + name string + scanner string + want string + }{ + { + name: "command scanner", + scanner: `id: demo + command: demo {{target}} + gate: + native: true`, + want: "User-defined scanner demo in profile review cannot use native gate policy", + }, + { + name: "built-in without native policy", + scanner: `id: cisco + gate: + native: true`, + want: "Built-in scanner cisco in profile review does not provide native gate policy", + }, + { + name: "disabled native-only gate", + scanner: `id: skillspector + gate: + native: false`, + want: "scanner gate must include blockOnExitCode, warnOnExitCode, or native: true", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - "+test.scanner+"\n") + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("err = %v", err) + } + }) + } +} + func TestResolveArgsAcceptsSingleExitCodeGateRule(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") diff --git a/internal/runner/clawhub_parity.go b/internal/runner/clawhub_parity.go index 5156637..bf2d116 100644 --- a/internal/runner/clawhub_parity.go +++ b/internal/runner/clawhub_parity.go @@ -80,6 +80,10 @@ func clawHubSkillSpectorAnalysis(artifact Artifact, checkedAtOverride *int64) (a } func normalizeClawHubSkillSpector(raw json.RawMessage, checkedAt int64) (clawHubNormalizedSkillSpectorAnalysis, error) { + return normalizeSkillSpector(raw, checkedAt, maxClawHubSkillSpectorIssues) +} + +func normalizeSkillSpector(raw json.RawMessage, checkedAt int64, issueLimit int) (clawHubNormalizedSkillSpectorAnalysis, error) { var parsed any if err := json.Unmarshal(raw, &parsed); err != nil { return clawHubNormalizedSkillSpectorAnalysis{}, fmt.Errorf("parse SkillSpector JSON: %w", err) @@ -100,9 +104,13 @@ func normalizeClawHubSkillSpector(raw json.RawMessage, checkedAt int64) (clawHub if value := readMapNumber(record, "issue_count", "issueCount", "finding_count", "findingCount"); value != nil { issueCount = int(*value) } - issues := make([]clawHubSkillSpectorIssue, 0, min(len(issueValues), maxClawHubSkillSpectorIssues)) + issueCapacity := len(issueValues) + if issueLimit > 0 { + issueCapacity = min(issueCapacity, issueLimit) + } + issues := make([]clawHubSkillSpectorIssue, 0, issueCapacity) for index, value := range issueValues { - if len(issues) >= maxClawHubSkillSpectorIssues { + if issueLimit > 0 && len(issues) >= issueLimit { break } if issue := normalizeClawHubSkillSpectorIssue(value, index); issue != nil { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index f1b181a..ab80219 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -66,6 +66,7 @@ func (rule ExitCodeRule) Matches(exitCode int) bool { type ScannerGatePolicy struct { BlockOnExitCode *ExitCodeRule WarnOnExitCode *ExitCodeRule + Native bool } type BenchmarkOptions struct { @@ -134,10 +135,14 @@ type Artifact struct { } type FiredGateRule struct { - Scanner string `json:"scanner"` - Rule string `json:"rule"` - ExitCode int `json:"exitCode"` - Action string `json:"action"` + Scanner string `json:"scanner"` + Rule string `json:"rule"` + ExitCode *int `json:"exitCode,omitempty"` + Value string `json:"value,omitempty"` + FindingCode string `json:"findingCode,omitempty"` + FindingTitle string `json:"findingTitle,omitempty"` + FindingSeverity string `json:"findingSeverity,omitempty"` + Action string `json:"action"` } type RunTargetsResult struct { @@ -2078,6 +2083,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str timeout = 20 * time.Minute } output, runErr := runner.CommandRunner.Run(command, args, cwd, timeout) + exitCode := gateEligibleExitCode(output.ExitCode) raw, readErr := os.ReadFile(resultPath) completedAt := time.Now().UTC().Format(time.RFC3339Nano) if runErr != nil { @@ -2090,6 +2096,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str CompletedAt: completedAt, Command: fullCommand, Error: message + ": SkillSpector scanner returned invalid JSON", + ExitCode: exitCode, Raw: nil, }, nil } @@ -2099,6 +2106,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str CompletedAt: completedAt, Command: fullCommand, Error: message, + ExitCode: exitCode, Raw: json.RawMessage(raw), }, nil } @@ -2108,6 +2116,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str CompletedAt: completedAt, Command: fullCommand, Error: message, + ExitCode: exitCode, Raw: nil, }, nil } @@ -2118,6 +2127,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str CompletedAt: completedAt, Command: fullCommand, Error: "SkillSpector scanner did not write JSON output.", + ExitCode: exitCode, Raw: nil, }, nil } @@ -2128,6 +2138,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str CompletedAt: completedAt, Command: fullCommand, Error: "SkillSpector scanner returned invalid JSON", + ExitCode: exitCode, Raw: nil, }, nil } @@ -2137,6 +2148,7 @@ func (runner ExternalScannerRunner) runSkillSpector(target string, startedAt str CompletedAt: completedAt, Command: fullCommand, Error: "", + ExitCode: exitCode, Raw: json.RawMessage(raw), }, nil } @@ -2261,24 +2273,97 @@ func evaluateGate(artifact *Artifact, opts Options) { } evaluated[scanner] = true result := artifact.Scanners[scanner] - if result.Status != "completed" || result.ExitCode == nil { + if result.Status != "completed" { continue } policy := opts.GateRules[scanner] + if policy.Native { + evaluateNativeGate(artifact, scanner, result) + } + if result.ExitCode == nil { + continue + } if policy.BlockOnExitCode != nil && policy.BlockOnExitCode.Matches(*result.ExitCode) { artifact.GateRules = append(artifact.GateRules, FiredGateRule{ - Scanner: scanner, Rule: "blockOnExitCode", ExitCode: *result.ExitCode, Action: "block", + Scanner: scanner, Rule: "blockOnExitCode", ExitCode: result.ExitCode, Action: "block", }) - artifact.Gate = "block" + setGateAction(artifact, "block") } if policy.WarnOnExitCode != nil && policy.WarnOnExitCode.Matches(*result.ExitCode) { artifact.GateRules = append(artifact.GateRules, FiredGateRule{ - Scanner: scanner, Rule: "warnOnExitCode", ExitCode: *result.ExitCode, Action: "warn", + Scanner: scanner, Rule: "warnOnExitCode", ExitCode: result.ExitCode, Action: "warn", }) - if artifact.Gate == "pass" { - artifact.Gate = "warn" - } + setGateAction(artifact, "warn") + } + } +} + +func evaluateNativeGate(artifact *Artifact, scanner string, result ScannerResult) { + if len(result.Raw) == 0 { + return + } + if scanner == "clawscan-static" { + var report staticScannerReport + if err := json.Unmarshal(result.Raw, &report); err != nil { + return + } + for _, finding := range report.Findings { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, + Rule: "nativeFinding", + FindingCode: finding.ID, + FindingTitle: finding.Title, + FindingSeverity: finding.Severity, + Action: "warn", + }) + setGateAction(artifact, "warn") + } + return + } + if scanner != "skillspector" { + return + } + analysis, err := normalizeSkillSpector(result.Raw, 0, 0) + if err != nil { + return + } + if canonicalGateValue(analysis.Recommendation) == "DO_NOT_INSTALL" { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "nativeRecommendation", Value: analysis.Recommendation, Action: "block", + }) + setGateAction(artifact, "block") + } + for _, issue := range analysis.Issues { + action := "" + switch strings.ToUpper(issue.Severity) { + case "CRITICAL": + action = "block" + case "HIGH": + action = "warn" } + if action == "" { + continue + } + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, + Rule: "nativeFindingSeverity", + FindingCode: issue.IssueID, + FindingTitle: firstNonEmpty(issue.Pattern, issue.Explanation), + FindingSeverity: issue.Severity, + Action: action, + }) + setGateAction(artifact, action) + } +} + +func canonicalGateValue(value string) string { + value = strings.ToUpper(strings.TrimSpace(value)) + return strings.NewReplacer(" ", "_", "-", "_").Replace(value) +} + +func setGateAction(artifact *Artifact, action string) { + if action == "block" || action == "warn" && artifact.Gate == "pass" { + artifact.Gate = action } } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index e9a7c23..cc287d1 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -1557,7 +1557,7 @@ func TestRunBlocksWhenNonzeroExitCodeRuleFires(t *testing.T) { if artifact.Gate != "block" { t.Fatalf("gate = %q", artifact.Gate) } - want := []FiredGateRule{{Scanner: "clawscan-static", Rule: "blockOnExitCode", ExitCode: 2, Action: "block"}} + want := []FiredGateRule{{Scanner: "clawscan-static", Rule: "blockOnExitCode", ExitCode: intPointer(2), Action: "block"}} if !reflect.DeepEqual(artifact.GateRules, want) { t.Fatalf("gate rules = %#v", artifact.GateRules) } @@ -1618,6 +1618,218 @@ func TestRunExitCodeGateActionsAndPrecedence(t *testing.T) { } } +func TestRunAppliesNativeSkillSpectorGatePolicy(t *testing.T) { + manyFindings := make([]map[string]string, maxClawHubSkillSpectorIssues+1) + for index := range manyFindings { + manyFindings[index] = map[string]string{"rule_id": fmt.Sprintf("LOW-%d", index+1), "severity": "LOW"} + } + manyFindings[len(manyFindings)-1] = map[string]string{"rule_id": "LATE-CRITICAL", "severity": "CRITICAL"} + lateCriticalRaw, err := json.Marshal(map[string]any{"filtered_findings": manyFindings}) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + raw json.RawMessage + want string + rule string + value string + findingCode string + severity string + }{ + { + name: "safe passes", + raw: json.RawMessage(`{"risk_assessment":{"recommendation":"SAFE"},"filtered_findings":[]}`), + want: "pass", + }, + { + name: "caution passes without high findings", + raw: json.RawMessage(`{"risk_assessment":{"recommendation":"CAUTION"},"filtered_findings":[{"rule_id":"MED-1","severity":"MEDIUM"}]}`), + want: "pass", + }, + { + name: "do not install blocks", + raw: json.RawMessage(`{"risk_assessment":{"recommendation":"DO_NOT_INSTALL"},"filtered_findings":[]}`), + want: "block", + rule: "nativeRecommendation", + value: "DO_NOT_INSTALL", + }, + { + name: "critical finding blocks", + raw: json.RawMessage(`{"risk_assessment":{"recommendation":"CAUTION"},"filtered_findings":[{"rule_id":"CRIT-1","severity":"CRITICAL"}]}`), + want: "block", + rule: "nativeFindingSeverity", + findingCode: "CRIT-1", + severity: "CRITICAL", + }, + { + name: "high finding warns", + raw: json.RawMessage(`{"risk_assessment":{"recommendation":"CAUTION"},"filtered_findings":[{"rule_id":"HIGH-1","severity":"HIGH"}]}`), + want: "warn", + rule: "nativeFindingSeverity", + findingCode: "HIGH-1", + severity: "HIGH", + }, + { + name: "critical finding after prompt display cap blocks", + raw: lateCriticalRaw, + want: "block", + rule: "nativeFindingSeverity", + findingCode: "LATE-CRITICAL", + severity: "CRITICAL", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + original := append(json.RawMessage(nil), test.raw...) + artifact, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"skillspector"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"skillspector": {Native: true}}, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: map[string]ScannerResult{ + "skillspector": {Status: "completed", Raw: test.raw}, + }}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != test.want { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + if artifact.Judge != nil { + t.Fatalf("native gate unexpectedly invoked judge: %#v", artifact.Judge) + } + if test.rule == "" { + if len(artifact.GateRules) != 0 { + t.Fatalf("gate rules = %#v", artifact.GateRules) + } + } else { + if len(artifact.GateRules) != 1 { + t.Fatalf("gate rules = %#v", artifact.GateRules) + } + rule := artifact.GateRules[0] + if rule.Rule != test.rule || rule.Value != test.value || rule.FindingCode != test.findingCode || rule.FindingSeverity != test.severity { + t.Fatalf("gate rule = %#v", rule) + } + } + if !bytes.Equal(artifact.Scanners["skillspector"].Raw, original) { + t.Fatalf("raw evidence changed\nwant %s\ngot %s", original, artifact.Scanners["skillspector"].Raw) + } + }) + } +} + +func TestRunAppliesWarnOnlyNativeStaticGatePolicy(t *testing.T) { + tests := []struct { + name string + raw json.RawMessage + want string + fired int + code string + }{ + { + name: "clean report passes", + raw: json.RawMessage(`{"schemaVersion":"clawscan-static-v1","findings":[]}`), + want: "pass", + }, + { + name: "medium finding warns", + raw: json.RawMessage(`{"schemaVersion":"clawscan-static-v1","findings":[{"id":"static.prompt_injection","title":"Prompt injection","severity":"medium"}]}`), + want: "warn", + fired: 1, + code: "static.prompt_injection", + }, + { + name: "high finding still only warns", + raw: json.RawMessage(`{"schemaVersion":"clawscan-static-v1","findings":[{"id":"static.destructive_shell","title":"Destructive shell","severity":"high"}]}`), + want: "warn", + fired: 1, + code: "static.destructive_shell", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + artifact, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"clawscan-static": {Native: true}}, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: map[string]ScannerResult{ + "clawscan-static": {Status: "completed", Raw: test.raw}, + }}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != test.want || len(artifact.GateRules) != test.fired { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + for _, rule := range artifact.GateRules { + if rule.Action != "warn" || rule.FindingCode != test.code { + t.Fatalf("static rule action = %q, rule = %#v", rule.Action, rule) + } + } + }) + } +} + +func TestRunNativeAndExitCodeGateRulesComposeOnOneScanner(t *testing.T) { + exitCode := 1 + commandRunner := &recordingCommandRunner{ + writeOutput: `{"filtered_findings":[{"rule_id":"HIGH-1","severity":"HIGH"}]}`, + err: errCommandFailed, + exitCode: &exitCode, + } + artifact, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"skillspector"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"skillspector": { + Native: true, + BlockOnExitCode: &ExitCodeRule{Codes: []int{1}}, + }}, + }, RunContext{ + Env: map[string]string{}, + CommandRunner: commandRunner, + SkillSpectorCommand: []string{"skillspector"}, + }) + if err != nil { + t.Fatal(err) + } + result := artifact.Scanners["skillspector"] + if result.Status != "completed" || result.ExitCode == nil || *result.ExitCode != exitCode { + t.Fatalf("scanner result = %#v", result) + } + if artifact.Gate != "block" || len(artifact.GateRules) != 2 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + if artifact.GateRules[0].Action != "warn" || artifact.GateRules[1].Action != "block" { + t.Fatalf("gate rule order/actions = %#v", artifact.GateRules) + } +} + +func TestRunNativeGatePolicyPreservesRawEvidenceIdentity(t *testing.T) { + raw := json.RawMessage("{\n \"risk_assessment\": {\"recommendation\": \"DO_NOT_INSTALL\"},\n \"filtered_findings\": []\n}\n") + run := func(native bool) Artifact { + t.Helper() + artifact, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"skillspector"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"skillspector": {Native: native}}, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: map[string]ScannerResult{ + "skillspector": {Status: "completed", Raw: append(json.RawMessage(nil), raw...)}, + }}}) + if err != nil { + t.Fatal(err) + } + return artifact + } + + withoutPolicy := run(false) + withPolicy := run(true) + if !bytes.Equal(withoutPolicy.Scanners["skillspector"].Raw, raw) { + t.Fatalf("raw evidence without policy changed: %q", withoutPolicy.Scanners["skillspector"].Raw) + } + if !bytes.Equal(withPolicy.Scanners["skillspector"].Raw, raw) { + t.Fatalf("raw evidence with policy changed: %q", withPolicy.Scanners["skillspector"].Raw) + } + if !bytes.Equal(withPolicy.Scanners["skillspector"].Raw, withoutPolicy.Scanners["skillspector"].Raw) { + t.Fatal("raw evidence differs with native policy enabled") + } +} + func TestRunBlockGateBeatsWarnAcrossScanners(t *testing.T) { target := t.TempDir() artifact, err := Run(Options{