diff --git a/README.md b/README.md index 8cc3ea6..e440faf 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ scanner_completed: 2 scanner_failed: 0 scanner_skipped: 0 issues_found: 2 +gate: pass errors: 0 full_results: /tmp/clawscan-csv-summarizer.json ``` diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index 97ae10b..cefb420 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -337,6 +337,15 @@ func printRunSummary(w io.Writer, result runner.RunTargetsResult, outputPath str fmt.Fprintf(w, "scanner_other: %d\n", summary.ScannerOther) } fmt.Fprintf(w, "issues_found: %d\n", summary.IssuesFound) + fmt.Fprintf(w, "gate: %s", summary.Gate) + if len(summary.GateRules) > 0 { + details := make([]string, 0, len(summary.GateRules)) + for _, rule := range summary.GateRules { + details = append(details, fmt.Sprintf("%s exit %d -> %s", rule.Scanner, rule.ExitCode, rule.Action)) + } + fmt.Fprintf(w, " (%s)", strings.Join(details, ", ")) + } + fmt.Fprintln(w) if summary.HasJudge { fmt.Fprintf(w, "judge_completed: %d\n", summary.JudgeCompleted) fmt.Fprintf(w, "judge_failed: %d\n", summary.JudgeFailed) @@ -404,6 +413,8 @@ type runSummary struct { ScannerSkipped int ScannerOther int IssuesFound int + Gate string + GateRules []runner.FiredGateRule HasJudge bool JudgeCompleted int JudgeFailed int @@ -415,7 +426,7 @@ type runSummary struct { } func summarizeRunTargets(result runner.RunTargetsResult) runSummary { - var summary runSummary + summary := runSummary{Gate: "pass"} if result.Batch != nil { summary.Profile = result.Batch.Profile summary.Profiles = result.Batch.Summary.ProfileCount @@ -436,6 +447,10 @@ func (summary *runSummary) addArtifact(artifact runner.Artifact) { summary.Profile = artifact.Profile } summary.Targets++ + summary.GateRules = append(summary.GateRules, artifact.GateRules...) + if artifact.Gate == "block" || (artifact.Gate == "warn" && summary.Gate == "pass") { + summary.Gate = artifact.Gate + } for _, result := range artifact.Scanners { switch result.Status { case "completed": diff --git a/cmd/clawscan/main_test.go b/cmd/clawscan/main_test.go index 37d2e2f..9902ab9 100644 --- a/cmd/clawscan/main_test.go +++ b/cmd/clawscan/main_test.go @@ -2,11 +2,14 @@ package main import ( "encoding/json" + "fmt" "io" "os" "path/filepath" "strings" "testing" + + "github.com/openclaw/clawscan/internal/runner" ) func TestRunCommandPrintsHelp(t *testing.T) { @@ -511,6 +514,86 @@ func TestRunCommandWritesDefaultOutputAndPrintsKeyValueSummary(t *testing.T) { } } +func TestPrintRunSummaryIncludesGateVerdictAndFiredRule(t *testing.T) { + artifact := runner.Artifact{ + Gate: "block", + GateRules: []runner.FiredGateRule{ + {Scanner: "my-scanner", Rule: "blockOnExitCode", ExitCode: 3, Action: "block"}, + }, + Scanners: map[string]runner.ScannerResult{}, + } + var output strings.Builder + printRunSummary(&output, runner.RunTargetsResult{Single: &artifact}, "") + if !strings.Contains(output.String(), "gate: block (my-scanner exit 3 -> block)") { + t.Fatalf("summary missing gate rule:\n%s", output.String()) + } +} + +func TestPrintRunSummaryKeepsBlockAcrossBatchOrder(t *testing.T) { + for _, runs := range [][]runner.Artifact{ + {{Gate: "warn", Scanners: map[string]runner.ScannerResult{}}, {Gate: "block", Scanners: map[string]runner.ScannerResult{}}}, + {{Gate: "block", Scanners: map[string]runner.ScannerResult{}}, {Gate: "warn", Scanners: map[string]runner.ScannerResult{}}}, + } { + summary := summarizeRunTargets(runner.RunTargetsResult{Batch: &runner.BatchArtifact{Runs: runs}}) + if summary.Gate != "block" { + t.Fatalf("gate = %q for runs %#v", summary.Gate, runs) + } + } +} + +func TestRunCommandAppliesRecordOnlyExitCodeGate(t *testing.T) { + for _, exitCode := range []int{0, 3} { + t.Run(fmt.Sprintf("exit-%d", exitCode), func(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeSkill(t, target, "# Gate\n") + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, fmt.Sprintf(`version: 1 +profiles: + review: + scanners: + - id: demo-scanner + command: | + printf '{"ok":true}\n' + test -d {{target}} + exit %d + gate: + blockOnExitCode: %d +`, exitCode, exitCode)) + artifactPath := filepath.Join(dir, "artifact.json") + + stdout := captureStdout(t, func() { + if err := run([]string{ + target, "--config", config, "--profile", "review", + "--sandbox", "off", "--output", artifactPath, + }, []string{}); err != nil { + t.Fatal(err) + } + }) + + raw, err := os.ReadFile(artifactPath) + if err != nil { + t.Fatal(err) + } + var artifact runner.Artifact + if err := json.Unmarshal(raw, &artifact); err != nil { + t.Fatal(err) + } + result := artifact.Scanners["demo-scanner"] + if result.Status != "completed" || result.ExitCode == nil || *result.ExitCode != exitCode { + t.Fatalf("scanner result = %#v", result) + } + if artifact.Gate != "block" || len(artifact.GateRules) != 1 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + wantSummary := fmt.Sprintf("gate: block (demo-scanner exit %d -> block)", exitCode) + if !strings.Contains(stdout, wantSummary) { + t.Fatalf("summary missing %q:\n%s", wantSummary, stdout) + } + }) + } +} + func TestRunCommandJSONDoesNotWriteDefaultOutput(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") diff --git a/docs/index.md b/docs/index.md index 5eacfcf..2a4cf19 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,6 +43,7 @@ scanner_completed: 2 scanner_failed: 0 scanner_skipped: 0 issues_found: 2 +gate: pass errors: 0 full_results: /tmp/clawscan-csv-summarizer.json ``` diff --git a/docs/scanners.md b/docs/scanners.md index cebe5d8..e8da77e 100644 --- a/docs/scanners.md +++ b/docs/scanners.md @@ -36,6 +36,8 @@ profiles: targets: - skill - plugin + gate: + blockOnExitCode: nonzero ``` String entries select built-in scanners. Object entries define a scanner for @@ -47,6 +49,36 @@ that config-backed run and accept these fields: | `command` | yes | Shell command to execute. Unquoted `{{target}}` is replaced with the safely passed resolved target; do not wrap the placeholder in shell quotes. | | `env` | no | Required environment variable names. Values stay in the process environment and are never stored in the config or artifact. | | `targets` | no | Supported target kinds: `skill`, `plugin`, and/or `url`. Defaults to `skill` and `url`. | +| `gate` | no | Exit-code policy with optional `blockOnExitCode` and `warnOnExitCode` rules. | + +Each exit-code rule accepts one integer from 0 through 124, a list such as +`[1, 2, 3]`, or the string `nonzero`. The block and warning rules may not +claim the same exit code. Exit codes 125 and above are reserved for shell, +container-runtime, and signal failures, so ClawScan does not treat them as +scanner verdicts. For example: + +```yaml +gate: + blockOnExitCode: [2, 3] + warnOnExitCode: 1 +``` + +After every selected scanner finishes, ClawScan records the strongest fired +action as the top-level artifact `gate`: `block` wins over `warn`, and an +artifact with no fired rules records `"gate": "pass"`. Each fired rule is also +listed in `gateRules` with its scanner ID, rule name, exit code, and action. +Gate actions are record-only: `block` does not stop later scanners or the +judge, and it does not change ClawScan's process exit status. For enforcement, +inspect `gate` and `gateRules` on a single run, `runs[].gate` and +`runs[].gateRules` in a batch, or `cases[].run.gate` and +`cases[].run.gateRules` in a benchmark. The human scan summary aggregates the +strongest batch action; the benchmark summary does not aggregate gate actions. + +Skipped scanners do not fire gate rules. A scanner result with status `failed` +also does not fire an exit-code rule; a nonzero command that still returned +valid JSON has status `completed` and can fire one. Valid JSON from a timeout, +signal, or reserved infrastructure exit is still preserved, but its omitted +`exitCode` means it cannot fire a gate rule. The command must write JSON to stdout. ClawScan preserves valid stdout as the scanner's raw evidence; empty or non-JSON stdout produces a failed scanner diff --git a/internal/profiles/resolver.go b/internal/profiles/resolver.go index 0c11add..0ae5561 100644 --- a/internal/profiles/resolver.go +++ b/internal/profiles/resolver.go @@ -47,9 +47,95 @@ type ProfileScanner struct { Command string Env []string Targets []string + Gate *ProfileScannerGate custom bool } +type ProfileScannerGate struct { + BlockOnExitCode *profileExitCodeRule `yaml:"blockOnExitCode,omitempty"` + WarnOnExitCode *profileExitCodeRule `yaml:"warnOnExitCode,omitempty"` +} + +type profileExitCodeRule struct { + Codes []int + Nonzero bool +} + +func (rule *profileExitCodeRule) UnmarshalYAML(node *yaml.Node) error { + node = resolvedYAMLNode(node) + switch node.Kind { + case yaml.ScalarNode: + if node.Tag == "!!str" && node.Value == "nonzero" { + rule.Nonzero = true + return nil + } + if node.Tag == "!!int" { + var code int + if err := node.Decode(&code); err == nil && code >= 0 && code <= runner.MaxGateExitCode { + rule.Codes = []int{code} + return nil + } + return fmt.Errorf("exit-code gate rule must contain only integers from 0 through %d", runner.MaxGateExitCode) + } + case yaml.SequenceNode: + if len(node.Content) == 0 { + return errors.New("exit-code gate rule must not be an empty list") + } + codes := make([]int, 0, len(node.Content)) + for _, item := range node.Content { + item = resolvedYAMLNode(item) + if item.Kind != yaml.ScalarNode || item.Tag != "!!int" { + return fmt.Errorf("exit-code gate rule must contain only integers from 0 through %d", runner.MaxGateExitCode) + } + var code int + if err := item.Decode(&code); err != nil || code < 0 || code > runner.MaxGateExitCode { + return fmt.Errorf("exit-code gate rule must contain only integers from 0 through %d", runner.MaxGateExitCode) + } + codes = append(codes, code) + } + rule.Codes = codes + return nil + } + return fmt.Errorf(`exit-code gate rule must be an integer from 0 through %d, a list of those integers, or "nonzero"`, runner.MaxGateExitCode) +} + +func (rule profileExitCodeRule) MarshalYAML() (interface{}, error) { + if rule.Nonzero { + return "nonzero", nil + } + switch len(rule.Codes) { + case 0: + return nil, errors.New("exit-code gate rule must include at least one exit code") + case 1: + return rule.Codes[0], nil + default: + return append([]int(nil), rule.Codes...), nil + } +} + +func (gate *ProfileScannerGate) UnmarshalYAML(node *yaml.Node) error { + node = resolvedYAMLNode(node) + if node.Kind != yaml.MappingNode { + return errors.New("scanner gate must be an object") + } + if len(node.Content) == 0 { + return errors.New("scanner gate must include blockOnExitCode or warnOnExitCode") + } + for index := 0; index < len(node.Content); index += 2 { + switch node.Content[index].Value { + case "blockOnExitCode", "warnOnExitCode": + value := resolvedYAMLNode(node.Content[index+1]) + if value.Tag == "!!null" { + return fmt.Errorf("scanner gate %s must not be null", node.Content[index].Value) + } + default: + return fmt.Errorf("field %s not found in type profiles.ProfileScannerGate", node.Content[index].Value) + } + } + type plainGate ProfileScannerGate + return node.Decode((*plainGate)(gate)) +} + func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { switch node.Kind { case yaml.ScalarNode: @@ -60,16 +146,23 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { case yaml.MappingNode: for index := 0; index < len(node.Content); index += 2 { switch node.Content[index].Value { - case "id", "command", "env", "targets": + case "id", "command", "env", "targets", "gate": + if node.Content[index].Value == "gate" { + gateNode := resolvedYAMLNode(node.Content[index+1]) + if gateNode.Kind != yaml.MappingNode { + return errors.New("scanner gate must be an object") + } + } default: return fmt.Errorf("field %s not found in type profiles.ProfileScanner", node.Content[index].Value) } } var value struct { - ID string `yaml:"id"` - Command string `yaml:"command"` - Env []string `yaml:"env,omitempty"` - Targets []string `yaml:"targets,omitempty"` + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + Gate *ProfileScannerGate `yaml:"gate,omitempty"` } if err := node.Decode(&value); err != nil { return err @@ -78,6 +171,7 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { scanner.Command = value.Command scanner.Env = value.Env scanner.Targets = value.Targets + scanner.Gate = value.Gate scanner.custom = true return nil default: @@ -85,16 +179,24 @@ func (scanner *ProfileScanner) UnmarshalYAML(node *yaml.Node) error { } } +func resolvedYAMLNode(node *yaml.Node) *yaml.Node { + for node != nil && node.Kind == yaml.AliasNode { + node = node.Alias + } + return node +} + func (scanner ProfileScanner) MarshalYAML() (interface{}, error) { if !scanner.custom { return scanner.ID, nil } return struct { - ID string `yaml:"id"` - Command string `yaml:"command"` - Env []string `yaml:"env,omitempty"` - Targets []string `yaml:"targets,omitempty"` - }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets}, nil + ID string `yaml:"id"` + Command string `yaml:"command"` + Env []string `yaml:"env,omitempty"` + Targets []string `yaml:"targets,omitempty"` + Gate *ProfileScannerGate `yaml:"gate,omitempty"` + }{scanner.ID, scanner.Command, scanner.Env, scanner.Targets, scanner.Gate}, nil } func profileScannerIDs(scanners []ProfileScanner) []string { @@ -127,6 +229,35 @@ func profileScannerRegistry(scanners []ProfileScanner) (runner.ScannerRegistry, return registry, nil } +func profileGateRules(scanners []ProfileScanner, selectedScannerIDs []string) map[string]runner.ScannerGatePolicy { + rules := map[string]runner.ScannerGatePolicy{} + selected := make(map[string]bool, len(selectedScannerIDs)) + for _, scannerID := range selectedScannerIDs { + selected[scannerID] = true + } + for _, scanner := range scanners { + if scanner.Gate == nil || !selected[scanner.ID] { + continue + } + policy := runner.ScannerGatePolicy{} + if scanner.Gate.BlockOnExitCode != nil { + policy.BlockOnExitCode = &runner.ExitCodeRule{ + Codes: append([]int(nil), scanner.Gate.BlockOnExitCode.Codes...), Nonzero: scanner.Gate.BlockOnExitCode.Nonzero, + } + } + if scanner.Gate.WarnOnExitCode != nil { + policy.WarnOnExitCode = &runner.ExitCodeRule{ + Codes: append([]int(nil), scanner.Gate.WarnOnExitCode.Codes...), Nonzero: scanner.Gate.WarnOnExitCode.Nonzero, + } + } + if policy.BlockOnExitCode == nil && policy.WarnOnExitCode == nil { + continue + } + rules[scanner.ID] = policy + } + return rules +} + type Sandbox struct { Mode string `yaml:"mode,omitempty"` Image string `yaml:"image,omitempty"` @@ -282,6 +413,7 @@ func resolveRunSetIntent(intent cliIntent, cwd string) (ResolvedRunSet, error) { return ResolvedRunSet{}, err } opts.Profile = profileName + opts.GateRules = profileGateRules(selected.profile.Scanners, opts.Scanners) opts.ConfigSource = configSource opts.DiscoverConfig = intent.discoverConfig if opts.Judge != nil { @@ -383,6 +515,7 @@ func resolveAllConfigProfiles(intent cliIntent, cwd string) (ResolvedRunSet, err return ResolvedRunSet{}, err } opts.Profile = profileName + opts.GateRules = profileGateRules(selected.profile.Scanners, opts.Scanners) opts.ConfigSource = filepath.Clean(projectPath) opts.OutputPath = "" opts.JSON = false @@ -830,6 +963,11 @@ func validateProfile(name string, profile Profile) error { if scanner.custom && runner.DefaultScannerRegistry().Contains(scanner.ID) { return fmt.Errorf("User-defined scanner %s collides with a built-in scanner ID", scanner.ID) } + if scanner.Gate != nil { + if code, overlaps := overlappingExitCodeRules(scanner.Gate.BlockOnExitCode, scanner.Gate.WarnOnExitCode); overlaps { + return fmt.Errorf("User-defined scanner %s in profile %s gate blockOnExitCode and warnOnExitCode both claim exit code %d", scanner.ID, name, code) + } + } for _, target := range scanner.Targets { switch target { case "skill", "plugin", "url": @@ -845,6 +983,41 @@ func validateProfile(name string, profile Profile) error { return nil } +func overlappingExitCodeRules(block *profileExitCodeRule, warn *profileExitCodeRule) (int, bool) { + if block == nil || warn == nil { + return 0, false + } + if block.Nonzero && warn.Nonzero { + return 1, true + } + if block.Nonzero { + for _, code := range warn.Codes { + if code != 0 { + return code, true + } + } + return 0, false + } + if warn.Nonzero { + for _, code := range block.Codes { + if code != 0 { + return code, true + } + } + return 0, false + } + warnCodes := make(map[int]bool, len(warn.Codes)) + for _, code := range warn.Codes { + warnCodes[code] = true + } + for _, code := range block.Codes { + if warnCodes[code] { + return code, true + } + } + return 0, false +} + func scannerTargetPlaceholdersAreUnquoted(command string) bool { matches := scannerTargetPlaceholderPattern.FindAllStringIndex(command, -1) matchIndex := 0 diff --git a/internal/profiles/resolver_test.go b/internal/profiles/resolver_test.go index c7b8a8a..accb291 100644 --- a/internal/profiles/resolver_test.go +++ b/internal/profiles/resolver_test.go @@ -6,11 +6,13 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" "testing" "time" "github.com/openclaw/clawscan/internal/runner" + "gopkg.in/yaml.v3" ) func TestResolveArgsUsesEmbeddedClawHubProfile(t *testing.T) { @@ -783,6 +785,229 @@ profiles: } } +func TestResolveArgsParsesUserDefinedScannerExitCodeGateRules(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: nonzero + - id: warner + command: warner {{target}} + gate: + warnOnExitCode: [1, 2, 3] +`) + + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + block := opts.GateRules["blocker"].BlockOnExitCode + if block == nil || !block.Nonzero { + t.Fatalf("block rule = %#v", block) + } + warn := opts.GateRules["warner"].WarnOnExitCode + if warn == nil || !reflect.DeepEqual(warn.Codes, []int{1, 2, 3}) { + t.Fatalf("warn rule = %#v", warn) + } +} + +func TestResolveArgsAcceptsSingleExitCodeGateRule(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: 7 +`) + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := opts.GateRules["blocker"].BlockOnExitCode.Codes; !reflect.DeepEqual(got, []int{7}) { + t.Fatalf("codes = %#v", got) + } +} + +func TestProfileScannerExitCodeGateRulesRoundTripYAML(t *testing.T) { + scanner := ProfileScanner{ + ID: "demo", Command: "demo {{target}}", custom: true, + Gate: &ProfileScannerGate{ + BlockOnExitCode: &profileExitCodeRule{Nonzero: true}, + WarnOnExitCode: &profileExitCodeRule{Codes: []int{0}}, + }, + } + encoded, err := yaml.Marshal(scanner) + if err != nil { + t.Fatal(err) + } + var decoded ProfileScanner + if err := yaml.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("round trip failed for %s: %v", encoded, err) + } + if decoded.Gate == nil || !decoded.Gate.BlockOnExitCode.Nonzero || !reflect.DeepEqual(decoded.Gate.WarnOnExitCode.Codes, []int{0}) { + t.Fatalf("decoded scanner = %#v from %s", decoded, encoded) + } +} + +func TestResolveArgsRejectsInvalidExitCodeGateRules(t *testing.T) { + tests := []struct { + name string + rules string + want string + }{ + {name: "negative", rules: "blockOnExitCode: -1", want: "must contain only integers from 0 through 124"}, + {name: "reserved scalar", rules: "blockOnExitCode: 125", want: "must contain only integers from 0 through 124"}, + {name: "reserved list member", rules: "blockOnExitCode: [1, 125]", want: "must contain only integers from 0 through 124"}, + {name: "non integer", rules: "blockOnExitCode: nope", want: `must be an integer from 0 through 124, a list of those integers, or "nonzero"`}, + {name: "empty list", rules: "blockOnExitCode: []", want: "must not be an empty list"}, + {name: "null rule", rules: "blockOnExitCode: null", want: "scanner gate blockOnExitCode must not be null"}, + {name: "empty gate", rules: "{}", want: "scanner gate must include blockOnExitCode or warnOnExitCode"}, + {name: "null gate", rules: "null", want: "scanner gate must be an object"}, + {name: "block nonzero overlap", rules: "blockOnExitCode: nonzero\n warnOnExitCode: [0, 2]", want: "blockOnExitCode and warnOnExitCode both claim exit code 2"}, + {name: "warn nonzero overlap", rules: "blockOnExitCode: [0, 2]\n warnOnExitCode: nonzero", want: "blockOnExitCode and warnOnExitCode both claim exit code 2"}, + {name: "both nonzero overlap", rules: "blockOnExitCode: nonzero\n warnOnExitCode: nonzero", want: "blockOnExitCode and warnOnExitCode both claim exit code 1"}, + {name: "list overlap", rules: "blockOnExitCode: [1, 2]\n warnOnExitCode: [2, 3]", want: "blockOnExitCode and warnOnExitCode both claim exit code 2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, "version: 1\nprofiles:\n review:\n scanners:\n - id: demo\n command: demo {{target}}\n gate:\n "+test.rules+"\n") + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("err = %v", err) + } + }) + } +} + +func TestResolveArgsAcceptsAliasedExitCodeGateRule(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: blocker + command: blocker {{target}} + gate: + blockOnExitCode: &shared-code 7 + - id: warner + command: warner {{target}} + gate: + warnOnExitCode: *shared-code +`) + opts, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err != nil { + t.Fatal(err) + } + if got := opts.GateRules["warner"].WarnOnExitCode.Codes; !reflect.DeepEqual(got, []int{7}) { + t.Fatalf("aliased codes = %#v", got) + } +} + +func TestResolveArgsRejectsGateAliasToNull(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: demo + command: demo {{target}} + env: &empty null + gate: *empty +`) + _, err := ResolveArgs([]string{"./skill", "--config", config, "--profile", "review"}, dir) + if err == nil || !strings.Contains(err.Error(), "scanner gate must be an object") { + t.Fatalf("err = %v", err) + } +} + +func TestGateRuleForProfileScannerExcludedByCLIOverrideIsDropped(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "skill") + writeFile(t, filepath.Join(target, "SKILL.md"), "# Demo\n") + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + review: + scanners: + - id: absent-scanner + command: absent {{target}} + gate: + blockOnExitCode: nonzero +`) + opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review", "--scanner", "clawscan-static", "--sandbox", "off"}, dir) + if err != nil { + t.Fatal(err) + } + if len(opts.GateRules) != 0 { + t.Fatalf("gate rules = %#v", opts.GateRules) + } + commandRunner := &profileCommandRunner{stdout: `{}`} + artifact, err := runner.Run(opts, runner.RunContext{Env: map[string]string{}, CommandRunner: commandRunner}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "pass" || len(artifact.GateRules) != 0 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + + benchmark, err := ResolveBenchmarkRunSet("clawhub-security-signals", []string{ + "--config", config, "--profile", "review", "--scanner", "clawscan-static", "--sandbox", "off", + }, dir) + if err != nil { + t.Fatal(err) + } + if got := benchmark.Options[0].GateRules; len(got) != 0 { + t.Fatalf("benchmark gate rules = %#v", got) + } +} + +func TestAllProfilesCLIOverrideDropsExcludedGateRules(t *testing.T) { + dir := t.TempDir() + config := filepath.Join(dir, ".clawscan.yml") + writeFile(t, config, `version: 1 +profiles: + alpha: + scanners: + - id: alpha-scanner + command: alpha {{target}} + gate: + blockOnExitCode: nonzero + beta: + scanners: + - id: beta-scanner + command: beta {{target}} + gate: + warnOnExitCode: 2 +`) + resolved, err := ResolveRunSet([]string{ + "--config", config, "--scanner", "clawscan-static", "--sandbox", "off", + }, dir) + if err != nil { + t.Fatal(err) + } + if !resolved.AllProfiles || len(resolved.Options) != 2 { + t.Fatalf("resolved = %#v", resolved) + } + for _, opts := range resolved.Options { + if len(opts.GateRules) != 0 { + t.Fatalf("%s gate rules = %#v", opts.Profile, opts.GateRules) + } + } +} + func TestResolveArgsRejectsUserDefinedScannerIDCollision(t *testing.T) { dir := t.TempDir() config := filepath.Join(dir, ".clawscan.yml") @@ -1042,15 +1267,34 @@ profiles: scanners: - id: fixture-scanner command: alpha-scan {{target}} + gate: + blockOnExitCode: 3 beta: scanners: - id: fixture-scanner command: beta-scan {{target}} + gate: + warnOnExitCode: nonzero `) resolved, err := ResolveRunSet([]string{target, "--config", config}, dir) if err != nil { t.Fatal(err) } + for _, opts := range resolved.Options { + policy := opts.GateRules["fixture-scanner"] + switch opts.Profile { + case "alpha": + if policy.BlockOnExitCode == nil || !reflect.DeepEqual(policy.BlockOnExitCode.Codes, []int{3}) { + t.Fatalf("alpha gate policy = %#v", policy) + } + case "beta": + if policy.WarnOnExitCode == nil || !policy.WarnOnExitCode.Nonzero { + t.Fatalf("beta gate policy = %#v", policy) + } + default: + t.Fatalf("unexpected profile %q", opts.Profile) + } + } commandRunner := &profileCommandRunner{stdout: `{}`} batch, err := runner.RunProfileBatch(resolved.Options, runner.RunContext{ Env: map[string]string{}, CommandRunner: commandRunner, @@ -1183,6 +1427,8 @@ profiles: - PLUGIN_SCANNER_TOKEN targets: - plugin + gate: + blockOnExitCode: nonzero `) opts, err := ResolveArgs([]string{target, "--config", config, "--profile", "review"}, dir) if err != nil { @@ -1205,6 +1451,9 @@ profiles: if result.Status != "skipped" || !strings.Contains(result.Error, "does not support skill targets") { t.Fatalf("result = %#v", result) } + if artifact.Gate != "pass" || len(artifact.GateRules) != 0 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } if commandRunner.command != "" { t.Fatalf("unsupported scanner executed: %q %#v", commandRunner.command, commandRunner.args) } diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index 2f6dcc6..17b9011 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -202,6 +202,9 @@ func RunBenchmark(opts Options, ctx RunContext) (BenchmarkArtifact, error) { benchmarkOpts := *opts.Benchmark benchmarkOpts.ID = adapter.ID() opts.Benchmark = &benchmarkOpts + if err := validateGateRuleScanners(opts); err != nil { + return BenchmarkArtifact{}, err + } if opts.Benchmark.PredictionsOutputPath != "" && !adapter.SupportsPredictionsOutput() { return BenchmarkArtifact{}, fmt.Errorf("predictions output is only supported for %s", openClawBenchmarkID) } diff --git a/internal/runner/benchmark_registry_test.go b/internal/runner/benchmark_registry_test.go index 1342765..dde6da1 100644 --- a/internal/runner/benchmark_registry_test.go +++ b/internal/runner/benchmark_registry_test.go @@ -69,7 +69,11 @@ func TestRunBenchmarkExecutesUserDefinedScannerFromRunRegistry(t *testing.T) { opts := benchmarkTestOptions(t, "clawhub-security-signals", "eval_holdout", 0, 0, "") opts.Scanners = []string{"fixture-scanner"} opts.ScannerRegistry = registry - commandRunner := &recordingCommandRunner{stdout: `{"verdict":"clean"}`} + opts.GateRules = map[string]ScannerGatePolicy{ + "fixture-scanner": {BlockOnExitCode: &ExitCodeRule{Codes: []int{3}}}, + } + exitCode := 3 + commandRunner := &recordingCommandRunner{stdout: `{"verdict":"clean"}`, err: errCommandFailed, exitCode: &exitCode} artifact, err := RunBenchmark(opts, RunContext{ Env: map[string]string{}, CommandRunner: commandRunner, BenchmarkClient: staticBenchmarkClient{rows: []OpenClawBenchmarkRow{{ @@ -87,6 +91,37 @@ func TestRunBenchmarkExecutesUserDefinedScannerFromRunRegistry(t *testing.T) { if result.Status != "completed" || string(result.Raw) != `{"verdict":"clean"}` { t.Fatalf("result = %#v", result) } + if artifact.Cases[0].Run.Gate != "block" || len(artifact.Cases[0].Run.GateRules) != 1 { + t.Fatalf("gate = %q, rules = %#v", artifact.Cases[0].Run.Gate, artifact.Cases[0].Run.GateRules) + } +} + +func TestRunBenchmarkRejectsUnrequestedGateScannerBeforeDatasetIO(t *testing.T) { + opts := benchmarkTestOptions(t, "clawhub-security-signals", "eval_holdout", 0, 0, "") + opts.GateRules = map[string]ScannerGatePolicy{ + "absent-scanner": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}, + } + _, err := RunBenchmark(opts, RunContext{ + Env: map[string]string{}, + BenchmarkClient: panicBenchmarkClient{}, + }) + if err == nil || err.Error() != "gate rule references scanner absent-scanner, but it was not requested" { + t.Fatalf("err = %v", err) + } +} + +type panicBenchmarkClient struct{} + +func (panicBenchmarkClient) FetchOpenClawRows(string, string, int, int) ([]OpenClawBenchmarkRow, error) { + panic("benchmark client called before gate validation") +} + +func (panicBenchmarkClient) FetchSkillTrustBenchRows(string, string, int, int) ([]SkillTrustBenchRow, error) { + panic("benchmark client called before gate validation") +} + +func (panicBenchmarkClient) MaterializeSkillTrustBenchRow(string, SkillTrustBenchRow) (string, error) { + panic("benchmark client called before gate validation") } type stubBenchmarkAdapter struct { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 1f40745..390f167 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -39,6 +39,33 @@ type Options struct { JSON bool Judge *JudgeOptions Sandbox SandboxOptions + GateRules map[string]ScannerGatePolicy +} + +type ExitCodeRule struct { + Codes []int + Nonzero bool +} + +// MaxGateExitCode excludes shell and container runtime infrastructure failures +// from scanner verdict policy. +const MaxGateExitCode = 124 + +func (rule ExitCodeRule) Matches(exitCode int) bool { + if rule.Nonzero { + return exitCode != 0 + } + for _, code := range rule.Codes { + if code == exitCode { + return true + } + } + return false +} + +type ScannerGatePolicy struct { + BlockOnExitCode *ExitCodeRule + WarnOnExitCode *ExitCodeRule } type BenchmarkOptions struct { @@ -83,8 +110,9 @@ type CommandRunner interface { } type CommandOutput struct { - Stdout string - Stderr string + Stdout string + Stderr string + ExitCode *int } type Artifact struct { @@ -100,9 +128,18 @@ type Artifact struct { Env map[string]string `json:"env"` Sandbox SandboxMetadata `json:"sandbox"` Scanners map[string]ScannerResult `json:"scanners"` + Gate string `json:"gate"` + GateRules []FiredGateRule `json:"gateRules"` Judge *JudgeResult `json:"judge"` } +type FiredGateRule struct { + Scanner string `json:"scanner"` + Rule string `json:"rule"` + ExitCode int `json:"exitCode"` + Action string `json:"action"` +} + type RunTargetsResult struct { Single *Artifact Batch *BatchArtifact @@ -159,6 +196,7 @@ type ScannerResult struct { Command []string `json:"command"` Error string `json:"error"` OutputPath string `json:"outputPath,omitempty"` + ExitCode *int `json:"exitCode,omitempty"` Raw json.RawMessage `json:"raw"` } @@ -371,6 +409,9 @@ func ValidateRequirements(opts Options, env map[string]string) error { } func Run(opts Options, ctx RunContext) (Artifact, error) { + if err := validateGateRuleScanners(opts); err != nil { + return Artifact{}, err + } env := ctx.Env if env == nil { env = EnvMap(os.Environ()) @@ -430,7 +471,12 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { } artifact.Context = json.RawMessage(context) } + scanned := make(map[string]bool, len(opts.Scanners)) for _, scanner := range opts.Scanners { + if scanned[scanner] { + continue + } + scanned[scanner] = true scannerStartedAt := now().UTC().Format(time.RFC3339Nano) scannerTimerStarted := time.Now() result, err := scannerResult(opts, scanner, target, scannerStartedAt, scannerRunner) @@ -440,6 +486,7 @@ func Run(opts Options, ctx RunContext) (Artifact, error) { result.DurationMs = time.Since(scannerTimerStarted).Milliseconds() artifact.Scanners[scanner] = result } + evaluateGate(&artifact, opts) if opts.Judge != nil { result, err := RunJudge(*opts.Judge, artifact, commandRunner, 20*time.Minute, env, sandbox.Mode) if err != nil { @@ -2108,10 +2155,18 @@ func (runner defaultCommandRunner) Run(command string, args []string, cwd string cmd.Stdout = &stdout cmd.Stderr = &stderr err := cmd.Run() - if ctx.Err() == context.DeadlineExceeded { + timedOut := ctx.Err() == context.DeadlineExceeded + if timedOut { err = fmt.Errorf("command timed out after %s", timeout) } - return CommandOutput{Stdout: stdout.String(), Stderr: stderr.String()}, err + output := CommandOutput{Stdout: stdout.String(), Stderr: stderr.String()} + if !timedOut && cmd.ProcessState != nil { + exitCode := cmd.ProcessState.ExitCode() + if exitCode >= 0 { + output.ExitCode = &exitCode + } + } + return output, err } func NewArtifact(opts Options, resolvedPath string, startedAt string, completedAt string, env map[string]string) Artifact { @@ -2145,10 +2200,59 @@ func NewArtifact(opts Options, resolvedPath string, startedAt string, completedA Env: envPresence(opts, env), Sandbox: mustSandboxMetadata(opts, env), Scanners: scanners, + Gate: "pass", + GateRules: []FiredGateRule{}, Judge: nil, } } +func validateGateRuleScanners(opts Options) error { + requested := make(map[string]bool, len(opts.Scanners)) + for _, scanner := range opts.Scanners { + requested[scanner] = true + } + var unrequested []string + for scanner := range opts.GateRules { + if !requested[scanner] { + unrequested = append(unrequested, scanner) + } + } + if len(unrequested) == 0 { + return nil + } + sort.Strings(unrequested) + return fmt.Errorf("gate rule references scanner %s, but it was not requested", unrequested[0]) +} + +func evaluateGate(artifact *Artifact, opts Options) { + evaluated := make(map[string]bool, len(opts.Scanners)) + for _, scanner := range opts.Scanners { + if evaluated[scanner] { + continue + } + evaluated[scanner] = true + result := artifact.Scanners[scanner] + if result.Status != "completed" || result.ExitCode == nil { + continue + } + policy := opts.GateRules[scanner] + if policy.BlockOnExitCode != nil && policy.BlockOnExitCode.Matches(*result.ExitCode) { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "blockOnExitCode", ExitCode: *result.ExitCode, Action: "block", + }) + artifact.Gate = "block" + } + if policy.WarnOnExitCode != nil && policy.WarnOnExitCode.Matches(*result.ExitCode) { + artifact.GateRules = append(artifact.GateRules, FiredGateRule{ + Scanner: scanner, Rule: "warnOnExitCode", ExitCode: *result.ExitCode, Action: "warn", + }) + if artifact.Gate == "pass" { + artifact.Gate = "warn" + } + } + } +} + func WriteJSON(w io.Writer, value interface{}) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 66b0726..7d21ad7 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -11,6 +11,8 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" + "runtime" "strings" "testing" "time" @@ -1121,6 +1123,28 @@ func TestArtifactConfigSourceField_FlagsOnly(t *testing.T) { } } +func TestNewArtifactAlwaysIncludesPassingGate(t *testing.T) { + opts, err := ParseArgs([]string{"./my-skill", "--scanner", "clawscan-static"}) + if err != nil { + t.Fatal(err) + } + + artifact := NewArtifact(opts, "/tmp/my-skill", "start", "complete", map[string]string{}) + if artifact.Gate != "pass" { + t.Fatalf("gate = %q", artifact.Gate) + } + raw, err := json.Marshal(artifact) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(raw, []byte(`"gate":"pass"`)) { + t.Fatalf("artifact omitted gate: %s", raw) + } + if !bytes.Contains(raw, []byte(`"gateRules":[]`)) { + t.Fatalf("artifact gateRules default is not an empty array: %s", raw) + } +} + func TestRunWritesScannerOnlyArtifact(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -1179,6 +1203,193 @@ func TestRunIncludesDurationMsForScannerResults(t *testing.T) { assertScannerDurationJSON(t, artifact, "skillspector") } +func TestRunBlocksWhenNonzeroExitCodeRuleFires(t *testing.T) { + target := t.TempDir() + exitCode := 2 + opts := Options{ + Target: target, Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}, + }, + } + artifact, err := Run(opts, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{ + results: map[string]ScannerResult{"clawscan-static": {Status: "completed", ExitCode: &exitCode}}, + }}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" { + t.Fatalf("gate = %q", artifact.Gate) + } + want := []FiredGateRule{{Scanner: "clawscan-static", Rule: "blockOnExitCode", ExitCode: 2, Action: "block"}} + if !reflect.DeepEqual(artifact.GateRules, want) { + t.Fatalf("gate rules = %#v", artifact.GateRules) + } +} + +func TestRunExitCodeGateActionsAndPrecedence(t *testing.T) { + tests := []struct { + name string + results map[string]ScannerResult + rules map[string]ScannerGatePolicy + want string + fired int + }{ + { + name: "zero does not fire nonzero", results: gateResults(0), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + { + name: "warning fires", results: gateResults(3), + rules: map[string]ScannerGatePolicy{"clawscan-static": {WarnOnExitCode: &ExitCodeRule{Codes: []int{3}}}}, + want: "warn", fired: 1, + }, + { + name: "listed code fires", results: gateResults(2), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Codes: []int{1, 2, 3}}}}, + want: "block", fired: 1, + }, + { + name: "unlisted code passes", results: gateResults(4), + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Codes: []int{1, 2, 3}}}}, + want: "pass", + }, + { + name: "skipped scanner does not fire", results: map[string]ScannerResult{"clawscan-static": {Status: "skipped", ExitCode: intPointer(2)}}, + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + { + name: "failed scanner does not fire", results: map[string]ScannerResult{"clawscan-static": {Status: "failed", ExitCode: intPointer(2)}}, + rules: map[string]ScannerGatePolicy{"clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + want: "pass", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + target := t.TempDir() + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, GateRules: test.rules, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: test.results}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != test.want || len(artifact.GateRules) != test.fired { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + }) + } +} + +func TestRunBlockGateBeatsWarnAcrossScanners(t *testing.T) { + target := t.TempDir() + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static", "skillspector"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {WarnOnExitCode: &ExitCodeRule{Codes: []int{1}}}, + "skillspector": {BlockOnExitCode: &ExitCodeRule{Codes: []int{2}}}, + }, + }, RunContext{Env: map[string]string{}, ScannerRunner: &gateScannerRunner{results: map[string]ScannerResult{ + "clawscan-static": {Status: "completed", ExitCode: intPointer(1)}, + "skillspector": {Status: "completed", ExitCode: intPointer(2)}, + }}}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" || len(artifact.GateRules) != 2 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } +} + +func TestRunRepeatedScannerFlagFiresGateRuleOnce(t *testing.T) { + target := t.TempDir() + scannerRunner := &gateScannerRunner{results: gateResults(2)} + artifact, err := Run(Options{ + Target: target, Scanners: []string{"clawscan-static", "clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{ + "clawscan-static": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}, + }, + }, RunContext{Env: map[string]string{}, ScannerRunner: scannerRunner}) + if err != nil { + t.Fatal(err) + } + if artifact.Gate != "block" || len(artifact.GateRules) != 1 { + t.Fatalf("gate = %q, rules = %#v", artifact.Gate, artifact.GateRules) + } + if scannerRunner.calls != 1 { + t.Fatalf("scanner ran %d times", scannerRunner.calls) + } +} + +func TestRunRejectsGateRuleForUnrequestedScannerBeforeScanning(t *testing.T) { + scannerRunner := &gateScannerRunner{results: gateResults(0)} + _, err := Run(Options{ + Target: t.TempDir(), Scanners: []string{"clawscan-static"}, Sandbox: SandboxOptions{Mode: SandboxModeOff}, + GateRules: map[string]ScannerGatePolicy{"absent-scanner": {BlockOnExitCode: &ExitCodeRule{Nonzero: true}}}, + }, RunContext{Env: map[string]string{}, ScannerRunner: scannerRunner}) + if err == nil || err.Error() != "gate rule references scanner absent-scanner, but it was not requested" { + t.Fatalf("err = %v", err) + } + if scannerRunner.calls != 0 { + t.Fatalf("scanner ran %d times", scannerRunner.calls) + } +} + +func TestDefaultCommandRunnerCapturesProcessExitCode(t *testing.T) { + const helperEnv = "CLAWSCAN_TEST_COMMAND_MODE" + switch os.Getenv(helperEnv) { + case "exit": + os.Exit(7) + case "timeout": + time.Sleep(time.Minute) + return + } + env := EnvMap(os.Environ()) + env[helperEnv] = "exit" + output, err := (defaultCommandRunner{Env: env}).Run( + os.Args[0], + []string{"-test.run=^TestDefaultCommandRunnerCapturesProcessExitCode$"}, + "", + time.Minute, + ) + if err == nil { + t.Fatal("helper process unexpectedly succeeded") + } + if output.ExitCode == nil || *output.ExitCode != 7 { + t.Fatalf("exit code = %#v", output.ExitCode) + } + + env[helperEnv] = "timeout" + output, err = (defaultCommandRunner{Env: env}).Run( + os.Args[0], + []string{"-test.run=^TestDefaultCommandRunnerCapturesProcessExitCode$"}, + "", + time.Second, + ) + if err == nil || !strings.Contains(err.Error(), "command timed out") { + t.Fatalf("err = %v", err) + } + if output.ExitCode != nil { + t.Fatalf("timed-out command exit code = %d", *output.ExitCode) + } + + if runtime.GOOS != "windows" { + output, err = (defaultCommandRunner{Env: env}).Run( + "/bin/sh", + []string{"-c", "kill -TERM $$"}, + "", + time.Minute, + ) + if err == nil { + t.Fatal("signaled helper process unexpectedly succeeded") + } + if output.ExitCode != nil { + t.Fatalf("signaled command exit code = %d", *output.ExitCode) + } + } +} + func TestRunIncludesDurationMsForFixtureScannerResults(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "skill") @@ -4093,6 +4304,27 @@ func (skippedScannerRunner) RunScanner(name string, target string, startedAt str }, nil } +type gateScannerRunner struct { + results map[string]ScannerResult + calls int +} + +func (runner *gateScannerRunner) RunScanner(name string, _ string, startedAt string) (ScannerResult, error) { + runner.calls++ + result := runner.results[name] + result.StartedAt = startedAt + result.CompletedAt = startedAt + return result, nil +} + +func gateResults(exitCode int) map[string]ScannerResult { + return map[string]ScannerResult{"clawscan-static": {Status: "completed", ExitCode: intPointer(exitCode)}} +} + +func intPointer(value int) *int { + return &value +} + type staticScannerRunner struct { results map[string]ScannerResult } @@ -4337,6 +4569,7 @@ type recordingCommandRunner struct { stdout string stderr string err error + exitCode *int runHook func(command string, args []string, cwd string) error } @@ -4377,7 +4610,7 @@ func (r *recordingCommandRunner) Run(command string, args []string, cwd string, if stdout == "" { stdout = "ok" } - return CommandOutput{Stdout: stdout, Stderr: r.stderr}, r.err + return CommandOutput{Stdout: stdout, Stderr: r.stderr, ExitCode: r.exitCode}, r.err } var errCommandFailed = errors.New("exit status 1") diff --git a/internal/runner/scanner_registry_test.go b/internal/runner/scanner_registry_test.go index ed892f3..1d5b23c 100644 --- a/internal/runner/scanner_registry_test.go +++ b/internal/runner/scanner_registry_test.go @@ -3,6 +3,7 @@ package runner import ( "encoding/json" "errors" + "fmt" "path/filepath" "strings" "testing" @@ -156,7 +157,10 @@ func TestUserDefinedScannerPreservesValidJSONOnCommandFailure(t *testing.T) { if err != nil { t.Fatal(err) } - commandRunner := &recordingCommandRunner{stdout: `{"findings":["detected"]}`, stderr: "findings detected", err: errCommandFailed} + exitCode := 2 + commandRunner := &recordingCommandRunner{ + stdout: `{"findings":["detected"]}`, stderr: "findings detected", err: errCommandFailed, exitCode: &exitCode, + } result, err := (ExternalScannerRunner{ Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") @@ -201,6 +205,51 @@ func TestUserDefinedScannerRejectsMissingOrInvalidJSON(t *testing.T) { } } +func TestUserDefinedScannerRecordsExitCode(t *testing.T) { + exitCode := 2 + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`, err: errCommandFailed, exitCode: &exitCode} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.ExitCode == nil || *result.ExitCode != 2 { + t.Fatalf("exit code = %#v", result.ExitCode) + } +} + +func TestUserDefinedScannerPreservesEvidenceWithoutGatingInfrastructureExitCodes(t *testing.T) { + for _, exitCode := range []int{-1, 125, 126, 127, 137} { + t.Run(fmt.Sprint(exitCode), func(t *testing.T) { + adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ + ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, + }) + registry, err := NewScannerRegistry(adapter) + if err != nil { + t.Fatal(err) + } + commandRunner := &recordingCommandRunner{stdout: `{}`, err: errCommandFailed, exitCode: &exitCode} + result, err := (ExternalScannerRunner{ + Registry: registry, CommandRunner: commandRunner, Env: map[string]string{}, SandboxMode: SandboxModeOff, + }).RunScanner("demo", t.TempDir(), "2026-07-21T00:00:00Z") + if err != nil { + t.Fatal(err) + } + if result.Status != "completed" || result.ExitCode != nil || string(result.Raw) != `{}` { + t.Fatalf("result = %#v", result) + } + }) + } +} + func TestUserDefinedScannerInterpolatesDollarTargetLiterally(t *testing.T) { adapter := NewUserDefinedScanner(UserDefinedScannerConfig{ ID: "demo", Command: "demo {{target}}", Targets: []string{"skill"}, diff --git a/internal/runner/user_defined_scanner.go b/internal/runner/user_defined_scanner.go index b30e38e..48a62c2 100644 --- a/internal/runner/user_defined_scanner.go +++ b/internal/runner/user_defined_scanner.go @@ -76,6 +76,7 @@ func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, targe timeout = 20 * time.Minute } output, runErr := runner.CommandRunner.Run(shell.command, args, userDefinedScannerCWD(target), timeout) + exitCode := gateEligibleExitCode(output.ExitCode) completedAt := time.Now().UTC().Format(time.RFC3339Nano) raw := strings.TrimSpace(output.Stdout) if runErr != nil { @@ -83,26 +84,33 @@ func (adapter userDefinedScannerAdapter) Run(runner ExternalScannerRunner, targe if json.Valid([]byte(raw)) { return ScannerResult{ Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Error: message, Raw: json.RawMessage(raw), + Error: message, ExitCode: exitCode, Raw: json.RawMessage(raw), }, nil } return ScannerResult{ Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Error: message, + Error: message, ExitCode: exitCode, }, nil } if !json.Valid([]byte(raw)) { return ScannerResult{ Status: "failed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), + Error: fmt.Sprintf("User-defined scanner %s returned invalid JSON", adapter.config.ID), ExitCode: exitCode, }, nil } return ScannerResult{ Status: "completed", StartedAt: startedAt, CompletedAt: completedAt, Command: fullCommand, - Raw: json.RawMessage(raw), + ExitCode: exitCode, Raw: json.RawMessage(raw), }, nil } +func gateEligibleExitCode(exitCode *int) *int { + if exitCode == nil || *exitCode < 0 || *exitCode > MaxGateExitCode { + return nil + } + return exitCode +} + func userDefinedScannerShell(goos string, sandboxMode string) judgeShellSpec { if sandboxMode == SandboxModeDocker { return judgeShellForGOOS("linux")