diff --git a/cmd/mutate.go b/cmd/mutate.go index 175721a..c80fd44 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -17,12 +17,12 @@ import ( ) var ( - mutateCases string - mutateTarget string - mutateOutput string - mutateOperators string - mutateReportFormat string - mutateConcurrency int + mutateCases string + mutateTarget string + mutateOutput string + mutateOperators string + mutateReportFormat string + mutateConcurrency int mutateOperatorConcurrency int ) @@ -38,7 +38,7 @@ Requires hurl on PATH. Test cases must be previously generated with 'caseforge g Exit codes: 0 — run complete, no survivors - 6 — one or more mutations survived + 6 — one or more mutations survived, or --min-score threshold breached Examples: caseforge mutate --cases ./cases --target http://localhost:8080 @@ -60,6 +60,8 @@ func init() { mutateCmd.Flags().StringVar(&mutateReportFormat, "report-format", "json", `Comma-separated report formats: json,markdown,html,all`) mutateCmd.Flags().Bool("history", false, "Print mutation score history (does not run mutations; --target not required)") mutateCmd.Flags().Int("history-limit", 10, "Maximum number of historical runs to display") + mutateCmd.Flags().Float64("min-score", 0, + "Per-operation minimum mutation score (0.0–1.0); exit 6 if any operation scores below this") mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)") mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)") mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix") @@ -96,6 +98,11 @@ func runMutate(cmd *cobra.Command, _ []string) error { return err } + minScore, _ := cmd.Flags().GetFloat64("min-score") + if minScore < 0 || minScore > 1.0 { + return fmt.Errorf("--min-score must be between 0.0 and 1.0, got %.2f", minScore) + } + ops, err := resolveOperators(mutateOperators) if err != nil { return err @@ -187,6 +194,23 @@ func runMutate(cmd *cobra.Command, _ []string) error { } } + if minScore > 0 { + var failing []mutation.OperationScore + for _, op := range run.OperationScores { + if op.MutationScore < minScore { + failing = append(failing, op) + } + } + if len(failing) > 0 { + color.New(color.FgRed).Fprintf(out, + "\n✗ %d operation(s) below --min-score %.0f%%:\n", len(failing), minScore*100) + for _, op := range failing { + fmt.Fprintf(out, " %-52s %3.0f%%\n", op.Operation, op.MutationScore*100) + } + os.Exit(ExitPartialSuccess) + } + } + if run.Survivors > 0 { os.Exit(ExitPartialSuccess) } diff --git a/internal/mutation/engine.go b/internal/mutation/engine.go index 61ffc2b..4bc0e26 100644 --- a/internal/mutation/engine.go +++ b/internal/mutation/engine.go @@ -19,6 +19,9 @@ type indexFile struct { TestCases []struct { ID string `json:"id"` Title string `json:"title"` + Source struct { + SpecPath string `json:"spec_path"` + } `json:"source"` } `json:"test_cases"` } @@ -69,10 +72,11 @@ func Run(opts RunOptions) (MutationRun, error) { survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID) mu.Lock() results = append(results, CaseMutationResult{ - CaseID: tc.ID, - Title: tc.Title, - Operator: op.Name(), - Survived: survived, + CaseID: tc.ID, + Title: tc.Title, + Operation: tc.Operation, + Operator: op.Name(), + Survived: survived, }) mu.Unlock() return nil @@ -106,16 +110,17 @@ func Run(opts RunOptions) (MutationRun, error) { } return MutationRun{ - Target: opts.Target, - CasesDir: opts.CasesDir, - Operators: opNames, - TotalCases: len(cases), - TotalRuns: total, - Killed: killed, - Survivors: survivors, - MutationScore: score, - Results: results, - GeneratedAt: time.Now().UTC().Format("2006-01-02T15:04:05Z"), + Target: opts.Target, + CasesDir: opts.CasesDir, + Operators: opNames, + TotalCases: len(cases), + TotalRuns: total, + Killed: killed, + Survivors: survivors, + MutationScore: score, + Results: results, + OperationScores: ComputeOperationScores(results), + GeneratedAt: time.Now().UTC().Format("2006-01-02T15:04:05Z"), }, nil } @@ -152,8 +157,9 @@ func runOnce(proxyAddr, casesDir, caseID string) bool { } type caseRef struct { - ID string - Title string + ID string + Title string + Operation string // source.spec_path; "(unknown)" if absent } func loadCases(casesDir string) ([]caseRef, error) { @@ -167,7 +173,11 @@ func loadCases(casesDir string) ([]caseRef, error) { } refs := make([]caseRef, len(idx.TestCases)) for i, tc := range idx.TestCases { - refs[i] = caseRef{ID: tc.ID, Title: tc.Title} + op := tc.Source.SpecPath + if op == "" { + op = "(unknown)" + } + refs[i] = caseRef{ID: tc.ID, Title: tc.Title, Operation: op} } return refs, nil } diff --git a/internal/mutation/render.go b/internal/mutation/render.go index f08bad0..487dd75 100644 --- a/internal/mutation/render.go +++ b/internal/mutation/render.go @@ -19,6 +19,8 @@ func RenderMarkdown(run MutationRun) string { fmt.Fprintf(&b, "Mutation Score: **%d/%d killed (%d%%)** · Survivors: **%d**\n\n", run.Killed, run.TotalRuns, pct, run.Survivors) + b.WriteString(renderOperationTable(run)) + if len(run.Clusters) > 0 { b.WriteString("## Survivor Summary (by risk)\n\n") b.WriteString("| Case | Title | Risk | Survived Operators |\n") @@ -48,6 +50,57 @@ func RenderMarkdown(run MutationRun) string { return b.String() } +// renderOperationTable returns the per-operation Markdown table, or "" if no scores. +func renderOperationTable(run MutationRun) string { + if len(run.OperationScores) == 0 { + return "" + } + var b strings.Builder + fmt.Fprintf(&b, "## Per-Operation Mutation Score\n\n") + fmt.Fprintf(&b, "| Operation | Score | Killed | Survivors |\n") + fmt.Fprintf(&b, "|-----------|-------|--------|-----------|\n") + for _, op := range run.OperationScores { + pct := int(op.MutationScore * 100) + badge := "" + switch { + case op.MutationScore == 1.0: + badge = " ✓" + case op.MutationScore < 0.7: + badge = " ⚠" + } + fmt.Fprintf(&b, "| %s | %d%%%s | %d/%d | %d |\n", + op.Operation, pct, badge, op.Killed, op.TotalRuns, op.Survivors) + } + b.WriteString("\n") + return b.String() +} + +// renderOperationTableHTML returns the per-operation HTML table, or "" if no scores. +func renderOperationTableHTML(run MutationRun) string { + if len(run.OperationScores) == 0 { + return "" + } + var b strings.Builder + b.WriteString("

Per-Operation Mutation Score

\n") + b.WriteString("\n\n") + for _, op := range run.OperationScores { + pct := int(op.MutationScore * 100) + rowClass := "" + switch { + case op.MutationScore == 1.0: + rowClass = " class=\"op-high\"" + case op.MutationScore < 0.7: + rowClass = " class=\"op-low\"" + } + fmt.Fprintf(&b, "\n", + rowClass, + html.EscapeString(op.Operation), + pct, op.Killed, op.TotalRuns, op.Survivors) + } + b.WriteString("
OperationScoreKilledSurvivors
%s%d%%%d/%d%d
\n") + return b.String() +} + // RenderHTML returns a self-contained HTML mutation report (no external resources). func RenderHTML(run MutationRun) string { // build operator×case result grid @@ -86,6 +139,8 @@ td{padding:4px 8px;border:1px solid #e5e7eb;white-space:nowrap} .killed{background:#dcfce7;color:#166534;text-align:center} .survived{background:#fee2e2;color:#991b1b;text-align:center} .na{background:#f9fafb;color:#9ca3af;text-align:center} +.op-low{background:#fef2f2} +.op-high{background:#f0fdf4} h2{margin-top:2rem} `) @@ -130,6 +185,7 @@ h2{margin-top:2rem} b.WriteString("\n") } + b.WriteString(renderOperationTableHTML(run)) b.WriteString("\n") return b.String() } diff --git a/internal/mutation/render_test.go b/internal/mutation/render_test.go index 12385fc..24fc465 100644 --- a/internal/mutation/render_test.go +++ b/internal/mutation/render_test.go @@ -112,3 +112,57 @@ func TestRenderHTML_NoSurvivors(t *testing.T) { t.Error("must not show Survivors by Risk table when no clusters") } } + +func TestRenderMarkdown_OperationTable(t *testing.T) { + run := sampleRun() + run.OperationScores = []mutation.OperationScore{ + {Operation: "GET /pets", TotalRuns: 2, Killed: 1, Survivors: 1, MutationScore: 0.5}, + {Operation: "POST /pets", TotalRuns: 2, Killed: 2, Survivors: 0, MutationScore: 1.0}, + } + out := mutation.RenderMarkdown(run) + if !strings.Contains(out, "Per-Operation Mutation Score") { + t.Error("expected per-operation section header") + } + if !strings.Contains(out, "GET /pets") { + t.Error("expected GET /pets row") + } + if !strings.Contains(out, "50%") { + t.Error("expected 50% score for GET /pets") + } + if !strings.Contains(out, "⚠") { + t.Error("expected ⚠ badge for score < 70%") + } + if !strings.Contains(out, "✓") { + t.Error("expected ✓ badge for 100% score") + } +} + +func TestRenderMarkdown_NoOperationScores(t *testing.T) { + run := sampleRun() + run.OperationScores = nil + out := mutation.RenderMarkdown(run) + if strings.Contains(out, "Per-Operation") { + t.Error("per-operation section must be absent when OperationScores is nil") + } +} + +func TestRenderHTML_OperationTable(t *testing.T) { + run := sampleRun() + run.OperationScores = []mutation.OperationScore{ + {Operation: "GET /pets", TotalRuns: 2, Killed: 1, Survivors: 1, MutationScore: 0.5}, + {Operation: "DELETE /pets/{id}", TotalRuns: 2, Killed: 2, Survivors: 0, MutationScore: 1.0}, + } + out := mutation.RenderHTML(run) + if !strings.Contains(out, "Per-Operation") { + t.Error("expected Per-Operation section in HTML") + } + if !strings.Contains(out, "GET /pets") { + t.Error("expected GET /pets row in HTML") + } + if !strings.Contains(out, "fef2f2") { + t.Error("expected light-red background for score < 70%") + } + if !strings.Contains(out, "f0fdf4") { + t.Error("expected light-green background for 100% score") + } +} diff --git a/internal/mutation/score.go b/internal/mutation/score.go new file mode 100644 index 0000000..1211788 --- /dev/null +++ b/internal/mutation/score.go @@ -0,0 +1,47 @@ +package mutation + +import "sort" + +// ComputeOperationScores groups CaseMutationResults by operation and returns +// per-operation scores sorted weakest-first (score ascending, then alphabetically). +// Results with empty Operation are grouped under "(unknown)". +func ComputeOperationScores(results []CaseMutationResult) []OperationScore { + type stats struct{ killed, survivors int } + byOp := map[string]*stats{} + for _, r := range results { + op := r.Operation + if op == "" { + op = "(unknown)" + } + if byOp[op] == nil { + byOp[op] = &stats{} + } + if r.Survived { + byOp[op].survivors++ + } else { + byOp[op].killed++ + } + } + scores := make([]OperationScore, 0, len(byOp)) + for op, s := range byOp { + total := s.killed + s.survivors + score := 0.0 + if total > 0 { + score = float64(s.killed) / float64(total) + } + scores = append(scores, OperationScore{ + Operation: op, + TotalRuns: total, + Killed: s.killed, + Survivors: s.survivors, + MutationScore: score, + }) + } + sort.Slice(scores, func(i, j int) bool { + if scores[i].MutationScore != scores[j].MutationScore { + return scores[i].MutationScore < scores[j].MutationScore + } + return scores[i].Operation < scores[j].Operation + }) + return scores +} diff --git a/internal/mutation/score_test.go b/internal/mutation/score_test.go new file mode 100644 index 0000000..87f2fcd --- /dev/null +++ b/internal/mutation/score_test.go @@ -0,0 +1,66 @@ +package mutation_test + +import ( + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func TestComputeOperationScores_Basic(t *testing.T) { + results := []mutation.CaseMutationResult{ + {CaseID: "TC-0001", Operation: "GET /pets", Operator: "field_drop", Survived: false}, + {CaseID: "TC-0001", Operation: "GET /pets", Operator: "status_swap_2xx", Survived: false}, + {CaseID: "TC-0002", Operation: "POST /pets", Operator: "field_drop", Survived: true}, + {CaseID: "TC-0002", Operation: "POST /pets", Operator: "status_swap_2xx", Survived: false}, + } + scores := mutation.ComputeOperationScores(results) + + if len(scores) != 2 { + t.Fatalf("expected 2 operations, got %d", len(scores)) + } + // weakest first: POST /pets = 0.5, GET /pets = 1.0 + if scores[0].Operation != "POST /pets" { + t.Errorf("expected weakest first, got %s", scores[0].Operation) + } + if scores[0].Killed != 1 || scores[0].Survivors != 1 || scores[0].TotalRuns != 2 { + t.Errorf("POST /pets stats wrong: %+v", scores[0]) + } + if scores[1].Operation != "GET /pets" { + t.Errorf("expected GET /pets second, got %s", scores[1].Operation) + } + if scores[1].MutationScore != 1.0 { + t.Errorf("GET /pets should be 100%%, got %f", scores[1].MutationScore) + } +} + +func TestComputeOperationScores_Empty(t *testing.T) { + scores := mutation.ComputeOperationScores(nil) + if len(scores) != 0 { + t.Fatalf("expected empty slice, got %d entries", len(scores)) + } +} + +func TestComputeOperationScores_UnknownFallback(t *testing.T) { + results := []mutation.CaseMutationResult{ + {CaseID: "TC-0001", Operation: "", Operator: "field_drop", Survived: true}, + } + scores := mutation.ComputeOperationScores(results) + if len(scores) != 1 { + t.Fatalf("expected 1 entry, got %d", len(scores)) + } + if scores[0].Operation != "(unknown)" { + t.Errorf("expected '(unknown)', got %q", scores[0].Operation) + } +} + +func TestComputeOperationScores_TieBreak(t *testing.T) { + // Equal scores must be sorted alphabetically by operation name + results := []mutation.CaseMutationResult{ + {Operation: "POST /z", Operator: "field_drop", Survived: true}, + {Operation: "POST /a", Operator: "field_drop", Survived: true}, + } + scores := mutation.ComputeOperationScores(results) + if scores[0].Operation != "POST /a" { + t.Errorf("expected alphabetical tiebreak, got %s first", scores[0].Operation) + } +} diff --git a/internal/mutation/types.go b/internal/mutation/types.go index 04947eb..e39a2f8 100644 --- a/internal/mutation/types.go +++ b/internal/mutation/types.go @@ -15,10 +15,11 @@ type Operator interface { // CaseMutationResult records whether one operator survived against one test case. type CaseMutationResult struct { - CaseID string `json:"case_id"` - Title string `json:"title"` - Operator string `json:"operator"` - Survived bool `json:"survived"` // true = mutation was not caught by assertions + CaseID string `json:"case_id"` + Title string `json:"title"` + Operation string `json:"operation"` // from source.spec_path; "(unknown)" if absent + Operator string `json:"operator"` + Survived bool `json:"survived"` // true = mutation was not caught by assertions } // SurvivorCluster groups all operators that survived a single test case. @@ -29,6 +30,15 @@ type SurvivorCluster struct { RiskScore float64 `json:"risk_score"` // len(Operators) / total operators in run } +// OperationScore holds the per-operation mutation score breakdown. +type OperationScore struct { + Operation string `json:"operation"` // "METHOD /path" + TotalRuns int `json:"total_runs"` + Killed int `json:"killed"` + Survivors int `json:"survivors"` + MutationScore float64 `json:"mutation_score"` // killed / total_runs +} + // SuggestedAssertion is one LLM-suggested assertion to add to an existing test case. type SuggestedAssertion struct { Target string `json:"target"` @@ -47,18 +57,19 @@ type FeedbackItem struct { // MutationRun is the top-level report for a complete mutation run. type MutationRun struct { - Target string `json:"target"` - CasesDir string `json:"cases_dir"` - Operators []string `json:"operators"` - TotalCases int `json:"total_cases"` - TotalRuns int `json:"total_runs"` // cases × operators - Killed int `json:"killed"` - Survivors int `json:"survivors"` - MutationScore float64 `json:"mutation_score"` // killed / total_runs - Results []CaseMutationResult `json:"results"` - Clusters []SurvivorCluster `json:"clusters,omitempty"` // Phase 2 - Feedback []FeedbackItem `json:"feedback,omitempty"` // Phase 2 - GeneratedAt string `json:"generated_at"` + Target string `json:"target"` + CasesDir string `json:"cases_dir"` + Operators []string `json:"operators"` + TotalCases int `json:"total_cases"` + TotalRuns int `json:"total_runs"` // cases × operators + Killed int `json:"killed"` + Survivors int `json:"survivors"` + MutationScore float64 `json:"mutation_score"` // killed / total_runs + Results []CaseMutationResult `json:"results"` + Clusters []SurvivorCluster `json:"clusters,omitempty"` // Phase 2 + OperationScores []OperationScore `json:"operation_scores,omitempty"` + Feedback []FeedbackItem `json:"feedback,omitempty"` // Phase 2 + GeneratedAt string `json:"generated_at"` } // RunOptions configures a mutation run. diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 62285fe..0d5aed5 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1733,6 +1733,102 @@ fi echo "" +# AT-410 – AT-412: per-operation mutation score +# ───────────────────────────────────────────────────────────────────────────── + +echo "# AT-410 – AT-412: per-operation score" + +# AT-410: existing flags unaffected (sanity) +contains "AT-410" "mutate --help still contains cases flag" "cases" "'$BIN' mutate --help" + +# AT-411 & AT-412: full run produces per-operation data +if command -v hurl >/dev/null 2>&1; then + AT411_CASES=$(mktemp -d) + AT411_REPORT=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT411_CASES" >/dev/null 2>&1 + + PORT411=$(random_port) + "$BIN" sandbox --spec "$WORKDIR/petstore.yaml" --port $PORT411 >/dev/null 2>&1 & + SBX411_PID=$! + for i in $(seq 1 20); do curl -sf "http://127.0.0.1:$PORT411/pets" >/dev/null 2>&1 && break; sleep 0.1; done + + "$BIN" mutate --cases "$AT411_CASES" \ + --target "http://127.0.0.1:$PORT411" \ + --output "$AT411_REPORT" \ + --report-format markdown,json \ + --operator field_drop || true + kill $SBX411_PID 2>/dev/null || true + + contains "AT-411" "mutation-report.md contains Per-Operation" "Per-Operation" \ + "cat '$AT411_REPORT/mutation-report.md'" + + contains "AT-412" "mutation-report.json contains operation_scores" "operation_scores" \ + "cat '$AT411_REPORT/mutation-report.json'" + + rm -rf "$AT411_CASES" "$AT411_REPORT" +else + skip "AT-411" "per-operation markdown section" "hurl not installed" + skip "AT-412" "operation_scores in JSON" "hurl not installed" +fi + +echo "" + +# AT-413 – AT-416: --min-score CI gate +# ───────────────────────────────────────────────────────────────────────────── + +echo "# AT-413 – AT-416: min-score CI gate" + +# AT-413: flag appears in help +contains "AT-413" "min-score flag in help" "min-score" "'$BIN' mutate --help" + +# AT-414: out-of-range value fails before running +run "AT-414" "--min-score 1.5 returns error immediately" \ + "! '$BIN' mutate --cases /tmp --target http://x --min-score 1.5 >/dev/null 2>&1" + +# AT-415 & AT-416: full integration with hurl +if command -v hurl >/dev/null 2>&1; then + AT415_CASES=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT415_CASES" >/dev/null 2>&1 + + PORT415=$(random_port) + "$BIN" sandbox --spec "$WORKDIR/petstore.yaml" --port $PORT415 >/dev/null 2>&1 & + SBX415_PID=$! + for i in $(seq 1 20); do curl -sf "http://127.0.0.1:$PORT415/pets" >/dev/null 2>&1 && break; sleep 0.1; done + + # AT-415: --min-score 0 (default) exits 0 when no survivors + exits_with "AT-415" "--min-score 0 exits 0 when no survivors" "0" \ + "'$BIN' mutate --cases '$AT415_CASES' --target \"http://127.0.0.1:$PORT415\" --min-score 0 --operator status_swap_2xx" + kill $SBX415_PID 2>/dev/null || true + + # AT-416: --min-score 0.99 exits 6 and prints "below --min-score" when operators survive + # Use a hand-crafted cases dir (TC-XXXX.hurl naming) so runOnce can actually find the file. + # The hurl only checks status — field_drop on an array body returns unchanged → mutation survives. + AT416_CASES=$(mktemp -d) + printf '{"test_cases":[{"id":"TC-0416","title":"GET /pets","source":{"spec_path":"GET /pets"}}]}' \ + > "$AT416_CASES/index.json" + printf 'GET {{base_url}}/pets\nHTTP 200\n' > "$AT416_CASES/TC-0416.hurl" + + PORT416=$(random_port) + "$BIN" sandbox --spec "$WORKDIR/petstore.yaml" --port $PORT416 >/dev/null 2>&1 & + SBX416_PID=$! + for i in $(seq 1 20); do curl -sf "http://127.0.0.1:$PORT416/pets" >/dev/null 2>&1 && break; sleep 0.1; done + + contains "AT-416" "--min-score 0.99 prints below threshold message" "below --min-score" \ + "'$BIN' mutate --cases '$AT416_CASES' --target \"http://127.0.0.1:$PORT416\" --min-score 0.99 --operator field_drop || true" + kill $SBX416_PID 2>/dev/null || true + + rm -rf "$AT415_CASES" "$AT416_CASES" +else + skip "AT-415" "--min-score 0 no-op" "hurl not installed" + skip "AT-416" "--min-score 0.99 breach" "hurl not installed" +fi + +echo "" + # ------------------------------------------------------- # Summary # -------------------------------------------------------