Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 31 additions & 7 deletions cmd/mutate.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@ import (
)

var (
mutateCases string
mutateTarget string
mutateOutput string
mutateOperators string
mutateReportFormat string
mutateConcurrency int
mutateCases string
mutateTarget string
mutateOutput string
mutateOperators string
mutateReportFormat string
mutateConcurrency int
mutateOperatorConcurrency int
)

Expand All @@ -38,7 +38,7 @@ Requires hurl on PATH. Test cases must be previously generated with 'caseforge g

Exit codes:
0 — run complete, no survivors
6 — one or more mutations survived
6 — one or more mutations survived, or --min-score threshold breached

Examples:
caseforge mutate --cases ./cases --target http://localhost:8080
Expand All @@ -60,6 +60,8 @@ func init() {
mutateCmd.Flags().StringVar(&mutateReportFormat, "report-format", "json", `Comma-separated report formats: json,markdown,html,all`)
mutateCmd.Flags().Bool("history", false, "Print mutation score history (does not run mutations; --target not required)")
mutateCmd.Flags().Int("history-limit", 10, "Maximum number of historical runs to display")
mutateCmd.Flags().Float64("min-score", 0,
"Per-operation minimum mutation score (0.0–1.0); exit 6 if any operation scores below this")
mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)")
mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)")
mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix")
Expand Down Expand Up @@ -96,6 +98,11 @@ func runMutate(cmd *cobra.Command, _ []string) error {
return err
}

minScore, _ := cmd.Flags().GetFloat64("min-score")
if minScore < 0 || minScore > 1.0 {
return fmt.Errorf("--min-score must be between 0.0 and 1.0, got %.2f", minScore)
}

ops, err := resolveOperators(mutateOperators)
if err != nil {
return err
Expand Down Expand Up @@ -187,6 +194,23 @@ func runMutate(cmd *cobra.Command, _ []string) error {
}
}

if minScore > 0 {
var failing []mutation.OperationScore
for _, op := range run.OperationScores {
if op.MutationScore < minScore {
failing = append(failing, op)
}
}
if len(failing) > 0 {
color.New(color.FgRed).Fprintf(out,
"\n✗ %d operation(s) below --min-score %.0f%%:\n", len(failing), minScore*100)
for _, op := range failing {
fmt.Fprintf(out, " %-52s %3.0f%%\n", op.Operation, op.MutationScore*100)
}
os.Exit(ExitPartialSuccess)
}
}

if run.Survivors > 0 {
os.Exit(ExitPartialSuccess)
}
Expand Down
44 changes: 27 additions & 17 deletions internal/mutation/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ type indexFile struct {
TestCases []struct {
ID string `json:"id"`
Title string `json:"title"`
Source struct {
SpecPath string `json:"spec_path"`
} `json:"source"`
} `json:"test_cases"`
}

Expand Down Expand Up @@ -69,10 +72,11 @@ func Run(opts RunOptions) (MutationRun, error) {
survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID)
mu.Lock()
results = append(results, CaseMutationResult{
CaseID: tc.ID,
Title: tc.Title,
Operator: op.Name(),
Survived: survived,
CaseID: tc.ID,
Title: tc.Title,
Operation: tc.Operation,
Operator: op.Name(),
Survived: survived,
})
mu.Unlock()
return nil
Expand Down Expand Up @@ -106,16 +110,17 @@ func Run(opts RunOptions) (MutationRun, error) {
}

return MutationRun{
Target: opts.Target,
CasesDir: opts.CasesDir,
Operators: opNames,
TotalCases: len(cases),
TotalRuns: total,
Killed: killed,
Survivors: survivors,
MutationScore: score,
Results: results,
GeneratedAt: time.Now().UTC().Format("2006-01-02T15:04:05Z"),
Target: opts.Target,
CasesDir: opts.CasesDir,
Operators: opNames,
TotalCases: len(cases),
TotalRuns: total,
Killed: killed,
Survivors: survivors,
MutationScore: score,
Results: results,
OperationScores: ComputeOperationScores(results),
GeneratedAt: time.Now().UTC().Format("2006-01-02T15:04:05Z"),
}, nil
}

Expand Down Expand Up @@ -152,8 +157,9 @@ func runOnce(proxyAddr, casesDir, caseID string) bool {
}

type caseRef struct {
ID string
Title string
ID string
Title string
Operation string // source.spec_path; "(unknown)" if absent
}

func loadCases(casesDir string) ([]caseRef, error) {
Expand All @@ -167,7 +173,11 @@ func loadCases(casesDir string) ([]caseRef, error) {
}
refs := make([]caseRef, len(idx.TestCases))
for i, tc := range idx.TestCases {
refs[i] = caseRef{ID: tc.ID, Title: tc.Title}
op := tc.Source.SpecPath
if op == "" {
op = "(unknown)"
}
refs[i] = caseRef{ID: tc.ID, Title: tc.Title, Operation: op}
}
return refs, nil
}
56 changes: 56 additions & 0 deletions internal/mutation/render.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ func RenderMarkdown(run MutationRun) string {
fmt.Fprintf(&b, "Mutation Score: **%d/%d killed (%d%%)** · Survivors: **%d**\n\n",
run.Killed, run.TotalRuns, pct, run.Survivors)

b.WriteString(renderOperationTable(run))

if len(run.Clusters) > 0 {
b.WriteString("## Survivor Summary (by risk)\n\n")
b.WriteString("| Case | Title | Risk | Survived Operators |\n")
Expand Down Expand Up @@ -48,6 +50,57 @@ func RenderMarkdown(run MutationRun) string {
return b.String()
}

// renderOperationTable returns the per-operation Markdown table, or "" if no scores.
func renderOperationTable(run MutationRun) string {
if len(run.OperationScores) == 0 {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, "## Per-Operation Mutation Score\n\n")
fmt.Fprintf(&b, "| Operation | Score | Killed | Survivors |\n")
fmt.Fprintf(&b, "|-----------|-------|--------|-----------|\n")
for _, op := range run.OperationScores {
pct := int(op.MutationScore * 100)
badge := ""
switch {
case op.MutationScore == 1.0:
badge = " ✓"
case op.MutationScore < 0.7:
badge = " ⚠"
}
fmt.Fprintf(&b, "| %s | %d%%%s | %d/%d | %d |\n",
op.Operation, pct, badge, op.Killed, op.TotalRuns, op.Survivors)
}
b.WriteString("\n")
return b.String()
}

// renderOperationTableHTML returns the per-operation HTML table, or "" if no scores.
func renderOperationTableHTML(run MutationRun) string {
if len(run.OperationScores) == 0 {
return ""
}
var b strings.Builder
b.WriteString("<h2>Per-Operation Mutation Score</h2>\n")
b.WriteString("<table>\n<tr><th>Operation</th><th>Score</th><th>Killed</th><th>Survivors</th></tr>\n")
for _, op := range run.OperationScores {
pct := int(op.MutationScore * 100)
rowClass := ""
switch {
case op.MutationScore == 1.0:
rowClass = " class=\"op-high\""
case op.MutationScore < 0.7:
rowClass = " class=\"op-low\""
}
fmt.Fprintf(&b, "<tr%s><td>%s</td><td>%d%%</td><td>%d/%d</td><td>%d</td></tr>\n",
rowClass,
html.EscapeString(op.Operation),
pct, op.Killed, op.TotalRuns, op.Survivors)
}
b.WriteString("</table>\n")
return b.String()
}

// RenderHTML returns a self-contained HTML mutation report (no external resources).
func RenderHTML(run MutationRun) string {
// build operator×case result grid
Expand Down Expand Up @@ -86,6 +139,8 @@ td{padding:4px 8px;border:1px solid #e5e7eb;white-space:nowrap}
.killed{background:#dcfce7;color:#166534;text-align:center}
.survived{background:#fee2e2;color:#991b1b;text-align:center}
.na{background:#f9fafb;color:#9ca3af;text-align:center}
.op-low{background:#fef2f2}
.op-high{background:#f0fdf4}
h2{margin-top:2rem}
</style></head><body>
`)
Expand Down Expand Up @@ -130,6 +185,7 @@ h2{margin-top:2rem}
b.WriteString("</table>\n")
}

b.WriteString(renderOperationTableHTML(run))
b.WriteString("</body></html>\n")
return b.String()
}
54 changes: 54 additions & 0 deletions internal/mutation/render_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,57 @@ func TestRenderHTML_NoSurvivors(t *testing.T) {
t.Error("must not show Survivors by Risk table when no clusters")
}
}

func TestRenderMarkdown_OperationTable(t *testing.T) {
run := sampleRun()
run.OperationScores = []mutation.OperationScore{
{Operation: "GET /pets", TotalRuns: 2, Killed: 1, Survivors: 1, MutationScore: 0.5},
{Operation: "POST /pets", TotalRuns: 2, Killed: 2, Survivors: 0, MutationScore: 1.0},
}
out := mutation.RenderMarkdown(run)
if !strings.Contains(out, "Per-Operation Mutation Score") {
t.Error("expected per-operation section header")
}
if !strings.Contains(out, "GET /pets") {
t.Error("expected GET /pets row")
}
if !strings.Contains(out, "50%") {
t.Error("expected 50% score for GET /pets")
}
if !strings.Contains(out, "⚠") {
t.Error("expected ⚠ badge for score < 70%")
}
if !strings.Contains(out, "✓") {
t.Error("expected ✓ badge for 100% score")
}
}

func TestRenderMarkdown_NoOperationScores(t *testing.T) {
run := sampleRun()
run.OperationScores = nil
out := mutation.RenderMarkdown(run)
if strings.Contains(out, "Per-Operation") {
t.Error("per-operation section must be absent when OperationScores is nil")
}
}

func TestRenderHTML_OperationTable(t *testing.T) {
run := sampleRun()
run.OperationScores = []mutation.OperationScore{
{Operation: "GET /pets", TotalRuns: 2, Killed: 1, Survivors: 1, MutationScore: 0.5},
{Operation: "DELETE /pets/{id}", TotalRuns: 2, Killed: 2, Survivors: 0, MutationScore: 1.0},
}
out := mutation.RenderHTML(run)
if !strings.Contains(out, "Per-Operation") {
t.Error("expected Per-Operation section in HTML")
}
if !strings.Contains(out, "GET /pets") {
t.Error("expected GET /pets row in HTML")
}
if !strings.Contains(out, "fef2f2") {
t.Error("expected light-red background for score < 70%")
}
if !strings.Contains(out, "f0fdf4") {
t.Error("expected light-green background for 100% score")
}
}
47 changes: 47 additions & 0 deletions internal/mutation/score.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package mutation

import "sort"

// ComputeOperationScores groups CaseMutationResults by operation and returns
// per-operation scores sorted weakest-first (score ascending, then alphabetically).
// Results with empty Operation are grouped under "(unknown)".
func ComputeOperationScores(results []CaseMutationResult) []OperationScore {
type stats struct{ killed, survivors int }
byOp := map[string]*stats{}
for _, r := range results {
op := r.Operation
if op == "" {
op = "(unknown)"
}
if byOp[op] == nil {
byOp[op] = &stats{}
}
if r.Survived {
byOp[op].survivors++
} else {
byOp[op].killed++
}
}
scores := make([]OperationScore, 0, len(byOp))
for op, s := range byOp {
total := s.killed + s.survivors
score := 0.0
if total > 0 {
score = float64(s.killed) / float64(total)
}
scores = append(scores, OperationScore{
Operation: op,
TotalRuns: total,
Killed: s.killed,
Survivors: s.survivors,
MutationScore: score,
})
}
sort.Slice(scores, func(i, j int) bool {
if scores[i].MutationScore != scores[j].MutationScore {
return scores[i].MutationScore < scores[j].MutationScore
}
return scores[i].Operation < scores[j].Operation
})
return scores
}
Loading
Loading