diff --git a/cmd/mutate.go b/cmd/mutate.go index 5ee0d83..175721a 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -17,11 +17,13 @@ import ( ) var ( - mutateCases string - mutateTarget string - mutateOutput string - mutateOperators string - mutateConcurrency int + mutateCases string + mutateTarget string + mutateOutput string + mutateOperators string + mutateReportFormat string + mutateConcurrency int + mutateOperatorConcurrency int ) var mutateCmd = &cobra.Command{ @@ -48,26 +50,52 @@ Examples: func init() { rootCmd.AddCommand(mutateCmd) - mutateCmd.Flags().StringVar(&mutateCases, "cases", "", "Directory containing index.json and .hurl files (required)") - _ = mutateCmd.MarkFlagRequired("cases") - mutateCmd.Flags().StringVar(&mutateTarget, "target", "", "API base URL, e.g. http://localhost:8080 (required)") - _ = mutateCmd.MarkFlagRequired("target") + mutateCmd.Flags().StringVar(&mutateCases, "cases", "", "Directory containing index.json and .hurl files (required when not using --history)") + mutateCmd.Flags().StringVar(&mutateTarget, "target", "", "API base URL, e.g. http://localhost:8080 (required when not using --history)") mutateCmd.Flags().StringVar(&mutateOutput, "output", "", "Directory to write mutation-report.json (optional)") mutateCmd.Flags().StringVar(&mutateOperators, "operator", "", "Comma-separated operator names to run (default: all 12)") mutateCmd.Flags().String("spec", "", "OpenAPI spec file (optional; passed to LLM in Phase 2)") mutateCmd.Flags().IntVar(&mutateConcurrency, "concurrency", 4, "Number of cases processed concurrently per operator") + mutateCmd.Flags().IntVar(&mutateOperatorConcurrency, "operator-concurrency", 2, "Number of operators to run in parallel") + mutateCmd.Flags().StringVar(&mutateReportFormat, "report-format", "json", `Comma-separated report formats: json,markdown,html,all`) + mutateCmd.Flags().Bool("history", false, "Print mutation score history (does not run mutations; --target not required)") + mutateCmd.Flags().Int("history-limit", 10, "Maximum number of historical runs to display") mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)") mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)") mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix") } func runMutate(cmd *cobra.Command, _ []string) error { + if historyFlag, _ := cmd.Flags().GetBool("history"); historyFlag { + limit, _ := cmd.Flags().GetInt("history-limit") + runs, err := mutation.LoadHistory("", limit) + if err != nil { + return fmt.Errorf("loading history: %w", err) + } + fmt.Fprint(cmd.OutOrStdout(), mutation.RenderHistory(runs)) + return nil + } + + // --cases and --target are required when not using --history + if mutateCases == "" { + return fmt.Errorf("required flag \"cases\" not set") + } + if mutateTarget == "" { + return fmt.Errorf("required flag \"target\" not set") + } + feedbackFlag, _ := cmd.Flags().GetBool("feedback") autoFixFlag, _ := cmd.Flags().GetBool("auto-fix") if autoFixFlag && !feedbackFlag { return fmt.Errorf("--auto-fix requires --feedback") } + // Validate --report-format unconditionally so a typo is always caught early. + reportFormats, err := parseReportFormats(mutateReportFormat) + if err != nil { + return err + } + ops, err := resolveOperators(mutateOperators) if err != nil { return err @@ -77,10 +105,11 @@ func runMutate(cmd *cobra.Command, _ []string) error { fmt.Fprintf(out, "Running %d operator(s) × cases in %s...\n", len(ops), mutateCases) opts := mutation.RunOptions{ - Target: mutateTarget, - CasesDir: mutateCases, - Operators: ops, - Concurrency: mutateConcurrency, + Target: mutateTarget, + CasesDir: mutateCases, + Operators: ops, + Concurrency: mutateConcurrency, + OperatorConcurrency: mutateOperatorConcurrency, } run, err := mutation.Run(opts) @@ -146,11 +175,16 @@ func runMutate(cmd *cobra.Command, _ []string) error { _ = mutation.Persist("", run) if mutateOutput != "" { - if err := mutation.WriteReport(mutateOutput, run); err != nil { + if err := mutation.WriteReport(mutateOutput, run, reportFormats); err != nil { return fmt.Errorf("writing report: %w", err) } - fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", - filepath.Join(mutateOutput, "mutation-report.json")) + extMap := map[string]string{"json": ".json", "markdown": ".md", "html": ".html"} + for _, f := range reportFormats { + if ext, ok := extMap[f]; ok { + fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", + filepath.Join(mutateOutput, "mutation-report"+ext)) + } + } } if run.Survivors > 0 { @@ -222,3 +256,39 @@ func countSuggestions(items []mutation.FeedbackItem) int { } return n } + +// parseReportFormats splits the comma-separated format string, expands "all" +// to ["json", "markdown", "html"], and deduplicates. Returns ["json"] if empty. +// Returns an error for unrecognized format names. +func parseReportFormats(s string) ([]string, error) { + if s == "" { + return []string{"json"}, nil + } + seen := map[string]bool{} + var out []string + for fmtName := range strings.SplitSeq(s, ",") { + fmtName = strings.ToLower(strings.TrimSpace(fmtName)) + if fmtName == "all" { + for _, fn := range []string{"json", "markdown", "html"} { + if !seen[fn] { + seen[fn] = true + out = append(out, fn) + } + } + } else if fmtName != "" { + switch fmtName { + case "json", "markdown", "html": + if !seen[fmtName] { + seen[fmtName] = true + out = append(out, fmtName) + } + default: + return nil, fmt.Errorf("unknown report format %q; valid: json, markdown, html, all", fmtName) + } + } + } + if len(out) == 0 { + return []string{"json"}, nil + } + return out, nil +} diff --git a/internal/mutation/engine.go b/internal/mutation/engine.go index 0742f4c..61ffc2b 100644 --- a/internal/mutation/engine.go +++ b/internal/mutation/engine.go @@ -34,45 +34,56 @@ func Run(opts RunOptions) (MutationRun, error) { return MutationRun{}, fmt.Errorf("no test cases found in %s/index.json", opts.CasesDir) } - proxy, err := NewProxy(opts.Target) - if err != nil { - return MutationRun{}, fmt.Errorf("starting proxy: %w", err) + operators := opts.Operators + if len(operators) == 0 { + operators = Registry() } - defer proxy.Close() var ( mu sync.Mutex results []CaseMutationResult ) - operators := opts.Operators - if len(operators) == 0 { - operators = Registry() - } + opSem := make(chan struct{}, opts.OperatorConcurrency) + g, _ := errgroup.WithContext(context.Background()) for _, op := range operators { - proxy.SetActive(op) - g, _ := errgroup.WithContext(context.Background()) - sem := make(chan struct{}, opts.Concurrency) - - for _, tc := range cases { - g.Go(func() error { - sem <- struct{}{} - defer func() { <-sem }() - survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID) - mu.Lock() - results = append(results, CaseMutationResult{ - CaseID: tc.ID, - Title: tc.Title, - Operator: op.Name(), - Survived: survived, + g.Go(func() error { + opSem <- struct{}{} + defer func() { <-opSem }() + + proxy, err := NewProxy(opts.Target) + if err != nil { + fmt.Fprintf(os.Stderr, "warn: operator %s: proxy start failed: %v\n", op.Name(), err) + return nil // skip this operator; don't abort others + } + defer proxy.Close() + proxy.SetActive(op) + + caseSem := make(chan struct{}, opts.Concurrency) + cg, _ := errgroup.WithContext(context.Background()) + for _, tc := range cases { + cg.Go(func() error { + caseSem <- struct{}{} + defer func() { <-caseSem }() + survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID) + mu.Lock() + results = append(results, CaseMutationResult{ + CaseID: tc.ID, + Title: tc.Title, + Operator: op.Name(), + Survived: survived, + }) + mu.Unlock() + return nil }) - mu.Unlock() - return nil - }) - } - _ = g.Wait() + } + _ = cg.Wait() + return nil + }) } + // goroutines always return nil (proxy failures are logged and skipped) + _ = g.Wait() killed, survivors := 0, 0 for _, r := range results { diff --git a/internal/mutation/engine_test.go b/internal/mutation/engine_test.go index 953b074..126cb30 100644 --- a/internal/mutation/engine_test.go +++ b/internal/mutation/engine_test.go @@ -117,3 +117,57 @@ func TestEngineRun_NoHurlFiles(t *testing.T) { t.Fatalf("expected 1 total run, got %d", run.TotalRuns) } } + +func TestEngineRun_TwoOperatorsParallel(t *testing.T) { + if _, err := exec.LookPath("hurl"); err != nil { + t.Skip("hurl not installed") + } + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"id":1}`)) + })) + defer backend.Close() + + dir := t.TempDir() + buildTestHurlFile(t, dir, "TC-PAR", "GET {{base_url}}/\nHTTP 200\n") + buildTestIndexJSON(t, dir, "TC-PAR", "parallel test") + + opts := mutation.RunOptions{ + Target: backend.URL, + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewFieldDropOperator(), mutation.NewStatusSwap2xxOperator()}, + Concurrency: 1, + OperatorConcurrency: 2, + } + run, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) + } + if run.TotalRuns != 2 { + t.Fatalf("expected 2 total runs (2 operators × 1 case), got %d", run.TotalRuns) + } + ops := map[string]bool{} + for _, r := range run.Results { + ops[r.Operator] = true + } + if !ops["field_drop"] || !ops["status_swap_2xx"] { + t.Error("both operators must appear in results when run with OperatorConcurrency=2") + } +} + +func TestEngineRun_OperatorConcurrencyZeroDefault(t *testing.T) { + dir := t.TempDir() + buildTestIndexJSON(t, dir, "TC-DEF", "defaults test") + opts := mutation.RunOptions{ + Target: "http://127.0.0.1:1", + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewFieldDropOperator()}, + Concurrency: 1, + OperatorConcurrency: 0, // must not panic; defaults to 2 + } + _, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) // missing hurl file is OK (returns killed), no panic expected + } +} diff --git a/internal/mutation/history.go b/internal/mutation/history.go new file mode 100644 index 0000000..c2e599c --- /dev/null +++ b/internal/mutation/history.go @@ -0,0 +1,115 @@ +package mutation + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// LoadHistory reads up to limit most-recent MutationRun files from baseDir. +// If baseDir is empty it defaults to ".caseforge/mutation/runs". +// Returns nil slice (not error) when the directory does not exist yet. +func LoadHistory(baseDir string, limit int) ([]MutationRun, error) { + if baseDir == "" { + baseDir = filepath.Join(".caseforge", "mutation", "runs") + } + entries, err := os.ReadDir(baseDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + // sort descending by filename (timestamp prefix ensures chronological order) + sort.Slice(entries, func(i, j int) bool { + return entries[i].Name() > entries[j].Name() + }) + // filter to .json only before applying limit so non-JSON files don't count + var jsonEntries []os.DirEntry + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".json") { + jsonEntries = append(jsonEntries, e) + } + } + if limit > 0 && len(jsonEntries) > limit { + jsonEntries = jsonEntries[:limit] + } + runs := make([]MutationRun, 0, len(jsonEntries)) + for _, e := range jsonEntries { + data, err := os.ReadFile(filepath.Join(baseDir, e.Name())) + if err != nil { + continue // skip unreadable files + } + var run MutationRun + if err := json.Unmarshal(data, &run); err != nil { + continue // skip corrupt files + } + runs = append(runs, run) + } + return runs, nil +} + +// RenderHistory returns a terminal-friendly trend table. +// Runs must be ordered newest-first (as returned by LoadHistory). +func RenderHistory(runs []MutationRun) string { + if len(runs) == 0 { + return "No mutation history found. Run `caseforge mutate` first.\n" + } + var b strings.Builder + fmt.Fprintf(&b, "Mutation History (last %d runs)\n", len(runs)) + fmt.Fprintf(&b, "%s\n", strings.Repeat("─", 70)) + fmt.Fprintf(&b, " %-26s %5s %10s %9s %4s\n", + "Run", "Score", "Killed", "Survivors", "Δ") + for i, run := range runs { + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + delta := "" + if i+1 < len(runs) { + prev := runs[i+1] + prevPct := 0 + if prev.TotalRuns > 0 { + prevPct = int(prev.MutationScore * 100) + } + diff := pct - prevPct + if diff > 0 { + delta = fmt.Sprintf("↑+%d%%", diff) + } else if diff < 0 { + delta = fmt.Sprintf("↓%d%%", -diff) + } else { + delta = "—" + } + } else { + delta = "—" + } + ts := run.GeneratedAt + if ts == "" { + ts = "(unknown)" + } + fmt.Fprintf(&b, " %-26s %4d%% %4d/%-4d %9d %s\n", + ts, pct, run.Killed, run.TotalRuns, run.Survivors, delta) + } + fmt.Fprintf(&b, "%s\n", strings.Repeat("─", 70)) + // 7-day average + cutoff := time.Now().AddDate(0, 0, -7) + var recent []int + for _, run := range runs { + t, err := time.Parse("2006-01-02T15:04:05Z", run.GeneratedAt) + if err == nil && t.After(cutoff) && run.TotalRuns > 0 { + recent = append(recent, int(run.MutationScore*100)) + } + } + if len(recent) > 0 { + sum := 0 + for _, v := range recent { + sum += v + } + fmt.Fprintf(&b, "Avg score (last 7 days): %d%%\n", sum/len(recent)) + } + return b.String() +} diff --git a/internal/mutation/history_test.go b/internal/mutation/history_test.go new file mode 100644 index 0000000..d227ebd --- /dev/null +++ b/internal/mutation/history_test.go @@ -0,0 +1,136 @@ +package mutation_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func writeRunFile(t *testing.T, dir, name string, run mutation.MutationRun) { + t.Helper() + data, err := json.Marshal(run) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), data, 0644); err != nil { + t.Fatal(err) + } +} + +func TestLoadHistory_Empty(t *testing.T) { + runs, err := mutation.LoadHistory(t.TempDir()+"/nonexistent", 10) + if err != nil { + t.Fatalf("expected nil error for missing dir, got %v", err) + } + if runs != nil { + t.Fatalf("expected nil slice for missing dir, got %v", runs) + } +} + +func TestLoadHistory_ReadsFiles(t *testing.T) { + dir := t.TempDir() + for i := 1; i <= 3; i++ { + writeRunFile(t, dir, fmt.Sprintf("2026-05-0%d.json", i), mutation.MutationRun{ + GeneratedAt: fmt.Sprintf("2026-05-0%dT12:00:00Z", i), + TotalRuns: 10, + Killed: i * 2, + MutationScore: float64(i*2) / 10.0, + }) + } + runs, err := mutation.LoadHistory(dir, 0) + if err != nil { + t.Fatal(err) + } + if len(runs) != 3 { + t.Fatalf("expected 3 runs, got %d", len(runs)) + } + // Newest first: 2026-05-03 should come before 2026-05-01 + if runs[0].GeneratedAt < runs[2].GeneratedAt { + t.Error("runs must be ordered newest-first") + } +} + +func TestLoadHistory_Limit(t *testing.T) { + dir := t.TempDir() + for i := 1; i <= 5; i++ { + writeRunFile(t, dir, fmt.Sprintf("2026-05-%02d.json", i), mutation.MutationRun{ + GeneratedAt: fmt.Sprintf("2026-05-%02dT12:00:00Z", i), + }) + } + runs, err := mutation.LoadHistory(dir, 3) + if err != nil { + t.Fatal(err) + } + if len(runs) != 3 { + t.Fatalf("expected 3 runs with limit=3, got %d", len(runs)) + } +} + +func TestLoadHistory_SkipsCorrupt(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "corrupt.json"), []byte("{bad json"), 0644); err != nil { + t.Fatal(err) + } + writeRunFile(t, dir, "valid.json", mutation.MutationRun{GeneratedAt: "2026-05-10T12:00:00Z"}) + runs, err := mutation.LoadHistory(dir, 0) + if err != nil { + t.Fatal(err) + } + if len(runs) != 1 { + t.Fatalf("expected 1 valid run, got %d", len(runs)) + } +} + +func TestRenderHistory_Empty(t *testing.T) { + out := mutation.RenderHistory(nil) + if !strings.Contains(out, "No mutation history found") { + t.Error("empty history must contain 'No mutation history found'") + } +} + +func TestRenderHistory_Trend(t *testing.T) { + runs := []mutation.MutationRun{ + {GeneratedAt: "2026-05-04T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, + {GeneratedAt: "2026-05-03T12:00:00Z", TotalRuns: 10, Killed: 8, MutationScore: 0.8}, + {GeneratedAt: "2026-05-02T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, + {GeneratedAt: "2026-05-01T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, + } + out := mutation.RenderHistory(runs) + // Second run (80%) vs third (60%) → increase: ↑+20% + if !strings.Contains(out, "↑+20%") { + t.Errorf("expected ↑+20%% delta, got:\n%s", out) + } + // First run (60%) vs second (80%) → decrease: ↓20% (no minus sign) + if !strings.Contains(out, "↓20%") { + t.Errorf("expected ↓20%% delta (no minus sign), got:\n%s", out) + } + if strings.Contains(out, "↓-") { + t.Errorf("delta must not contain redundant minus after ↓, got:\n%s", out) + } + // Third and fourth are equal → show — + if !strings.Contains(out, "—") { + t.Errorf("expected — delta for equal scores, got:\n%s", out) + } + // Last run shows — (no prior) +} + +func TestRenderHistory_7DayAvg(t *testing.T) { + now := time.Now().UTC() + recent := now.AddDate(0, 0, -1).Format("2006-01-02T15:04:05Z") + old := now.AddDate(0, 0, -10).Format("2006-01-02T15:04:05Z") + runs := []mutation.MutationRun{ + {GeneratedAt: recent, TotalRuns: 10, Killed: 8, MutationScore: 0.8}, + {GeneratedAt: old, TotalRuns: 10, Killed: 4, MutationScore: 0.4}, + } + out := mutation.RenderHistory(runs) + // Only the recent run (80%) should be in the 7-day average + if !strings.Contains(out, "Avg score (last 7 days): 80%") { + t.Errorf("expected 7-day avg of 80%%, got:\n%s", out) + } +} diff --git a/internal/mutation/render.go b/internal/mutation/render.go new file mode 100644 index 0000000..f08bad0 --- /dev/null +++ b/internal/mutation/render.go @@ -0,0 +1,135 @@ +// internal/mutation/render.go +package mutation + +import ( + "fmt" + "html" + "sort" + "strings" +) + +// RenderMarkdown returns a Markdown mutation report string. +func RenderMarkdown(run MutationRun) string { + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + var b strings.Builder + fmt.Fprintf(&b, "# Mutation Report — %s\n\n", run.GeneratedAt) + fmt.Fprintf(&b, "Mutation Score: **%d/%d killed (%d%%)** · Survivors: **%d**\n\n", + run.Killed, run.TotalRuns, pct, run.Survivors) + + if len(run.Clusters) > 0 { + b.WriteString("## Survivor Summary (by risk)\n\n") + b.WriteString("| Case | Title | Risk | Survived Operators |\n") + b.WriteString("|------|-------|------|--------------------|\n") + for _, c := range run.Clusters { + fmt.Fprintf(&b, "| %s | %s | %.2f | %s |\n", + c.CaseID, c.Title, c.RiskScore, strings.Join(c.Operators, ", ")) + } + b.WriteString("\n") + } + + if len(run.Feedback) > 0 { + b.WriteString("## Suggested Assertions\n\n") + for _, item := range run.Feedback { + fmt.Fprintf(&b, "### %s — %s\n\n", item.CaseID, item.Title) + for _, a := range item.SuggestedAssertions { + if a.Expected != nil { + fmt.Fprintf(&b, "- `%s` `%s` `%v`\n", a.Target, a.Operator, a.Expected) + } else { + fmt.Fprintf(&b, "- `%s` `%s`\n", a.Target, a.Operator) + } + } + b.WriteString("\n") + } + } + + return b.String() +} + +// RenderHTML returns a self-contained HTML mutation report (no external resources). +func RenderHTML(run MutationRun) string { + // build operator×case result grid + type cell struct{ survived, exists bool } + grid := map[string]map[string]cell{} + caseTitle := map[string]string{} + for _, r := range run.Results { + if grid[r.Operator] == nil { + grid[r.Operator] = map[string]cell{} + } + grid[r.Operator][r.CaseID] = cell{survived: r.Survived, exists: true} + caseTitle[r.CaseID] = r.Title + } + caseIDs := make([]string, 0, len(caseTitle)) + for id := range caseTitle { + caseIDs = append(caseIDs, id) + } + sort.Strings(caseIDs) + + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + + var b strings.Builder + b.WriteString("\n") + b.WriteString("Mutation Report\n") + b.WriteString(` +`) + b.WriteString("

Mutation Report

\n") + b.WriteString("
") + fmt.Fprintf(&b, "
Score
%d%%
", pct) + fmt.Fprintf(&b, "
Killed/Total
%d/%d
", run.Killed, run.TotalRuns) + fmt.Fprintf(&b, "
Survivors
%d
", run.Survivors) + fmt.Fprintf(&b, "
Generated
%s
", html.EscapeString(run.GeneratedAt)) + b.WriteString("
\n") + + b.WriteString("

Operator × Case Heatmap

\n\n") + for _, id := range caseIDs { + fmt.Fprintf(&b, "", html.EscapeString(caseTitle[id]), html.EscapeString(id)) + } + b.WriteString("\n") + for _, op := range run.Operators { + fmt.Fprintf(&b, "", html.EscapeString(op)) + for _, id := range caseIDs { + c := grid[op][id] + if !c.exists { + b.WriteString("") + } else if c.survived { + b.WriteString("") + } else { + b.WriteString("") + } + } + b.WriteString("\n") + } + b.WriteString("
Operator%s
%s—✗✓
\n") + + if len(run.Clusters) > 0 { + b.WriteString("

Survivors by Risk

\n\n\n") + for _, c := range run.Clusters { + fmt.Fprintf(&b, "\n", + html.EscapeString(c.CaseID), + html.EscapeString(c.Title), + c.RiskScore*100, + html.EscapeString(strings.Join(c.Operators, ", "))) + } + b.WriteString("
CaseTitleRiskOperators
%s%s%.0f%%%s
\n") + } + + b.WriteString("\n") + return b.String() +} diff --git a/internal/mutation/render_test.go b/internal/mutation/render_test.go new file mode 100644 index 0000000..12385fc --- /dev/null +++ b/internal/mutation/render_test.go @@ -0,0 +1,114 @@ +// internal/mutation/render_test.go +package mutation_test + +import ( + "strings" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func sampleRunWithClusters() mutation.MutationRun { + run := sampleRun() // defined in report_test.go — same package mutation_test + run.Clusters = mutation.ClusterSurvivors(run) + return run +} + +func sampleRunWithFeedback() mutation.MutationRun { + run := sampleRunWithClusters() + run.Feedback = []mutation.FeedbackItem{ + { + CaseID: "TC-0001", + Title: "GET /pets", + RiskScore: 0.5, + Diagnosis: "1 operator survived", + SuggestedAssertions: []mutation.SuggestedAssertion{ + {Target: "jsonpath $.id", Operator: "exists"}, + {Target: "status_code", Operator: "eq", Expected: float64(200)}, + }, + }, + } + return run +} + +func TestRenderMarkdown_NoSurvivors(t *testing.T) { + run := mutation.MutationRun{ + TotalRuns: 4, + Killed: 4, + Survivors: 0, + MutationScore: 1.0, + GeneratedAt: "2026-05-10T12:00:00Z", + } + md := mutation.RenderMarkdown(run) + if !strings.Contains(md, "100%") { + t.Error("must show 100% when no survivors") + } + if strings.Contains(md, "## Survivor Summary") { + t.Error("must not show Survivor Summary when no survivors") + } +} + +func TestRenderMarkdown_WithSurvivors(t *testing.T) { + run := sampleRunWithClusters() + md := mutation.RenderMarkdown(run) + if !strings.Contains(md, "## Survivor Summary") { + t.Error("must contain Survivor Summary section") + } + if !strings.Contains(md, "TC-0001") { + t.Error("TC-0001 must appear in Survivor Summary") + } + if !strings.Contains(md, "TC-0003") { + t.Error("TC-0003 must appear in Survivor Summary") + } +} + +func TestRenderMarkdown_WithFeedback(t *testing.T) { + run := sampleRunWithFeedback() + md := mutation.RenderMarkdown(run) + if !strings.Contains(md, "## Suggested Assertions") { + t.Error("must contain Suggested Assertions section when Feedback is non-empty") + } + if !strings.Contains(md, "jsonpath $.id") { + t.Error("must list the suggested assertion target") + } +} + +func TestRenderHTML_ContainsHeatmap(t *testing.T) { + run := sampleRunWithClusters() + out := mutation.RenderHTML(run) + if !strings.Contains(out, "/dev/null 2>&1; then + AT406_CASES=$(mktemp -d) + AT406_REPORT=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT406_CASES" >/dev/null 2>&1 + + run "AT-406" "mutate --report-format markdown creates mutation-report.md" \ + "PORT406=\$(random_port) + '$BIN' sandbox --spec '$WORKDIR/petstore.yaml' --port \$PORT406 >/dev/null 2>&1 & + SBX406_PID=\$! + for i in \$(seq 1 20); do curl -sf http://127.0.0.1:\$PORT406/pets >/dev/null 2>&1 && break; sleep 0.1; done + '$BIN' mutate --cases '$AT406_CASES' --target \"http://127.0.0.1:\$PORT406\" --output '$AT406_REPORT' --report-format markdown --operator field_drop || true + kill \$SBX406_PID 2>/dev/null || true + test -f '$AT406_REPORT/mutation-report.md'" + + contains "AT-407" "mutation-report.md contains Mutation Score" "Mutation Score" \ + "cat '$AT406_REPORT/mutation-report.md'" + + rm -rf "$AT406_CASES" "$AT406_REPORT" +else + skip "AT-406" "mutate markdown report" "hurl not installed" + skip "AT-407" "mutation-report.md content" "hurl not installed" +fi + +# AT-408 – AT-409: --history flag +# ───────────────────────────────────────────────────────────────────────────── + +echo "# AT-408 – AT-409: history flag" + +# AT-408: --history flag registered in help +contains "AT-408" "history flag in help" "history" "'$BIN' mutate --help" + +# AT-409: after one mutate run, --history prints "Mutation History" +if command -v hurl >/dev/null 2>&1; then + AT409_CASES=$(mktemp -d) + AT409_RUNS=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT409_CASES" >/dev/null 2>&1 + + PORT409=$(random_port) + "$BIN" sandbox --spec "$WORKDIR/petstore.yaml" --port $PORT409 >/dev/null 2>&1 & + SBX409_PID=$! + for i in $(seq 1 20); do curl -sf "http://127.0.0.1:$PORT409/pets" >/dev/null 2>&1 && break; sleep 0.1; done + + # Run mutate to create a history entry (persist writes to .caseforge/mutation/runs in cwd) + AT409_WORKDIR=$(mktemp -d) + (cd "$AT409_WORKDIR" && "$BIN" mutate --cases "$AT409_CASES" \ + --target "http://127.0.0.1:$PORT409" --operator field_drop || true) >/dev/null 2>&1 + + kill $SBX409_PID 2>/dev/null || true + + contains "AT-409" "--history prints Mutation History after a run" "Mutation History" \ + "(cd '$AT409_WORKDIR' && '$BIN' mutate --history)" + + rm -rf "$AT409_CASES" "$AT409_WORKDIR" +else + skip "AT-409" "--history after mutate run" "hurl not installed" +fi + echo "" # -------------------------------------------------------