From c9e7db684dc4285e3a9a1aa30231e35984547e09 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 17:32:29 +0800 Subject: [PATCH 01/15] feat(mutation): add OperatorConcurrency to RunOptions (default 2) --- internal/mutation/types.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/mutation/types.go b/internal/mutation/types.go index 5579260..04947eb 100644 --- a/internal/mutation/types.go +++ b/internal/mutation/types.go @@ -63,17 +63,21 @@ type MutationRun struct { // RunOptions configures a mutation run. type RunOptions struct { - Target string - CasesDir string - OutputDir string - Operators []Operator - Concurrency int + Target string + CasesDir string + OutputDir string + Operators []Operator + Concurrency int // cases per operator (default 4) + OperatorConcurrency int // operators in parallel (default 2) } func (o *RunOptions) setDefaults() { if o.Concurrency <= 0 { o.Concurrency = 4 } + if o.OperatorConcurrency <= 0 { + o.OperatorConcurrency = 2 + } } // persistDir returns the directory for run JSON files. From 0547eed9b0ae40bed28c17efd9096997de1cb63f Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 17:45:54 +0800 Subject: [PATCH 02/15] =?UTF-8?q?feat(mutation):=20parallelize=20operators?= =?UTF-8?q?=20=E2=80=94=20one=20proxy=20per=20operator,=20bounded=20by=20O?= =?UTF-8?q?peratorConcurrency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/mutation/engine.go | 68 +++++++++++++++++++------------- internal/mutation/engine_test.go | 54 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/internal/mutation/engine.go b/internal/mutation/engine.go index 0742f4c..e3c0d8b 100644 --- a/internal/mutation/engine.go +++ b/internal/mutation/engine.go @@ -34,44 +34,56 @@ func Run(opts RunOptions) (MutationRun, error) { return MutationRun{}, fmt.Errorf("no test cases found in %s/index.json", opts.CasesDir) } - proxy, err := NewProxy(opts.Target) - if err != nil { - return MutationRun{}, fmt.Errorf("starting proxy: %w", err) + operators := opts.Operators + if len(operators) == 0 { + operators = Registry() } - defer proxy.Close() var ( mu sync.Mutex results []CaseMutationResult ) - operators := opts.Operators - if len(operators) == 0 { - operators = Registry() - } + opSem := make(chan struct{}, opts.OperatorConcurrency) + g, _ := errgroup.WithContext(context.Background()) for _, op := range operators { - proxy.SetActive(op) - g, _ := errgroup.WithContext(context.Background()) - sem := make(chan struct{}, opts.Concurrency) - - for _, tc := range cases { - g.Go(func() error { - sem <- struct{}{} - defer func() { <-sem }() - survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID) - mu.Lock() - results = append(results, CaseMutationResult{ - CaseID: tc.ID, - Title: tc.Title, - Operator: op.Name(), - Survived: survived, + g.Go(func() error { + opSem <- struct{}{} + defer func() { <-opSem }() + + proxy, err := NewProxy(opts.Target) + if err != nil { + fmt.Fprintf(os.Stderr, "warn: operator %s: proxy start failed: %v\n", op.Name(), err) + return nil // skip this operator; don't abort others + } + defer proxy.Close() + proxy.SetActive(op) + + caseSem := make(chan struct{}, opts.Concurrency) + cg, _ := errgroup.WithContext(context.Background()) + for _, tc := range cases { + cg.Go(func() error { + caseSem <- struct{}{} + defer func() { <-caseSem }() + survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID) + mu.Lock() + results = append(results, CaseMutationResult{ + CaseID: tc.ID, + Title: tc.Title, + Operator: op.Name(), + Survived: survived, + }) + mu.Unlock() + return nil }) - mu.Unlock() - return nil - }) - } - _ = g.Wait() + } + _ = cg.Wait() + return nil + }) + } + if err := g.Wait(); err != nil { + return MutationRun{}, err } killed, survivors := 0, 0 diff --git a/internal/mutation/engine_test.go b/internal/mutation/engine_test.go index 953b074..126cb30 100644 --- a/internal/mutation/engine_test.go +++ b/internal/mutation/engine_test.go @@ -117,3 +117,57 @@ func TestEngineRun_NoHurlFiles(t *testing.T) { t.Fatalf("expected 1 total run, got %d", run.TotalRuns) } } + +func TestEngineRun_TwoOperatorsParallel(t *testing.T) { + if _, err := exec.LookPath("hurl"); err != nil { + t.Skip("hurl not installed") + } + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"id":1}`)) + })) + defer backend.Close() + + dir := t.TempDir() + buildTestHurlFile(t, dir, "TC-PAR", "GET {{base_url}}/\nHTTP 200\n") + buildTestIndexJSON(t, dir, "TC-PAR", "parallel test") + + opts := mutation.RunOptions{ + Target: backend.URL, + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewFieldDropOperator(), mutation.NewStatusSwap2xxOperator()}, + Concurrency: 1, + OperatorConcurrency: 2, + } + run, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) + } + if run.TotalRuns != 2 { + t.Fatalf("expected 2 total runs (2 operators × 1 case), got %d", run.TotalRuns) + } + ops := map[string]bool{} + for _, r := range run.Results { + ops[r.Operator] = true + } + if !ops["field_drop"] || !ops["status_swap_2xx"] { + t.Error("both operators must appear in results when run with OperatorConcurrency=2") + } +} + +func TestEngineRun_OperatorConcurrencyZeroDefault(t *testing.T) { + dir := t.TempDir() + buildTestIndexJSON(t, dir, "TC-DEF", "defaults test") + opts := mutation.RunOptions{ + Target: "http://127.0.0.1:1", + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewFieldDropOperator()}, + Concurrency: 1, + OperatorConcurrency: 0, // must not panic; defaults to 2 + } + _, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) // missing hurl file is OK (returns killed), no panic expected + } +} From 9606b397ce86a7421ba52cc0e779a65c56dc9874 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 17:46:06 +0800 Subject: [PATCH 03/15] =?UTF-8?q?refactor(mutation):=20clarify=20g.Wait()?= =?UTF-8?q?=20comment=20=E2=80=94=20goroutines=20never=20return=20non-nil?= =?UTF-8?q?=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/mutation/engine.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/mutation/engine.go b/internal/mutation/engine.go index e3c0d8b..61ffc2b 100644 --- a/internal/mutation/engine.go +++ b/internal/mutation/engine.go @@ -82,9 +82,8 @@ func Run(opts RunOptions) (MutationRun, error) { return nil }) } - if err := g.Wait(); err != nil { - return MutationRun{}, err - } + // goroutines always return nil (proxy failures are logged and skipped) + _ = g.Wait() killed, survivors := 0, 0 for _, r := range results { From bcf464e0dfaae72d22b8ab05162d8777c1fcac1b Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 17:46:51 +0800 Subject: [PATCH 04/15] feat(mutation): add --operator-concurrency flag (default 2) --- cmd/mutate.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/mutate.go b/cmd/mutate.go index 5ee0d83..7183512 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -17,11 +17,12 @@ import ( ) var ( - mutateCases string - mutateTarget string - mutateOutput string - mutateOperators string - mutateConcurrency int + mutateCases string + mutateTarget string + mutateOutput string + mutateOperators string + mutateConcurrency int + mutateOperatorConcurrency int ) var mutateCmd = &cobra.Command{ @@ -56,6 +57,7 @@ func init() { mutateCmd.Flags().StringVar(&mutateOperators, "operator", "", "Comma-separated operator names to run (default: all 12)") mutateCmd.Flags().String("spec", "", "OpenAPI spec file (optional; passed to LLM in Phase 2)") mutateCmd.Flags().IntVar(&mutateConcurrency, "concurrency", 4, "Number of cases processed concurrently per operator") + mutateCmd.Flags().IntVar(&mutateOperatorConcurrency, "operator-concurrency", 2, "Number of operators to run in parallel") mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)") mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)") mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix") @@ -77,10 +79,11 @@ func runMutate(cmd *cobra.Command, _ []string) error { fmt.Fprintf(out, "Running %d operator(s) × cases in %s...\n", len(ops), mutateCases) opts := mutation.RunOptions{ - Target: mutateTarget, - CasesDir: mutateCases, - Operators: ops, - Concurrency: mutateConcurrency, + Target: mutateTarget, + CasesDir: mutateCases, + Operators: ops, + Concurrency: mutateConcurrency, + OperatorConcurrency: mutateOperatorConcurrency, } run, err := mutation.Run(opts) From 3b55bae9d73030dac6c9a1c168fe93a0afdba514 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 17:57:13 +0800 Subject: [PATCH 05/15] =?UTF-8?q?test(mutation):=20AT-404=20=E2=80=94=20op?= =?UTF-8?q?erator-concurrency=20flag=20acceptance=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/acceptance.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 26d4a1d..02a8080 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1658,6 +1658,9 @@ else skip "AT-403" "mutate full end-to-end" "hurl not installed" fi +# AT-404: --operator-concurrency flag registered +contains "AT-404" "operator-concurrency flag in help" "operator-concurrency" "'$BIN' mutate --help" + echo "" # ------------------------------------------------------- From f84f1bc121a89c49f4c3d4a6975b0d35cdb2bba9 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 17:59:38 +0800 Subject: [PATCH 06/15] =?UTF-8?q?feat(mutation):=20RenderMarkdown=20and=20?= =?UTF-8?q?RenderHTML=20=E2=80=94=20Markdown=20and=20HTML=20report=20rende?= =?UTF-8?q?rers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/mutation/render.go | 135 +++++++++++++++++++++++++++++++ internal/mutation/render_test.go | 114 ++++++++++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 internal/mutation/render.go create mode 100644 internal/mutation/render_test.go diff --git a/internal/mutation/render.go b/internal/mutation/render.go new file mode 100644 index 0000000..f08bad0 --- /dev/null +++ b/internal/mutation/render.go @@ -0,0 +1,135 @@ +// internal/mutation/render.go +package mutation + +import ( + "fmt" + "html" + "sort" + "strings" +) + +// RenderMarkdown returns a Markdown mutation report string. +func RenderMarkdown(run MutationRun) string { + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + var b strings.Builder + fmt.Fprintf(&b, "# Mutation Report — %s\n\n", run.GeneratedAt) + fmt.Fprintf(&b, "Mutation Score: **%d/%d killed (%d%%)** · Survivors: **%d**\n\n", + run.Killed, run.TotalRuns, pct, run.Survivors) + + if len(run.Clusters) > 0 { + b.WriteString("## Survivor Summary (by risk)\n\n") + b.WriteString("| Case | Title | Risk | Survived Operators |\n") + b.WriteString("|------|-------|------|--------------------|\n") + for _, c := range run.Clusters { + fmt.Fprintf(&b, "| %s | %s | %.2f | %s |\n", + c.CaseID, c.Title, c.RiskScore, strings.Join(c.Operators, ", ")) + } + b.WriteString("\n") + } + + if len(run.Feedback) > 0 { + b.WriteString("## Suggested Assertions\n\n") + for _, item := range run.Feedback { + fmt.Fprintf(&b, "### %s — %s\n\n", item.CaseID, item.Title) + for _, a := range item.SuggestedAssertions { + if a.Expected != nil { + fmt.Fprintf(&b, "- `%s` `%s` `%v`\n", a.Target, a.Operator, a.Expected) + } else { + fmt.Fprintf(&b, "- `%s` `%s`\n", a.Target, a.Operator) + } + } + b.WriteString("\n") + } + } + + return b.String() +} + +// RenderHTML returns a self-contained HTML mutation report (no external resources). +func RenderHTML(run MutationRun) string { + // build operator×case result grid + type cell struct{ survived, exists bool } + grid := map[string]map[string]cell{} + caseTitle := map[string]string{} + for _, r := range run.Results { + if grid[r.Operator] == nil { + grid[r.Operator] = map[string]cell{} + } + grid[r.Operator][r.CaseID] = cell{survived: r.Survived, exists: true} + caseTitle[r.CaseID] = r.Title + } + caseIDs := make([]string, 0, len(caseTitle)) + for id := range caseTitle { + caseIDs = append(caseIDs, id) + } + sort.Strings(caseIDs) + + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + + var b strings.Builder + b.WriteString("\n") + b.WriteString("Mutation Report\n") + b.WriteString(` +`) + b.WriteString("

Mutation Report

\n") + b.WriteString("
") + fmt.Fprintf(&b, "
Score
%d%%
", pct) + fmt.Fprintf(&b, "
Killed/Total
%d/%d
", run.Killed, run.TotalRuns) + fmt.Fprintf(&b, "
Survivors
%d
", run.Survivors) + fmt.Fprintf(&b, "
Generated
%s
", html.EscapeString(run.GeneratedAt)) + b.WriteString("
\n") + + b.WriteString("

Operator × Case Heatmap

\n\n") + for _, id := range caseIDs { + fmt.Fprintf(&b, "", html.EscapeString(caseTitle[id]), html.EscapeString(id)) + } + b.WriteString("\n") + for _, op := range run.Operators { + fmt.Fprintf(&b, "", html.EscapeString(op)) + for _, id := range caseIDs { + c := grid[op][id] + if !c.exists { + b.WriteString("") + } else if c.survived { + b.WriteString("") + } else { + b.WriteString("") + } + } + b.WriteString("\n") + } + b.WriteString("
Operator%s
%s—✗✓
\n") + + if len(run.Clusters) > 0 { + b.WriteString("

Survivors by Risk

\n\n\n") + for _, c := range run.Clusters { + fmt.Fprintf(&b, "\n", + html.EscapeString(c.CaseID), + html.EscapeString(c.Title), + c.RiskScore*100, + html.EscapeString(strings.Join(c.Operators, ", "))) + } + b.WriteString("
CaseTitleRiskOperators
%s%s%.0f%%%s
\n") + } + + b.WriteString("\n") + return b.String() +} diff --git a/internal/mutation/render_test.go b/internal/mutation/render_test.go new file mode 100644 index 0000000..12385fc --- /dev/null +++ b/internal/mutation/render_test.go @@ -0,0 +1,114 @@ +// internal/mutation/render_test.go +package mutation_test + +import ( + "strings" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func sampleRunWithClusters() mutation.MutationRun { + run := sampleRun() // defined in report_test.go — same package mutation_test + run.Clusters = mutation.ClusterSurvivors(run) + return run +} + +func sampleRunWithFeedback() mutation.MutationRun { + run := sampleRunWithClusters() + run.Feedback = []mutation.FeedbackItem{ + { + CaseID: "TC-0001", + Title: "GET /pets", + RiskScore: 0.5, + Diagnosis: "1 operator survived", + SuggestedAssertions: []mutation.SuggestedAssertion{ + {Target: "jsonpath $.id", Operator: "exists"}, + {Target: "status_code", Operator: "eq", Expected: float64(200)}, + }, + }, + } + return run +} + +func TestRenderMarkdown_NoSurvivors(t *testing.T) { + run := mutation.MutationRun{ + TotalRuns: 4, + Killed: 4, + Survivors: 0, + MutationScore: 1.0, + GeneratedAt: "2026-05-10T12:00:00Z", + } + md := mutation.RenderMarkdown(run) + if !strings.Contains(md, "100%") { + t.Error("must show 100% when no survivors") + } + if strings.Contains(md, "## Survivor Summary") { + t.Error("must not show Survivor Summary when no survivors") + } +} + +func TestRenderMarkdown_WithSurvivors(t *testing.T) { + run := sampleRunWithClusters() + md := mutation.RenderMarkdown(run) + if !strings.Contains(md, "## Survivor Summary") { + t.Error("must contain Survivor Summary section") + } + if !strings.Contains(md, "TC-0001") { + t.Error("TC-0001 must appear in Survivor Summary") + } + if !strings.Contains(md, "TC-0003") { + t.Error("TC-0003 must appear in Survivor Summary") + } +} + +func TestRenderMarkdown_WithFeedback(t *testing.T) { + run := sampleRunWithFeedback() + md := mutation.RenderMarkdown(run) + if !strings.Contains(md, "## Suggested Assertions") { + t.Error("must contain Suggested Assertions section when Feedback is non-empty") + } + if !strings.Contains(md, "jsonpath $.id") { + t.Error("must list the suggested assertion target") + } +} + +func TestRenderHTML_ContainsHeatmap(t *testing.T) { + run := sampleRunWithClusters() + out := mutation.RenderHTML(run) + if !strings.Contains(out, " Date: Sun, 10 May 2026 18:03:05 +0800 Subject: [PATCH 07/15] =?UTF-8?q?feat(mutation):=20WriteReport=20accepts?= =?UTF-8?q?=20format=20list=20=E2=80=94=20json,=20markdown,=20html?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mutate.go | 2 +- internal/mutation/report.go | 31 +++++++++++++++++---- internal/mutation/report_test.go | 48 +++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/cmd/mutate.go b/cmd/mutate.go index 7183512..b0c8c2c 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -149,7 +149,7 @@ func runMutate(cmd *cobra.Command, _ []string) error { _ = mutation.Persist("", run) if mutateOutput != "" { - if err := mutation.WriteReport(mutateOutput, run); err != nil { + if err := mutation.WriteReport(mutateOutput, run, []string{"json"}); err != nil { return fmt.Errorf("writing report: %w", err) } fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", diff --git a/internal/mutation/report.go b/internal/mutation/report.go index 7b48f05..0bbdb0d 100644 --- a/internal/mutation/report.go +++ b/internal/mutation/report.go @@ -29,16 +29,35 @@ func TextSummary(run MutationRun) string { return sb.String() } -// WriteReport writes mutation-report.json to outputDir. -func WriteReport(outputDir string, run MutationRun) error { +// WriteReport writes mutation report files in the requested formats to outputDir. +// formats: slice containing any of "json", "markdown", "html". +func WriteReport(outputDir string, run MutationRun, formats []string) error { if err := os.MkdirAll(outputDir, 0755); err != nil { return err } - data, err := json.MarshalIndent(run, "", " ") - if err != nil { - return err + for _, f := range formats { + switch f { + case "json": + data, err := json.MarshalIndent(run, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(outputDir, "mutation-report.json"), data, 0644); err != nil { + return err + } + case "markdown": + if err := os.WriteFile(filepath.Join(outputDir, "mutation-report.md"), + []byte(RenderMarkdown(run)), 0644); err != nil { + return err + } + case "html": + if err := os.WriteFile(filepath.Join(outputDir, "mutation-report.html"), + []byte(RenderHTML(run)), 0644); err != nil { + return err + } + } } - return os.WriteFile(filepath.Join(outputDir, "mutation-report.json"), data, 0644) + return nil } // Persist writes a timestamped run file to baseDir. diff --git a/internal/mutation/report_test.go b/internal/mutation/report_test.go index 923f364..8fcf95a 100644 --- a/internal/mutation/report_test.go +++ b/internal/mutation/report_test.go @@ -47,7 +47,7 @@ func TestTextSummary(t *testing.T) { func TestWriteReport(t *testing.T) { dir := t.TempDir() run := sampleRun() - if err := mutation.WriteReport(dir, run); err != nil { + if err := mutation.WriteReport(dir, run, []string{"json"}); err != nil { t.Fatal(err) } data, err := os.ReadFile(filepath.Join(dir, "mutation-report.json")) @@ -63,6 +63,52 @@ func TestWriteReport(t *testing.T) { } } +func TestWriteReport_Markdown(t *testing.T) { + dir := t.TempDir() + run := sampleRun() + run.Clusters = mutation.ClusterSurvivors(run) + if err := mutation.WriteReport(dir, run, []string{"markdown"}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(dir, "mutation-report.md")) + if err != nil { + t.Fatalf("mutation-report.md not created: %v", err) + } + if !strings.Contains(string(data), "Mutation Score") { + t.Error("mutation-report.md must contain 'Mutation Score'") + } +} + +func TestWriteReport_HTML(t *testing.T) { + dir := t.TempDir() + run := sampleRun() + run.Clusters = mutation.ClusterSurvivors(run) + if err := mutation.WriteReport(dir, run, []string{"html"}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(dir, "mutation-report.html")) + if err != nil { + t.Fatalf("mutation-report.html not created: %v", err) + } + if !strings.Contains(string(data), " Date: Sun, 10 May 2026 18:07:16 +0800 Subject: [PATCH 08/15] fix(mutation): WriteReport expands "all" to json+markdown+html Add TestWriteReport_AllKeyword to verify the "all" shorthand. --- internal/mutation/report.go | 10 +++++++++- internal/mutation/report_test.go | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/internal/mutation/report.go b/internal/mutation/report.go index 0bbdb0d..c914de2 100644 --- a/internal/mutation/report.go +++ b/internal/mutation/report.go @@ -30,12 +30,20 @@ func TextSummary(run MutationRun) string { } // WriteReport writes mutation report files in the requested formats to outputDir. -// formats: slice containing any of "json", "markdown", "html". +// formats: slice of "json", "markdown", "html"; "all" expands to all three. func WriteReport(outputDir string, run MutationRun, formats []string) error { if err := os.MkdirAll(outputDir, 0755); err != nil { return err } + expanded := make([]string, 0, len(formats)) for _, f := range formats { + if f == "all" { + expanded = append(expanded, "json", "markdown", "html") + } else { + expanded = append(expanded, f) + } + } + for _, f := range expanded { switch f { case "json": data, err := json.MarshalIndent(run, "", " ") diff --git a/internal/mutation/report_test.go b/internal/mutation/report_test.go index 8fcf95a..e07a137 100644 --- a/internal/mutation/report_test.go +++ b/internal/mutation/report_test.go @@ -109,6 +109,20 @@ func TestWriteReport_All(t *testing.T) { } } +func TestWriteReport_AllKeyword(t *testing.T) { + dir := t.TempDir() + run := sampleRun() + run.Clusters = mutation.ClusterSurvivors(run) + if err := mutation.WriteReport(dir, run, []string{"all"}); err != nil { + t.Fatal(err) + } + for _, name := range []string{"mutation-report.json", "mutation-report.md", "mutation-report.html"} { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Errorf("\"all\" keyword: expected %s to exist: %v", name, err) + } + } +} + func TestPersist(t *testing.T) { dir := t.TempDir() run := sampleRun() From e56878765da8f530459109f053eb3bca1f2309f5 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 18:09:02 +0800 Subject: [PATCH 09/15] feat(mutation): add --report-format flag (json,markdown,html,all) --- cmd/mutate.go | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/cmd/mutate.go b/cmd/mutate.go index b0c8c2c..449ad67 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -21,6 +21,7 @@ var ( mutateTarget string mutateOutput string mutateOperators string + mutateReportFormat string mutateConcurrency int mutateOperatorConcurrency int ) @@ -58,6 +59,7 @@ func init() { mutateCmd.Flags().String("spec", "", "OpenAPI spec file (optional; passed to LLM in Phase 2)") mutateCmd.Flags().IntVar(&mutateConcurrency, "concurrency", 4, "Number of cases processed concurrently per operator") mutateCmd.Flags().IntVar(&mutateOperatorConcurrency, "operator-concurrency", 2, "Number of operators to run in parallel") + mutateCmd.Flags().StringVar(&mutateReportFormat, "report-format", "json", `Comma-separated report formats: json,markdown,html,all`) mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)") mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)") mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix") @@ -149,11 +151,17 @@ func runMutate(cmd *cobra.Command, _ []string) error { _ = mutation.Persist("", run) if mutateOutput != "" { - if err := mutation.WriteReport(mutateOutput, run, []string{"json"}); err != nil { + formats := parseReportFormats(mutateReportFormat) + if err := mutation.WriteReport(mutateOutput, run, formats); err != nil { return fmt.Errorf("writing report: %w", err) } - fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", - filepath.Join(mutateOutput, "mutation-report.json")) + extMap := map[string]string{"json": ".json", "markdown": ".md", "html": ".html"} + for _, f := range formats { + if ext, ok := extMap[f]; ok { + fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", + filepath.Join(mutateOutput, "mutation-report"+ext)) + } + } } if run.Survivors > 0 { @@ -225,3 +233,31 @@ func countSuggestions(items []mutation.FeedbackItem) int { } return n } + +// parseReportFormats splits the comma-separated format string, expands "all" +// to ["json", "markdown", "html"], and deduplicates. Returns ["json"] if empty. +func parseReportFormats(s string) []string { + if s == "" { + return []string{"json"} + } + seen := map[string]bool{} + var out []string + for fmtName := range strings.SplitSeq(s, ",") { + fmtName = strings.ToLower(strings.TrimSpace(fmtName)) + if fmtName == "all" { + for _, fn := range []string{"json", "markdown", "html"} { + if !seen[fn] { + seen[fn] = true + out = append(out, fn) + } + } + } else if fmtName != "" && !seen[fmtName] { + seen[fmtName] = true + out = append(out, fmtName) + } + } + if len(out) == 0 { + return []string{"json"} + } + return out +} From d72828dc6d938930c67f52ed28df3ab9fbd969fd Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 18:13:57 +0800 Subject: [PATCH 10/15] =?UTF-8?q?test(mutation):=20acceptance=20tests=20AT?= =?UTF-8?q?-405=E2=80=93AT-407=20=E2=80=94=20report-format=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/acceptance.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 02a8080..81aac44 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1661,6 +1661,40 @@ fi # AT-404: --operator-concurrency flag registered contains "AT-404" "operator-concurrency flag in help" "operator-concurrency" "'$BIN' mutate --help" +# AT-405 – AT-407: --report-format flag +# ───────────────────────────────────────────────────────────────────────────── + +echo "# AT-405 – AT-407: report-format flag" + +# AT-405: --report-format flag registered +contains "AT-405" "report-format flag in help" "report-format" "'$BIN' mutate --help" + +# AT-406 & AT-407: markdown report written and contains expected content +if command -v hurl >/dev/null 2>&1; then + AT406_CASES=$(mktemp -d) + AT406_REPORT=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT406_CASES" >/dev/null 2>&1 + + run "AT-406" "mutate --report-format markdown creates mutation-report.md" \ + "PORT406=\$(random_port) + '$BIN' sandbox --spec '$WORKDIR/petstore.yaml' --port \$PORT406 >/dev/null 2>&1 & + SBX406_PID=\$! + for i in \$(seq 1 20); do curl -sf http://127.0.0.1:\$PORT406/pets >/dev/null 2>&1 && break; sleep 0.1; done + '$BIN' mutate --cases '$AT406_CASES' --target \"http://127.0.0.1:\$PORT406\" --output '$AT406_REPORT' --report-format markdown --operator field_drop || true + kill \$SBX406_PID 2>/dev/null || true + test -f '$AT406_REPORT/mutation-report.md'" + + contains "AT-407" "mutation-report.md contains Mutation Score" "Mutation Score" \ + "cat '$AT406_REPORT/mutation-report.md'" + + rm -rf "$AT406_CASES" "$AT406_REPORT" +else + skip "AT-406" "mutate markdown report" "hurl not installed" + skip "AT-407" "mutation-report.md content" "hurl not installed" +fi + echo "" # ------------------------------------------------------- From 0e615fa7cdd95c1136c4ac808fb1b93f1d341f43 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 18:15:19 +0800 Subject: [PATCH 11/15] =?UTF-8?q?feat(mutation):=20LoadHistory=20and=20Ren?= =?UTF-8?q?derHistory=20=E2=80=94=20history=20trend=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/mutation/history.go | 111 ++++++++++++++++++++++++++ internal/mutation/history_test.go | 128 ++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 internal/mutation/history.go create mode 100644 internal/mutation/history_test.go diff --git a/internal/mutation/history.go b/internal/mutation/history.go new file mode 100644 index 0000000..3491e27 --- /dev/null +++ b/internal/mutation/history.go @@ -0,0 +1,111 @@ +package mutation + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// LoadHistory reads up to limit most-recent MutationRun files from baseDir. +// If baseDir is empty it defaults to ".caseforge/mutation/runs". +// Returns nil slice (not error) when the directory does not exist yet. +func LoadHistory(baseDir string, limit int) ([]MutationRun, error) { + if baseDir == "" { + baseDir = filepath.Join(".caseforge", "mutation", "runs") + } + entries, err := os.ReadDir(baseDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + // sort descending by filename (timestamp prefix ensures chronological order) + sort.Slice(entries, func(i, j int) bool { + return entries[i].Name() > entries[j].Name() + }) + if limit > 0 && len(entries) > limit { + entries = entries[:limit] + } + runs := make([]MutationRun, 0, len(entries)) + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".json") { + continue + } + data, err := os.ReadFile(filepath.Join(baseDir, e.Name())) + if err != nil { + continue // skip unreadable files + } + var run MutationRun + if err := json.Unmarshal(data, &run); err != nil { + continue // skip corrupt files + } + runs = append(runs, run) + } + return runs, nil +} + +// RenderHistory returns a terminal-friendly trend table. +// Runs must be ordered newest-first (as returned by LoadHistory). +func RenderHistory(runs []MutationRun) string { + if len(runs) == 0 { + return "No mutation history found. Run `caseforge mutate` first.\n" + } + var b strings.Builder + fmt.Fprintf(&b, "Mutation History (last %d runs)\n", len(runs)) + fmt.Fprintf(&b, "%s\n", strings.Repeat("─", 70)) + fmt.Fprintf(&b, " %-26s %5s %10s %9s %4s\n", + "Run", "Score", "Killed", "Survivors", "Δ") + for i, run := range runs { + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + delta := "" + if i+1 < len(runs) { + prev := runs[i+1] + prevPct := 0 + if prev.TotalRuns > 0 { + prevPct = int(prev.MutationScore * 100) + } + diff := pct - prevPct + if diff > 0 { + delta = fmt.Sprintf("↑+%d%%", diff) + } else if diff < 0 { + delta = fmt.Sprintf("↓%d%%", diff) + } else { + delta = "—" + } + } else { + delta = "—" + } + ts := run.GeneratedAt + if ts == "" { + ts = "(unknown)" + } + fmt.Fprintf(&b, " %-26s %4d%% %4d/%-4d %9d %s\n", + ts, pct, run.Killed, run.TotalRuns, run.Survivors, delta) + } + fmt.Fprintf(&b, "%s\n", strings.Repeat("─", 70)) + // 7-day average + cutoff := time.Now().AddDate(0, 0, -7) + var recent []int + for _, run := range runs { + t, err := time.Parse("2006-01-02T15:04:05Z", run.GeneratedAt) + if err == nil && t.After(cutoff) && run.TotalRuns > 0 { + recent = append(recent, int(run.MutationScore*100)) + } + } + if len(recent) > 0 { + sum := 0 + for _, v := range recent { + sum += v + } + fmt.Fprintf(&b, "Avg score (last 7 days): %d%%\n", sum/len(recent)) + } + return b.String() +} diff --git a/internal/mutation/history_test.go b/internal/mutation/history_test.go new file mode 100644 index 0000000..e6e7e4f --- /dev/null +++ b/internal/mutation/history_test.go @@ -0,0 +1,128 @@ +package mutation_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func writeRunFile(t *testing.T, dir, name string, run mutation.MutationRun) { + t.Helper() + data, err := json.Marshal(run) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), data, 0644); err != nil { + t.Fatal(err) + } +} + +func TestLoadHistory_Empty(t *testing.T) { + runs, err := mutation.LoadHistory(t.TempDir()+"/nonexistent", 10) + if err != nil { + t.Fatalf("expected nil error for missing dir, got %v", err) + } + if runs != nil { + t.Fatalf("expected nil slice for missing dir, got %v", runs) + } +} + +func TestLoadHistory_ReadsFiles(t *testing.T) { + dir := t.TempDir() + for i := 1; i <= 3; i++ { + writeRunFile(t, dir, fmt.Sprintf("2026-05-0%d.json", i), mutation.MutationRun{ + GeneratedAt: fmt.Sprintf("2026-05-0%dT12:00:00Z", i), + TotalRuns: 10, + Killed: i * 2, + MutationScore: float64(i*2) / 10.0, + }) + } + runs, err := mutation.LoadHistory(dir, 0) + if err != nil { + t.Fatal(err) + } + if len(runs) != 3 { + t.Fatalf("expected 3 runs, got %d", len(runs)) + } + // Newest first: 2026-05-03 should come before 2026-05-01 + if runs[0].GeneratedAt < runs[2].GeneratedAt { + t.Error("runs must be ordered newest-first") + } +} + +func TestLoadHistory_Limit(t *testing.T) { + dir := t.TempDir() + for i := 1; i <= 5; i++ { + writeRunFile(t, dir, fmt.Sprintf("2026-05-%02d.json", i), mutation.MutationRun{ + GeneratedAt: fmt.Sprintf("2026-05-%02dT12:00:00Z", i), + }) + } + runs, err := mutation.LoadHistory(dir, 3) + if err != nil { + t.Fatal(err) + } + if len(runs) != 3 { + t.Fatalf("expected 3 runs with limit=3, got %d", len(runs)) + } +} + +func TestLoadHistory_SkipsCorrupt(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "corrupt.json"), []byte("{bad json"), 0644); err != nil { + t.Fatal(err) + } + writeRunFile(t, dir, "valid.json", mutation.MutationRun{GeneratedAt: "2026-05-10T12:00:00Z"}) + runs, err := mutation.LoadHistory(dir, 0) + if err != nil { + t.Fatal(err) + } + if len(runs) != 1 { + t.Fatalf("expected 1 valid run, got %d", len(runs)) + } +} + +func TestRenderHistory_Empty(t *testing.T) { + out := mutation.RenderHistory(nil) + if !strings.Contains(out, "No mutation history found") { + t.Error("empty history must contain 'No mutation history found'") + } +} + +func TestRenderHistory_Trend(t *testing.T) { + runs := []mutation.MutationRun{ + {GeneratedAt: "2026-05-03T12:00:00Z", TotalRuns: 10, Killed: 8, MutationScore: 0.8}, + {GeneratedAt: "2026-05-02T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, + {GeneratedAt: "2026-05-01T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, + } + out := mutation.RenderHistory(runs) + // First run (newest) should show ↑+20% compared to second + if !strings.Contains(out, "↑+20%") { + t.Errorf("expected ↑+20%% delta, got:\n%s", out) + } + // Second and third are equal → show — + if !strings.Contains(out, "—") { + t.Errorf("expected — delta for equal scores, got:\n%s", out) + } + // Last run shows — (no prior) +} + +func TestRenderHistory_7DayAvg(t *testing.T) { + now := time.Now().UTC() + recent := now.AddDate(0, 0, -1).Format("2006-01-02T15:04:05Z") + old := now.AddDate(0, 0, -10).Format("2006-01-02T15:04:05Z") + runs := []mutation.MutationRun{ + {GeneratedAt: recent, TotalRuns: 10, Killed: 8, MutationScore: 0.8}, + {GeneratedAt: old, TotalRuns: 10, Killed: 4, MutationScore: 0.4}, + } + out := mutation.RenderHistory(runs) + // Only the recent run (80%) should be in the 7-day average + if !strings.Contains(out, "Avg score (last 7 days): 80%") { + t.Errorf("expected 7-day avg of 80%%, got:\n%s", out) + } +} From 426e05d1a8111e1f1dd705bcfd4d367d65ce6daf Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 18:17:40 +0800 Subject: [PATCH 12/15] feat(mutation): add --history and --history-limit flags --- cmd/mutate.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cmd/mutate.go b/cmd/mutate.go index 449ad67..db81019 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -50,22 +50,40 @@ Examples: func init() { rootCmd.AddCommand(mutateCmd) - mutateCmd.Flags().StringVar(&mutateCases, "cases", "", "Directory containing index.json and .hurl files (required)") - _ = mutateCmd.MarkFlagRequired("cases") - mutateCmd.Flags().StringVar(&mutateTarget, "target", "", "API base URL, e.g. http://localhost:8080 (required)") - _ = mutateCmd.MarkFlagRequired("target") + mutateCmd.Flags().StringVar(&mutateCases, "cases", "", "Directory containing index.json and .hurl files (required when not using --history)") + mutateCmd.Flags().StringVar(&mutateTarget, "target", "", "API base URL, e.g. http://localhost:8080 (required when not using --history)") mutateCmd.Flags().StringVar(&mutateOutput, "output", "", "Directory to write mutation-report.json (optional)") mutateCmd.Flags().StringVar(&mutateOperators, "operator", "", "Comma-separated operator names to run (default: all 12)") mutateCmd.Flags().String("spec", "", "OpenAPI spec file (optional; passed to LLM in Phase 2)") mutateCmd.Flags().IntVar(&mutateConcurrency, "concurrency", 4, "Number of cases processed concurrently per operator") mutateCmd.Flags().IntVar(&mutateOperatorConcurrency, "operator-concurrency", 2, "Number of operators to run in parallel") mutateCmd.Flags().StringVar(&mutateReportFormat, "report-format", "json", `Comma-separated report formats: json,markdown,html,all`) + mutateCmd.Flags().Bool("history", false, "Print mutation score history (does not run mutations; --target not required)") + mutateCmd.Flags().Int("history-limit", 10, "Maximum number of historical runs to display") mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)") mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)") mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix") } func runMutate(cmd *cobra.Command, _ []string) error { + if historyFlag, _ := cmd.Flags().GetBool("history"); historyFlag { + limit, _ := cmd.Flags().GetInt("history-limit") + runs, err := mutation.LoadHistory("", limit) + if err != nil { + return fmt.Errorf("loading history: %w", err) + } + fmt.Fprint(cmd.OutOrStdout(), mutation.RenderHistory(runs)) + return nil + } + + // --cases and --target are required when not using --history + if mutateCases == "" { + return fmt.Errorf("required flag \"cases\" not set") + } + if mutateTarget == "" { + return fmt.Errorf("required flag \"target\" not set") + } + feedbackFlag, _ := cmd.Flags().GetBool("feedback") autoFixFlag, _ := cmd.Flags().GetBool("auto-fix") if autoFixFlag && !feedbackFlag { From ea9aeffcbd986e1d8807311be8f14e18637cb56a Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 18:22:55 +0800 Subject: [PATCH 13/15] =?UTF-8?q?test(mutation):=20acceptance=20tests=20AT?= =?UTF-8?q?-408=E2=80=93AT-409=20=E2=80=94=20history=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/acceptance.sh | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 81aac44..62285fe 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1695,6 +1695,42 @@ else skip "AT-407" "mutation-report.md content" "hurl not installed" fi +# AT-408 – AT-409: --history flag +# ───────────────────────────────────────────────────────────────────────────── + +echo "# AT-408 – AT-409: history flag" + +# AT-408: --history flag registered in help +contains "AT-408" "history flag in help" "history" "'$BIN' mutate --help" + +# AT-409: after one mutate run, --history prints "Mutation History" +if command -v hurl >/dev/null 2>&1; then + AT409_CASES=$(mktemp -d) + AT409_RUNS=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT409_CASES" >/dev/null 2>&1 + + PORT409=$(random_port) + "$BIN" sandbox --spec "$WORKDIR/petstore.yaml" --port $PORT409 >/dev/null 2>&1 & + SBX409_PID=$! + for i in $(seq 1 20); do curl -sf "http://127.0.0.1:$PORT409/pets" >/dev/null 2>&1 && break; sleep 0.1; done + + # Run mutate to create a history entry (persist writes to .caseforge/mutation/runs in cwd) + AT409_WORKDIR=$(mktemp -d) + (cd "$AT409_WORKDIR" && "$BIN" mutate --cases "$AT409_CASES" \ + --target "http://127.0.0.1:$PORT409" --operator field_drop || true) >/dev/null 2>&1 + + kill $SBX409_PID 2>/dev/null || true + + contains "AT-409" "--history prints Mutation History after a run" "Mutation History" \ + "(cd '$AT409_WORKDIR' && '$BIN' mutate --history)" + + rm -rf "$AT409_CASES" "$AT409_WORKDIR" +else + skip "AT-409" "--history after mutate run" "hurl not installed" +fi + echo "" # ------------------------------------------------------- From 28db811d7c18b2b5ddf5c5ce72d813443f7c225f Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 18:27:24 +0800 Subject: [PATCH 14/15] fix(mutation): LoadHistory limit before JSON filter; reject unknown report formats early --- cmd/mutate.go | 37 ++++++++++++++++++++++++++---------- internal/mutation/history.go | 16 ++++++++++------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/cmd/mutate.go b/cmd/mutate.go index db81019..5fd1d1e 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -90,6 +90,16 @@ func runMutate(cmd *cobra.Command, _ []string) error { return fmt.Errorf("--auto-fix requires --feedback") } + // Validate --report-format early so a bad value fails before the run starts. + var reportFormats []string + if mutateOutput != "" { + var fmtErr error + reportFormats, fmtErr = parseReportFormats(mutateReportFormat) + if fmtErr != nil { + return fmtErr + } + } + ops, err := resolveOperators(mutateOperators) if err != nil { return err @@ -169,12 +179,11 @@ func runMutate(cmd *cobra.Command, _ []string) error { _ = mutation.Persist("", run) if mutateOutput != "" { - formats := parseReportFormats(mutateReportFormat) - if err := mutation.WriteReport(mutateOutput, run, formats); err != nil { + if err := mutation.WriteReport(mutateOutput, run, reportFormats); err != nil { return fmt.Errorf("writing report: %w", err) } extMap := map[string]string{"json": ".json", "markdown": ".md", "html": ".html"} - for _, f := range formats { + for _, f := range reportFormats { if ext, ok := extMap[f]; ok { fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", filepath.Join(mutateOutput, "mutation-report"+ext)) @@ -254,9 +263,10 @@ func countSuggestions(items []mutation.FeedbackItem) int { // parseReportFormats splits the comma-separated format string, expands "all" // to ["json", "markdown", "html"], and deduplicates. Returns ["json"] if empty. -func parseReportFormats(s string) []string { +// Returns an error for unrecognized format names. +func parseReportFormats(s string) ([]string, error) { if s == "" { - return []string{"json"} + return []string{"json"}, nil } seen := map[string]bool{} var out []string @@ -269,13 +279,20 @@ func parseReportFormats(s string) []string { out = append(out, fn) } } - } else if fmtName != "" && !seen[fmtName] { - seen[fmtName] = true - out = append(out, fmtName) + } else if fmtName != "" { + switch fmtName { + case "json", "markdown", "html": + if !seen[fmtName] { + seen[fmtName] = true + out = append(out, fmtName) + } + default: + return nil, fmt.Errorf("unknown report format %q; valid: json, markdown, html, all", fmtName) + } } } if len(out) == 0 { - return []string{"json"} + return []string{"json"}, nil } - return out + return out, nil } diff --git a/internal/mutation/history.go b/internal/mutation/history.go index 3491e27..0bfde1d 100644 --- a/internal/mutation/history.go +++ b/internal/mutation/history.go @@ -28,14 +28,18 @@ func LoadHistory(baseDir string, limit int) ([]MutationRun, error) { sort.Slice(entries, func(i, j int) bool { return entries[i].Name() > entries[j].Name() }) - if limit > 0 && len(entries) > limit { - entries = entries[:limit] - } - runs := make([]MutationRun, 0, len(entries)) + // filter to .json only before applying limit so non-JSON files don't count + var jsonEntries []os.DirEntry for _, e := range entries { - if !strings.HasSuffix(e.Name(), ".json") { - continue + if strings.HasSuffix(e.Name(), ".json") { + jsonEntries = append(jsonEntries, e) } + } + if limit > 0 && len(jsonEntries) > limit { + jsonEntries = jsonEntries[:limit] + } + runs := make([]MutationRun, 0, len(jsonEntries)) + for _, e := range jsonEntries { data, err := os.ReadFile(filepath.Join(baseDir, e.Name())) if err != nil { continue // skip unreadable files From 605c5493abbdc235f627bc3b5d5f5c92fc9b0a98 Mon Sep 17 00:00:00 2001 From: yuchou87 Date: Sun, 10 May 2026 20:28:21 +0800 Subject: [PATCH 15/15] =?UTF-8?q?fix(mutation):=20validate=20report=20form?= =?UTF-8?q?at=20unconditionally,=20guard=20WriteReport=20default=20case,?= =?UTF-8?q?=20fix=20=E2=86=93=20delta=20sign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mutate.go | 12 ++++-------- internal/mutation/history.go | 2 +- internal/mutation/history_test.go | 12 ++++++++++-- internal/mutation/report.go | 2 ++ internal/mutation/report_test.go | 12 ++++++++++++ 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/cmd/mutate.go b/cmd/mutate.go index 5fd1d1e..175721a 100644 --- a/cmd/mutate.go +++ b/cmd/mutate.go @@ -90,14 +90,10 @@ func runMutate(cmd *cobra.Command, _ []string) error { return fmt.Errorf("--auto-fix requires --feedback") } - // Validate --report-format early so a bad value fails before the run starts. - var reportFormats []string - if mutateOutput != "" { - var fmtErr error - reportFormats, fmtErr = parseReportFormats(mutateReportFormat) - if fmtErr != nil { - return fmtErr - } + // Validate --report-format unconditionally so a typo is always caught early. + reportFormats, err := parseReportFormats(mutateReportFormat) + if err != nil { + return err } ops, err := resolveOperators(mutateOperators) diff --git a/internal/mutation/history.go b/internal/mutation/history.go index 0bfde1d..c2e599c 100644 --- a/internal/mutation/history.go +++ b/internal/mutation/history.go @@ -80,7 +80,7 @@ func RenderHistory(runs []MutationRun) string { if diff > 0 { delta = fmt.Sprintf("↑+%d%%", diff) } else if diff < 0 { - delta = fmt.Sprintf("↓%d%%", diff) + delta = fmt.Sprintf("↓%d%%", -diff) } else { delta = "—" } diff --git a/internal/mutation/history_test.go b/internal/mutation/history_test.go index e6e7e4f..d227ebd 100644 --- a/internal/mutation/history_test.go +++ b/internal/mutation/history_test.go @@ -96,16 +96,24 @@ func TestRenderHistory_Empty(t *testing.T) { func TestRenderHistory_Trend(t *testing.T) { runs := []mutation.MutationRun{ + {GeneratedAt: "2026-05-04T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, {GeneratedAt: "2026-05-03T12:00:00Z", TotalRuns: 10, Killed: 8, MutationScore: 0.8}, {GeneratedAt: "2026-05-02T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, {GeneratedAt: "2026-05-01T12:00:00Z", TotalRuns: 10, Killed: 6, MutationScore: 0.6}, } out := mutation.RenderHistory(runs) - // First run (newest) should show ↑+20% compared to second + // Second run (80%) vs third (60%) → increase: ↑+20% if !strings.Contains(out, "↑+20%") { t.Errorf("expected ↑+20%% delta, got:\n%s", out) } - // Second and third are equal → show — + // First run (60%) vs second (80%) → decrease: ↓20% (no minus sign) + if !strings.Contains(out, "↓20%") { + t.Errorf("expected ↓20%% delta (no minus sign), got:\n%s", out) + } + if strings.Contains(out, "↓-") { + t.Errorf("delta must not contain redundant minus after ↓, got:\n%s", out) + } + // Third and fourth are equal → show — if !strings.Contains(out, "—") { t.Errorf("expected — delta for equal scores, got:\n%s", out) } diff --git a/internal/mutation/report.go b/internal/mutation/report.go index c914de2..500da73 100644 --- a/internal/mutation/report.go +++ b/internal/mutation/report.go @@ -63,6 +63,8 @@ func WriteReport(outputDir string, run MutationRun, formats []string) error { []byte(RenderHTML(run)), 0644); err != nil { return err } + default: + return fmt.Errorf("unknown report format %q", f) } } return nil diff --git a/internal/mutation/report_test.go b/internal/mutation/report_test.go index e07a137..6c5f8fb 100644 --- a/internal/mutation/report_test.go +++ b/internal/mutation/report_test.go @@ -123,6 +123,18 @@ func TestWriteReport_AllKeyword(t *testing.T) { } } +func TestWriteReport_UnknownFormat(t *testing.T) { + dir := t.TempDir() + run := sampleRun() + err := mutation.WriteReport(dir, run, []string{"pdf"}) + if err == nil { + t.Fatal("expected error for unknown format, got nil") + } + if !strings.Contains(err.Error(), "pdf") { + t.Errorf("error should name the bad format, got: %v", err) + } +} + func TestPersist(t *testing.T) { dir := t.TempDir() run := sampleRun()