diff --git a/README.md b/README.md index f6a17e1..9b8cf7c 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ caseforge lint --spec openapi.yaml | Command | Description | |---------|-------------| +| `mutate` | Run HTTP boundary mutations via a reverse proxy to find weak test assertions | | `rbt` | Risk-based testing: assess which operations are at risk from recent git changes | | `rbt index` | Auto-generate `caseforge-map.yaml` by analysing source code | | `explore` | Dynamically probe a live API and infer implicit validation rules | @@ -337,6 +338,42 @@ via `--data-pool` to seed realistic field values into generated chain probes. --format string terminal | json (default: terminal) ``` +### `caseforge mutate` + +Run HTTP boundary mutations through a reverse proxy between hurl and your API. For each operator × test case combination, the proxy alters the response before hurl evaluates assertions. Cases where hurl still passes are **survivors** — mutations your assertions failed to catch. + +Requires hurl on PATH and test cases previously generated with `caseforge gen`. + +``` +--cases string Directory containing index.json and .hurl files (required) +--target string API base URL, e.g. http://localhost:8080 (required) +--output string Directory to write mutation-report.json (optional) +--operator string Comma-separated operator names to run (default: all 12) +--concurrency int Cases processed concurrently per operator (default: 4) +--spec string OpenAPI spec file (passed to LLM for context with --feedback) +--feedback Run LLM analysis on survivors and suggest stronger assertions +--auto-fix Patch index.json with suggested assertions (requires --feedback) +--yes Skip confirmation prompt for --auto-fix +``` + +**12 operators:** `field_drop`, `field_type_swap`, `array_to_null`, `null_to_array`, `status_swap_2xx`, `error_inflation`, `pagination_off_by_one`, `empty_result_injection`, `content_type_swap`, `header_drop`, `date_format_swap`, `numeric_precision_loss` + +Exit codes: `0` — no survivors; `6` — one or more mutations survived. + +Each run is persisted to `.caseforge/mutation/runs/.json`. + +```bash +# Run all 12 operators, write JSON report +caseforge mutate --cases ./cases --target http://localhost:8080 --output ./reports + +# Run a specific operator only +caseforge mutate --cases ./cases --target http://localhost:8080 --operator field_drop + +# Phase 2: LLM feedback + auto-fix (requires provider in .caseforge.yaml) +caseforge mutate --cases ./cases --target http://localhost:8080 \ + --feedback --auto-fix --yes +``` + ### `caseforge sandbox` Start a local HTTP mock server that serves realistic responses generated from an OpenAPI spec. Stop with Ctrl-C (SIGINT/SIGTERM triggers graceful shutdown). diff --git a/cmd/mutate.go b/cmd/mutate.go new file mode 100644 index 0000000..5ee0d83 --- /dev/null +++ b/cmd/mutate.go @@ -0,0 +1,224 @@ +// cmd/mutate.go +package cmd + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/testmind-hq/caseforge/internal/config" + "github.com/testmind-hq/caseforge/internal/llm" + "github.com/testmind-hq/caseforge/internal/mutation" +) + +var ( + mutateCases string + mutateTarget string + mutateOutput string + mutateOperators string + mutateConcurrency int +) + +var mutateCmd = &cobra.Command{ + Use: "mutate", + Short: "Run HTTP boundary mutations to find weak test assertions", + Long: `Mutate starts a local reverse proxy between hurl and your API. +For each mutation operator × test case, the proxy alters the response +before hurl evaluates assertions. Cases where hurl still passes are +"survivors" — mutations your assertions failed to catch. + +Requires hurl on PATH. Test cases must be previously generated with 'caseforge gen'. + +Exit codes: + 0 — run complete, no survivors + 6 — one or more mutations survived + +Examples: + caseforge mutate --cases ./cases --target http://localhost:8080 + caseforge mutate --cases ./cases --target http://localhost:8080 --output ./reports + caseforge mutate --cases ./cases --target http://localhost:8080 --operator field_drop,status_swap_2xx`, + RunE: runMutate, + SilenceUsage: true, +} + +func init() { + rootCmd.AddCommand(mutateCmd) + mutateCmd.Flags().StringVar(&mutateCases, "cases", "", "Directory containing index.json and .hurl files (required)") + _ = mutateCmd.MarkFlagRequired("cases") + mutateCmd.Flags().StringVar(&mutateTarget, "target", "", "API base URL, e.g. http://localhost:8080 (required)") + _ = mutateCmd.MarkFlagRequired("target") + mutateCmd.Flags().StringVar(&mutateOutput, "output", "", "Directory to write mutation-report.json (optional)") + mutateCmd.Flags().StringVar(&mutateOperators, "operator", "", "Comma-separated operator names to run (default: all 12)") + mutateCmd.Flags().String("spec", "", "OpenAPI spec file (optional; passed to LLM in Phase 2)") + mutateCmd.Flags().IntVar(&mutateConcurrency, "concurrency", 4, "Number of cases processed concurrently per operator") + mutateCmd.Flags().Bool("feedback", false, "Run LLM feedback analysis on survivors (requires LLM provider in .caseforge.yaml)") + mutateCmd.Flags().Bool("auto-fix", false, "Patch index.json with suggested assertions (requires --feedback)") + mutateCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --auto-fix") +} + +func runMutate(cmd *cobra.Command, _ []string) error { + feedbackFlag, _ := cmd.Flags().GetBool("feedback") + autoFixFlag, _ := cmd.Flags().GetBool("auto-fix") + if autoFixFlag && !feedbackFlag { + return fmt.Errorf("--auto-fix requires --feedback") + } + + ops, err := resolveOperators(mutateOperators) + if err != nil { + return err + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Running %d operator(s) × cases in %s...\n", len(ops), mutateCases) + + opts := mutation.RunOptions{ + Target: mutateTarget, + CasesDir: mutateCases, + Operators: ops, + Concurrency: mutateConcurrency, + } + + run, err := mutation.Run(opts) + if err != nil { + return fmt.Errorf("mutation run: %w", err) + } + + run.Clusters = mutation.ClusterSurvivors(run) + + if feedbackFlag && run.Survivors > 0 { + cfg, cfgErr := config.Load() + if cfgErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warn: --feedback: failed to load config: %v\n", cfgErr) + } else { + provider := llm.NewProviderWithConfig(llm.ProviderConfig{ + APIKey: cfg.AI.APIKey, + Provider: cfg.AI.Provider, + Model: cfg.AI.Model, + BaseURL: cfg.AI.BaseURL, + Region: cfg.AI.Region, + }) + items, fbErr := mutation.Analyze(context.Background(), run, provider) + if fbErr == nil && len(items) > 0 { + run.Feedback = items + fmt.Fprintf(out, "\nFeedback (%d cases with weak assertions):\n", len(items)) + for _, item := range items { + fmt.Fprintf(out, " ⚠ [%s] %s risk=%.2f → %d suggested assertion(s)\n", + item.CaseID, item.Title, item.RiskScore, len(item.SuggestedAssertions)) + } + autoFix, _ := cmd.Flags().GetBool("auto-fix") + if autoFix { + yes, _ := cmd.Flags().GetBool("yes") + if err := runAutoFix(run, mutateCases, yes, out); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "auto-fix: %v\n", err) + } + } else { + fmt.Fprintln(out, "Run with --feedback --auto-fix to patch index.json") + } + } + } + } + + for _, r := range run.Results { + if r.Survived { + color.New(color.FgRed).Fprintf(out, " ✗ [%s] %s — not caught\n", r.Operator, r.Title) + } else { + color.New(color.FgGreen).Fprintf(out, " ✓ [%s] %s — caught\n", r.Operator, r.Title) + } + } + + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + if run.Survivors == 0 { + color.New(color.FgGreen).Fprintf(out, "\nMutation Score: %d/%d killed (%d%%) — no survivors\n", + run.Killed, run.TotalRuns, pct) + } else { + color.New(color.FgYellow).Fprintf(out, "\nMutation Score: %d/%d killed (%d%%)\nSurvivors: %d\n", + run.Killed, run.TotalRuns, pct, run.Survivors) + } + + _ = mutation.Persist("", run) + + if mutateOutput != "" { + if err := mutation.WriteReport(mutateOutput, run); err != nil { + return fmt.Errorf("writing report: %w", err) + } + fmt.Fprintf(cmd.ErrOrStderr(), "Report written to: %s\n", + filepath.Join(mutateOutput, "mutation-report.json")) + } + + if run.Survivors > 0 { + os.Exit(ExitPartialSuccess) + } + return nil +} + +func resolveOperators(filter string) ([]mutation.Operator, error) { + all := mutation.Registry() + if filter == "" { + return all, nil + } + names := strings.Split(filter, ",") + nameSet := map[string]bool{} + for _, n := range names { + nameSet[strings.TrimSpace(n)] = true + } + var ops []mutation.Operator + for _, op := range all { + if nameSet[op.Name()] { + ops = append(ops, op) + } + } + if len(ops) == 0 { + return nil, fmt.Errorf("no operators matched --operator %q; valid names: %s", + filter, operatorNames(all)) + } + return ops, nil +} + +func operatorNames(ops []mutation.Operator) string { + names := make([]string, len(ops)) + for i, op := range ops { + names[i] = op.Name() + } + return strings.Join(names, ", ") +} + +func runAutoFix(run mutation.MutationRun, casesDir string, skipConfirm bool, out io.Writer) error { + if len(run.Feedback) == 0 { + return nil + } + + fmt.Fprintf(out, "\nAuto-fix will append %d assertion(s) across %d case(s).\n", + countSuggestions(run.Feedback), len(run.Feedback)) + + if !skipConfirm { + fmt.Fprint(out, "Apply? [y/N] ") + var answer string + fmt.Scanln(&answer) + if answer != "y" && answer != "Y" { + fmt.Fprintln(out, "Skipped.") + return nil + } + } + + if err := mutation.PatchIndex(casesDir, run.Feedback); err != nil { + return fmt.Errorf("patching index.json: %w", err) + } + fmt.Fprintln(out, "index.json updated.") + return nil +} + +func countSuggestions(items []mutation.FeedbackItem) int { + n := 0 + for _, item := range items { + n += len(item.SuggestedAssertions) + } + return n +} diff --git a/cmd/mutate_test.go b/cmd/mutate_test.go new file mode 100644 index 0000000..2fecf1a --- /dev/null +++ b/cmd/mutate_test.go @@ -0,0 +1,32 @@ +// cmd/mutate_test.go +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMutateCmdRegistered(t *testing.T) { + found := false + for _, c := range rootCmd.Commands() { + if c.Name() == "mutate" { + found = true + break + } + } + assert.True(t, found, "mutate command must be registered on rootCmd") +} + +func TestMutateCmdRequiresFlags(t *testing.T) { + buf := &bytes.Buffer{} + rootCmd.SetOut(buf) + rootCmd.SetErr(buf) + rootCmd.SetArgs([]string{"mutate"}) + err := rootCmd.Execute() + if err == nil { + t.Fatal("mutate without --cases and --target must return error") + } + rootCmd.SetArgs(nil) +} diff --git a/go.mod b/go.mod index 7c0ae69..cd1bc43 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 + golang.org/x/sync v0.20.0 golang.org/x/tools v0.43.0 google.golang.org/genai v1.51.0 gopkg.in/yaml.v3 v3.0.1 @@ -106,7 +107,6 @@ require ( golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect - golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260120221211-b8f7ae30c516 // indirect diff --git a/internal/mutation/engine.go b/internal/mutation/engine.go new file mode 100644 index 0000000..0742f4c --- /dev/null +++ b/internal/mutation/engine.go @@ -0,0 +1,162 @@ +// internal/mutation/engine.go +package mutation + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/testmind-hq/caseforge/internal/runner" +) + +type indexFile struct { + TestCases []struct { + ID string `json:"id"` + Title string `json:"title"` + } `json:"test_cases"` +} + +// Run executes the full mutation loop and returns the aggregate MutationRun. +func Run(opts RunOptions) (MutationRun, error) { + opts.setDefaults() + + cases, err := loadCases(opts.CasesDir) + if err != nil { + return MutationRun{}, fmt.Errorf("loading index.json: %w", err) + } + if len(cases) == 0 { + return MutationRun{}, fmt.Errorf("no test cases found in %s/index.json", opts.CasesDir) + } + + proxy, err := NewProxy(opts.Target) + if err != nil { + return MutationRun{}, fmt.Errorf("starting proxy: %w", err) + } + defer proxy.Close() + + var ( + mu sync.Mutex + results []CaseMutationResult + ) + + operators := opts.Operators + if len(operators) == 0 { + operators = Registry() + } + + for _, op := range operators { + proxy.SetActive(op) + g, _ := errgroup.WithContext(context.Background()) + sem := make(chan struct{}, opts.Concurrency) + + for _, tc := range cases { + g.Go(func() error { + sem <- struct{}{} + defer func() { <-sem }() + survived := runOnce(proxy.Addr(), opts.CasesDir, tc.ID) + mu.Lock() + results = append(results, CaseMutationResult{ + CaseID: tc.ID, + Title: tc.Title, + Operator: op.Name(), + Survived: survived, + }) + mu.Unlock() + return nil + }) + } + _ = g.Wait() + } + + killed, survivors := 0, 0 + for _, r := range results { + if r.Survived { + survivors++ + } else { + killed++ + } + } + + opNames := make([]string, len(operators)) + for i, op := range operators { + opNames[i] = op.Name() + } + + total := killed + survivors + score := 0.0 + if total > 0 { + score = float64(killed) / float64(total) + } + + return MutationRun{ + Target: opts.Target, + CasesDir: opts.CasesDir, + Operators: opNames, + TotalCases: len(cases), + TotalRuns: total, + Killed: killed, + Survivors: survivors, + MutationScore: score, + Results: results, + GeneratedAt: time.Now().UTC().Format("2006-01-02T15:04:05Z"), + }, nil +} + +// runOnce runs a single .hurl file against the proxy. +// Returns true if the mutation survived (hurl passed = mutation not caught). +func runOnce(proxyAddr, casesDir, caseID string) bool { + hurlFile := filepath.Join(casesDir, caseID+".hurl") + if _, err := os.Stat(hurlFile); err != nil { + return false // missing file → count as killed + } + + tmpDir, err := os.MkdirTemp("", "caseforge-mutate-*") + if err != nil { + return false + } + defer os.RemoveAll(tmpDir) + + src, err := os.ReadFile(hurlFile) + if err != nil { + return false + } + if err := os.WriteFile(filepath.Join(tmpDir, caseID+".hurl"), src, 0644); err != nil { + return false + } + + r := runner.NewHurlRunner() + result, err := r.Run(tmpDir, map[string]string{"base_url": "http://" + proxyAddr}) + if err != nil { + return false + } + + // Mutation survived if hurl passed (assertions didn't catch the mutation) + return result.Passed > 0 && result.Failed == 0 +} + +type caseRef struct { + ID string + Title string +} + +func loadCases(casesDir string) ([]caseRef, error) { + data, err := os.ReadFile(filepath.Join(casesDir, "index.json")) + if err != nil { + return nil, err + } + var idx indexFile + if err := json.Unmarshal(data, &idx); err != nil { + return nil, err + } + refs := make([]caseRef, len(idx.TestCases)) + for i, tc := range idx.TestCases { + refs[i] = caseRef{ID: tc.ID, Title: tc.Title} + } + return refs, nil +} diff --git a/internal/mutation/engine_test.go b/internal/mutation/engine_test.go new file mode 100644 index 0000000..953b074 --- /dev/null +++ b/internal/mutation/engine_test.go @@ -0,0 +1,119 @@ +// internal/mutation/engine_test.go +package mutation_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func buildTestHurlFile(t *testing.T, dir, caseID string, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, caseID+".hurl"), []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +func buildTestIndexJSON(t *testing.T, dir, caseID, title string) { + t.Helper() + data, _ := json.Marshal(map[string]any{ + "test_cases": []map[string]any{{"id": caseID, "title": title}}, + }) + if err := os.WriteFile(filepath.Join(dir, "index.json"), data, 0644); err != nil { + t.Fatal(err) + } +} + +func TestEngineRun_KilledMutation(t *testing.T) { + if _, err := exec.LookPath("hurl"); err != nil { + t.Skip("hurl not installed") + } + + // Backend always returns 200 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`{"id":1}`)) + })) + defer backend.Close() + + dir := t.TempDir() + // Hurl file asserts HTTP 200 — status_swap_2xx changes it to 201 → hurl FAILS → mutation KILLED + buildTestHurlFile(t, dir, "TC-0001", "GET {{base_url}}/\nHTTP 200\n") + buildTestIndexJSON(t, dir, "TC-0001", "root GET") + + opts := mutation.RunOptions{ + Target: backend.URL, + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewStatusSwap2xxOperator()}, + Concurrency: 1, + } + run, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) + } + if run.Survivors != 0 { + t.Fatalf("status_swap_2xx must be killed by HTTP 200 assertion, got %d survivors", run.Survivors) + } + if run.Killed != 1 { + t.Fatalf("expected 1 killed, got %d", run.Killed) + } +} + +func TestEngineRun_SurvivorMutation(t *testing.T) { + if _, err := exec.LookPath("hurl"); err != nil { + t.Skip("hurl not installed") + } + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":1,"name":"test"}`)) + })) + defer backend.Close() + + dir := t.TempDir() + // Hurl only checks status == 200, not body fields → field_drop survives + buildTestHurlFile(t, dir, "TC-0002", "GET {{base_url}}/\nHTTP 200\n") + buildTestIndexJSON(t, dir, "TC-0002", "root GET body") + + opts := mutation.RunOptions{ + Target: backend.URL, + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewFieldDropOperator()}, + Concurrency: 1, + } + run, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) + } + if run.Survivors != 1 { + t.Fatalf("field_drop must survive when no body assertion exists, got %d survivors", run.Survivors) + } +} + +func TestEngineRun_NoHurlFiles(t *testing.T) { + dir := t.TempDir() + // Only index.json, no .hurl files — the case should be counted as killed (not error) + buildTestIndexJSON(t, dir, "TC-MISSING", "no hurl file") + + opts := mutation.RunOptions{ + Target: "http://127.0.0.1:1", // unreachable, but won't be called + CasesDir: dir, + Operators: []mutation.Operator{mutation.NewFieldDropOperator()}, + Concurrency: 1, + } + run, err := mutation.Run(opts) + if err != nil { + t.Fatal(err) + } + // Missing hurl file → runOnce returns false (not survived) → killed + if run.TotalRuns != 1 { + t.Fatalf("expected 1 total run, got %d", run.TotalRuns) + } +} diff --git a/internal/mutation/feedback.go b/internal/mutation/feedback.go new file mode 100644 index 0000000..108bb11 --- /dev/null +++ b/internal/mutation/feedback.go @@ -0,0 +1,193 @@ +// internal/mutation/feedback.go +package mutation + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/testmind-hq/caseforge/internal/llm" + "github.com/testmind-hq/caseforge/internal/output/render" + "github.com/testmind-hq/caseforge/internal/output/writer" +) + +// Analyze runs LLM OC-prompting on survivor clusters and returns FeedbackItems. +// Returns nil, nil if provider is unavailable or there are no clusters. +func Analyze(ctx context.Context, run MutationRun, provider llm.LLMProvider) ([]FeedbackItem, error) { + if provider == nil || !provider.IsAvailable() { + return nil, nil + } + if len(run.Clusters) == 0 { + return nil, nil + } + + var items []FeedbackItem + for _, cluster := range run.Clusters { + item, err := analyzeCluster(ctx, cluster, provider) + if err != nil { + fmt.Fprintf(os.Stderr, "warn: feedback for %s: %v\n", cluster.CaseID, err) + continue + } + items = append(items, item) + } + return items, nil +} + +func analyzeCluster(ctx context.Context, cluster SurvivorCluster, provider llm.LLMProvider) (FeedbackItem, error) { + observePrompt := BuildObservePrompt(cluster, nil) + + observeResp, err := llm.Retry(ctx, 3, func() (*llm.CompletionResponse, error) { + return provider.Complete(ctx, &llm.CompletionRequest{ + System: "You are an API testing expert. Return only valid JSON arrays.", + Messages: []llm.Message{{Role: "user", Content: observePrompt}}, + MaxTokens: 512, + }) + }) + if err != nil { + return FeedbackItem{}, fmt.Errorf("observe LLM call: %w", err) + } + + suggestions := parseSuggestedAssertions(observeResp.Text) + + return FeedbackItem{ + CaseID: cluster.CaseID, + Title: cluster.Title, + RiskScore: cluster.RiskScore, + Diagnosis: diagnosisFromCluster(cluster), + SuggestedAssertions: suggestions, + }, nil +} + +// BuildObservePrompt constructs the OC observation prompt for a cluster. +// Exported for testing. +func BuildObservePrompt(cluster SurvivorCluster, existingAssertions []string) string { + existing := strings.Join(existingAssertions, "\n- ") + if existing == "" { + existing = "(none)" + } + return fmt.Sprintf( + "Test case [%s] \"%s\" survived the following HTTP response mutations:\n%s\n\n"+ + "Existing assertions that did NOT catch these mutations:\n- %s\n\n"+ + "Return a JSON array of additional assertions that would detect these mutations.\n"+ + "Each assertion must have: \"target\" (e.g. \"jsonpath $.field\", \"status_code\", \"header Name\"),\n"+ + "\"operator\" (exists|eq|ne|lt|gt|gte|lte|contains|matches|is_iso8601|is_uuid), and optionally \"expected\".\n"+ + "Example: [{\"target\":\"jsonpath $.id\",\"operator\":\"exists\"}]\n"+ + "Return ONLY valid JSON. No explanation.", + cluster.CaseID, + cluster.Title, + "- "+strings.Join(cluster.Operators, "\n- "), + existing, + ) +} + +func parseSuggestedAssertions(text string) []SuggestedAssertion { + extracted := llm.ExtractJSON(text) + var raw []struct { + Target string `json:"target"` + Operator string `json:"operator"` + Expected any `json:"expected,omitempty"` + } + if err := json.Unmarshal([]byte(extracted), &raw); err != nil { + return nil + } + out := make([]SuggestedAssertion, len(raw)) + for i, r := range raw { + out[i] = SuggestedAssertion{Target: r.Target, Operator: r.Operator, Expected: r.Expected} + } + return out +} + +func diagnosisFromCluster(cluster SurvivorCluster) string { + return fmt.Sprintf("%d operator(s) survived (%s) — assertions do not cover response mutations", + len(cluster.Operators), strings.Join(cluster.Operators, ", ")) +} + +// PatchIndex appends suggested assertions from FeedbackItems to matching test cases +// in index.json and writes the updated file back. +func PatchIndex(casesDir string, items []FeedbackItem) error { + indexPath := filepath.Join(casesDir, "index.json") + data, err := os.ReadFile(indexPath) + if err != nil { + return fmt.Errorf("reading index.json: %w", err) + } + + var idx struct { + TestCases []json.RawMessage `json:"test_cases"` + } + if err := json.Unmarshal(data, &idx); err != nil { + return fmt.Errorf("parsing index.json: %w", err) + } + + patches := map[string][]SuggestedAssertion{} + for _, item := range items { + patches[item.CaseID] = item.SuggestedAssertions + } + + updated := make([]json.RawMessage, len(idx.TestCases)) + for i, raw := range idx.TestCases { + var tc map[string]json.RawMessage + if err := json.Unmarshal(raw, &tc); err != nil { + updated[i] = raw + continue + } + var id string + if idRaw, ok := tc["id"]; !ok { + updated[i] = raw + continue + } else { + _ = json.Unmarshal(idRaw, &id) + } + newAssertions, hasPatch := patches[id] + if !hasPatch { + updated[i] = raw + continue + } + + var steps []json.RawMessage + if err := json.Unmarshal(tc["steps"], &steps); err != nil || len(steps) == 0 { + updated[i] = raw + continue + } + var step map[string]json.RawMessage + if err := json.Unmarshal(steps[0], &step); err != nil { + updated[i] = raw + continue + } + var assertions []json.RawMessage + _ = json.Unmarshal(step["assertions"], &assertions) + for _, a := range newAssertions { + ab, _ := json.Marshal(a) + assertions = append(assertions, json.RawMessage(ab)) + } + ab, _ := json.Marshal(assertions) + step["assertions"] = json.RawMessage(ab) + sb, _ := json.Marshal(step) + steps[0] = json.RawMessage(sb) + stepsBytes, _ := json.Marshal(steps) + tc["steps"] = json.RawMessage(stepsBytes) + tcBytes, _ := json.Marshal(tc) + updated[i] = json.RawMessage(tcBytes) + } + + idx.TestCases = updated + out, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(indexPath, out, 0644); err != nil { + return err + } + + // Re-render .hurl files so patched assertions take effect + cases, err := writer.NewJSONSchemaWriter().Read(indexPath) + if err != nil { + return fmt.Errorf("reading patched index.json: %w", err) + } + if err := render.NewHurlRenderer("").Render(cases, casesDir); err != nil { + return fmt.Errorf("index.json patched but .hurl re-render failed: %w", err) + } + return nil +} diff --git a/internal/mutation/feedback_test.go b/internal/mutation/feedback_test.go new file mode 100644 index 0000000..630ef86 --- /dev/null +++ b/internal/mutation/feedback_test.go @@ -0,0 +1,76 @@ +// internal/mutation/feedback_test.go +package mutation_test + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/testmind-hq/caseforge/internal/llm" + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func TestAnalyze_UnavailableProvider(t *testing.T) { + run := sampleRun() + run.Clusters = mutation.ClusterSurvivors(run) + + // "noop" provider name → NoopProvider → IsAvailable() = false + provider := llm.NewProvider("", "noop", "") + items, err := mutation.Analyze(context.Background(), run, provider) + if err != nil { + t.Fatal(err) + } + if items != nil { + t.Fatal("unavailable provider must return nil feedback") + } +} + +func TestBuildObservePrompt(t *testing.T) { + cluster := mutation.SurvivorCluster{ + CaseID: "TC-0001", + Title: "GET /pets", + Operators: []string{"field_drop", "array_to_null"}, + RiskScore: 0.5, + } + existingAssertions := []string{"status_code gte 200", "status_code lt 300"} + prompt := mutation.BuildObservePrompt(cluster, existingAssertions) + if len(prompt) < 50 { + t.Fatalf("prompt too short: %q", prompt) + } + if !strings.Contains(prompt, "field_drop") { + t.Error("prompt must mention surviving operators") + } + if !strings.Contains(prompt, "TC-0001") { + t.Error("prompt must mention case ID") + } +} + +func TestPatchIndex(t *testing.T) { + dir := t.TempDir() + indexData := []byte(`{"test_cases":[{"id":"TC-0001","title":"GET /pets","steps":[{"id":"step-1","assertions":[{"target":"status_code","operator":"eq","expected":200}]}]}]}`) + if err := os.WriteFile(filepath.Join(dir, "index.json"), indexData, 0644); err != nil { + t.Fatal(err) + } + + items := []mutation.FeedbackItem{{ + CaseID: "TC-0001", + SuggestedAssertions: []mutation.SuggestedAssertion{ + {Target: "jsonpath $.id", Operator: "exists"}, + }, + }} + + if err := mutation.PatchIndex(dir, items); err != nil { + t.Fatal(err) + } + + data, err := os.ReadFile(filepath.Join(dir, "index.json")) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(data, []byte("jsonpath $.id")) { + t.Fatalf("patched index.json must contain new assertion:\n%s", data) + } +} diff --git a/internal/mutation/operator_body.go b/internal/mutation/operator_body.go new file mode 100644 index 0000000..e0e722a --- /dev/null +++ b/internal/mutation/operator_body.go @@ -0,0 +1,213 @@ +// internal/mutation/operator_body.go +package mutation + +import ( + "encoding/json" + "net/http" + "sort" + "time" +) + +// --- FieldDropOperator --- + +type fieldDropOperator struct{} + +func NewFieldDropOperator() Operator { return &fieldDropOperator{} } +func (o *fieldDropOperator) Name() string { return "field_drop" } + +func (o *fieldDropOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, keys, err := parseObjectBody(body) + if err != nil || len(keys) == 0 { + return body, nil + } + delete(m, keys[0]) + return json.Marshal(m) +} + +// --- FieldTypeSwapOperator --- + +type fieldTypeSwapOperator struct{} + +func NewFieldTypeSwapOperator() Operator { return &fieldTypeSwapOperator{} } +func (o *fieldTypeSwapOperator) Name() string { return "field_type_swap" } + +func (o *fieldTypeSwapOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, keys, err := parseObjectBody(body) + if err != nil { + return body, nil + } + for _, k := range keys { + switch v := m[k].(type) { + case string: + m[k] = float64(len(v)) + return json.Marshal(m) + case float64: + m[k] = "mutated" + return json.Marshal(m) + } + } + return body, nil +} + +// --- ArrayToNullOperator --- + +type arrayToNullOperator struct{} + +func NewArrayToNullOperator() Operator { return &arrayToNullOperator{} } +func (o *arrayToNullOperator) Name() string { return "array_to_null" } + +func (o *arrayToNullOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, keys, err := parseObjectBody(body) + if err != nil { + return body, nil + } + for _, k := range keys { + if _, ok := m[k].([]any); ok { + m[k] = nil + return json.Marshal(m) + } + } + return body, nil +} + +// --- NullToArrayOperator --- + +type nullToArrayOperator struct{} + +func NewNullToArrayOperator() Operator { return &nullToArrayOperator{} } +func (o *nullToArrayOperator) Name() string { return "null_to_array" } + +func (o *nullToArrayOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, keys, err := parseObjectBody(body) + if err != nil { + return body, nil + } + for _, k := range keys { + if m[k] == nil { + m[k] = []any{} + return json.Marshal(m) + } + } + return body, nil +} + +// --- PaginationOffByOneOperator --- + +var paginationFields = []string{"total", "count", "total_count", "totalCount", "size"} + +type paginationOffByOneOperator struct{} + +func NewPaginationOffByOneOperator() Operator { return &paginationOffByOneOperator{} } +func (o *paginationOffByOneOperator) Name() string { return "pagination_off_by_one" } + +func (o *paginationOffByOneOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, _, err := parseObjectBody(body) + if err != nil { + return body, nil + } + for _, field := range paginationFields { + if v, ok := m[field].(float64); ok { + m[field] = v - 1 + return json.Marshal(m) + } + } + return body, nil +} + +// --- EmptyResultInjectionOperator --- + +type emptyResultInjectionOperator struct{} + +func NewEmptyResultInjectionOperator() Operator { return &emptyResultInjectionOperator{} } +func (o *emptyResultInjectionOperator) Name() string { return "empty_result_injection" } + +func (o *emptyResultInjectionOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + var arr []any + if json.Unmarshal(body, &arr) == nil { + return []byte("[]"), nil + } + m, keys, err := parseObjectBody(body) + if err != nil { + return body, nil + } + for _, k := range keys { + if _, ok := m[k].([]any); ok { + m[k] = []any{} + return json.Marshal(m) + } + } + return body, nil +} + +// --- DateFormatSwapOperator --- + +type dateFormatSwapOperator struct{} + +func NewDateFormatSwapOperator() Operator { return &dateFormatSwapOperator{} } +func (o *dateFormatSwapOperator) Name() string { return "date_format_swap" } + +func (o *dateFormatSwapOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, keys, err := parseObjectBody(body) + if err != nil { + return body, nil + } + for _, k := range keys { + s, ok := m[k].(string) + if !ok { + continue + } + t, parseErr := time.Parse(time.RFC3339, s) + if parseErr != nil { + continue + } + m[k] = t.Unix() + return json.Marshal(m) + } + return body, nil +} + +// --- NumericPrecisionLossOperator --- + +type numericPrecisionLossOperator struct{} + +func NewNumericPrecisionLossOperator() Operator { return &numericPrecisionLossOperator{} } +func (o *numericPrecisionLossOperator) Name() string { return "numeric_precision_loss" } + +func (o *numericPrecisionLossOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { + m, keys, err := parseObjectBody(body) + if err != nil { + return body, nil + } + mutated := false + for _, k := range keys { + v, ok := m[k].(float64) + if !ok { + continue + } + truncated := float64(int64(v)) + if truncated != v { + m[k] = truncated + mutated = true + } + } + if !mutated { + return body, nil + } + return json.Marshal(m) +} + +// --- shared helper --- + +// parseObjectBody decodes body as a JSON object and returns the map and a sorted key slice. +func parseObjectBody(body []byte) (map[string]any, []string, error) { + var m map[string]any + if err := json.Unmarshal(body, &m); err != nil { + return nil, nil, err + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return m, keys, nil +} diff --git a/internal/mutation/operator_body_test.go b/internal/mutation/operator_body_test.go new file mode 100644 index 0000000..1d7a2d8 --- /dev/null +++ b/internal/mutation/operator_body_test.go @@ -0,0 +1,160 @@ +// internal/mutation/operator_body_test.go +package mutation_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func parseBody(t *testing.T, b []byte) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(b, &m); err != nil { + t.Fatalf("invalid JSON output %q: %v", b, err) + } + return m +} + +func TestFieldDrop(t *testing.T) { + op := mutation.NewFieldDropOperator() + body := []byte(`{"id":1,"name":"Alice","email":"a@b.com"}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + if len(m) != 2 { + t.Fatalf("expected 2 fields after drop, got %d: %v", len(m), m) + } +} + +func TestFieldDrop_NonJSON(t *testing.T) { + op := mutation.NewFieldDropOperator() + body := []byte(`not json`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal("Apply must not error on non-JSON body") + } + if string(out) != "not json" { + t.Fatal("non-JSON body must pass through unchanged") + } +} + +func TestFieldTypeSwap(t *testing.T) { + op := mutation.NewFieldTypeSwapOperator() + body := []byte(`{"name":"Alice","count":5}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + _, nameIsNum := m["name"].(float64) + _, countIsStr := m["count"].(string) + if !nameIsNum && !countIsStr { + t.Fatalf("expected one field type swap, got: %v", m) + } +} + +func TestArrayToNull(t *testing.T) { + op := mutation.NewArrayToNullOperator() + body := []byte(`{"items":[1,2,3],"name":"x"}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + if m["items"] != nil { + t.Fatalf("expected items=null, got %v", m["items"]) + } +} + +func TestNullToArray(t *testing.T) { + op := mutation.NewNullToArrayOperator() + body := []byte(`{"result":null}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + arr, ok := m["result"].([]any) + if !ok { + t.Fatalf("expected result=[], got %v (%T)", m["result"], m["result"]) + } + if len(arr) != 0 { + t.Fatalf("expected empty array, got %v", arr) + } +} + +func TestPaginationOffByOne(t *testing.T) { + op := mutation.NewPaginationOffByOneOperator() + body := []byte(`{"total":10,"items":[]}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + total, _ := m["total"].(float64) + if total != 9 && total != 11 { + t.Fatalf("expected total 9 or 11, got %v", m["total"]) + } +} + +func TestEmptyResultInjection_Array(t *testing.T) { + op := mutation.NewEmptyResultInjectionOperator() + body := []byte(`[{"id":1},{"id":2}]`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + if string(out) != "[]" { + t.Fatalf("expected [], got %s", out) + } +} + +func TestEmptyResultInjection_Object(t *testing.T) { + op := mutation.NewEmptyResultInjectionOperator() + body := []byte(`{"data":[1,2,3],"meta":{}}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + arr, ok := m["data"].([]any) + if !ok || len(arr) != 0 { + t.Fatalf("expected data=[], got %v", m["data"]) + } +} + +func TestDateFormatSwap(t *testing.T) { + op := mutation.NewDateFormatSwapOperator() + body := []byte(`{"created_at":"2024-01-15T10:30:00Z"}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + if m["created_at"] == "2024-01-15T10:30:00Z" { + t.Fatal("created_at should have been transformed to unix timestamp") + } +} + +func TestNumericPrecisionLoss(t *testing.T) { + op := mutation.NewNumericPrecisionLossOperator() + body := []byte(`{"price":9.99,"count":3}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + m := parseBody(t, out) + price, _ := m["price"].(float64) + if price != 9 { + t.Fatalf("expected price truncated to 9, got %v", price) + } + count, _ := m["count"].(float64) + if count != 3 { + t.Fatal("integer field must not be changed by precision loss") + } +} diff --git a/internal/mutation/operator_status.go b/internal/mutation/operator_status.go new file mode 100644 index 0000000..2bcfd74 --- /dev/null +++ b/internal/mutation/operator_status.go @@ -0,0 +1,84 @@ +// internal/mutation/operator_status.go +package mutation + +import ( + "net/http" + "sort" + "strings" +) + +// --- StatusSwap2xxOperator --- + +type statusSwap2xxOperator struct{} + +func NewStatusSwap2xxOperator() Operator { return &statusSwap2xxOperator{} } +func (o *statusSwap2xxOperator) Name() string { return "status_swap_2xx" } + +func (o *statusSwap2xxOperator) Apply(resp *http.Response, body []byte) ([]byte, error) { + switch resp.StatusCode { + case 200: + resp.StatusCode = 201 + case 201: + resp.StatusCode = 200 + case 204: + resp.StatusCode = 200 + } + return body, nil +} + +// --- ErrorInflationOperator --- + +type errorInflationOperator struct{} + +func NewErrorInflationOperator() Operator { return &errorInflationOperator{} } +func (o *errorInflationOperator) Name() string { return "error_inflation" } + +func (o *errorInflationOperator) Apply(resp *http.Response, body []byte) ([]byte, error) { + if resp.StatusCode >= 400 { + resp.StatusCode = 200 + } + return body, nil +} + +// --- ContentTypeSwapOperator --- + +type contentTypeSwapOperator struct{} + +func NewContentTypeSwapOperator() Operator { return &contentTypeSwapOperator{} } +func (o *contentTypeSwapOperator) Name() string { return "content_type_swap" } + +func (o *contentTypeSwapOperator) Apply(resp *http.Response, body []byte) ([]byte, error) { + ct := strings.ToLower(resp.Header.Get("Content-Type")) + if strings.HasPrefix(ct, "application/json") { + resp.Header.Set("Content-Type", "text/plain; charset=utf-8") + } + return body, nil +} + +// --- HeaderDropOperator --- + +// skipHeaders are not dropped because removing them would break hurl's response parsing. +var skipHeaders = map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + "Connection": true, +} + +type headerDropOperator struct{} + +func NewHeaderDropOperator() Operator { return &headerDropOperator{} } +func (o *headerDropOperator) Name() string { return "header_drop" } + +func (o *headerDropOperator) Apply(resp *http.Response, body []byte) ([]byte, error) { + keys := make([]string, 0, len(resp.Header)) + for k := range resp.Header { + if !skipHeaders[k] { + keys = append(keys, k) + } + } + sort.Strings(keys) + if len(keys) > 0 { + resp.Header.Del(keys[0]) + } + return body, nil +} diff --git a/internal/mutation/operator_status_test.go b/internal/mutation/operator_status_test.go new file mode 100644 index 0000000..7f6d130 --- /dev/null +++ b/internal/mutation/operator_status_test.go @@ -0,0 +1,121 @@ +// internal/mutation/operator_status_test.go +package mutation_test + +import ( + "net/http" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func makeResp(statusCode int, headers map[string]string) *http.Response { + h := http.Header{} + for k, v := range headers { + h.Set(k, v) + } + return &http.Response{StatusCode: statusCode, Header: h} +} + +func TestStatusSwap2xx_200to201(t *testing.T) { + op := mutation.NewStatusSwap2xxOperator() + r := makeResp(200, nil) + _, err := op.Apply(r, []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + if r.StatusCode != 201 { + t.Fatalf("expected 201, got %d", r.StatusCode) + } +} + +func TestStatusSwap2xx_201to200(t *testing.T) { + op := mutation.NewStatusSwap2xxOperator() + r := makeResp(201, nil) + _, err := op.Apply(r, []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + if r.StatusCode != 200 { + t.Fatalf("expected 200, got %d", r.StatusCode) + } +} + +func TestStatusSwap2xx_NoOp(t *testing.T) { + op := mutation.NewStatusSwap2xxOperator() + r := makeResp(404, nil) + _, err := op.Apply(r, []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + if r.StatusCode != 404 { + t.Fatal("non-2xx status must not be changed") + } +} + +func TestErrorInflation(t *testing.T) { + op := mutation.NewErrorInflationOperator() + r := makeResp(400, nil) + body := []byte(`{"error":"bad request"}`) + out, err := op.Apply(r, body) + if err != nil { + t.Fatal(err) + } + if r.StatusCode != 200 { + t.Fatalf("expected 200, got %d", r.StatusCode) + } + if string(out) != string(body) { + t.Fatal("body must pass through unchanged") + } +} + +func TestErrorInflation_NoOp(t *testing.T) { + op := mutation.NewErrorInflationOperator() + r := makeResp(200, nil) + _, err := op.Apply(r, []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + if r.StatusCode != 200 { + t.Fatal("2xx status must not be inflated") + } +} + +func TestContentTypeSwap(t *testing.T) { + op := mutation.NewContentTypeSwapOperator() + r := makeResp(200, map[string]string{"Content-Type": "application/json"}) + _, err := op.Apply(r, []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + if r.Header.Get("Content-Type") != "text/plain; charset=utf-8" { + t.Fatalf("expected text/plain; charset=utf-8, got %s", r.Header.Get("Content-Type")) + } +} + +func TestHeaderDrop(t *testing.T) { + op := mutation.NewHeaderDropOperator() + r := makeResp(200, map[string]string{ + "X-Request-Id": "abc", + "X-Custom": "val", + }) + _, err := op.Apply(r, []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + // Exactly one header should have been dropped + remaining := 0 + for k := range r.Header { + if !skipHeaders[k] { + remaining++ + } + } + if remaining != 1 { + t.Fatalf("expected 1 non-skip header remaining, got %d: %v", remaining, r.Header) + } +} + +var skipHeaders = map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + "Connection": true, +} diff --git a/internal/mutation/proxy.go b/internal/mutation/proxy.go new file mode 100644 index 0000000..3da1454 --- /dev/null +++ b/internal/mutation/proxy.go @@ -0,0 +1,106 @@ +// internal/mutation/proxy.go +package mutation + +import ( + "bytes" + "fmt" + "io" + "net" + "net/http" + "net/http/httputil" + "net/url" + "sync" +) + +// Registry returns all 12 built-in mutation operators in a fixed order. +func Registry() []Operator { + return []Operator{ + NewFieldDropOperator(), + NewFieldTypeSwapOperator(), + NewArrayToNullOperator(), + NewNullToArrayOperator(), + NewStatusSwap2xxOperator(), + NewErrorInflationOperator(), + NewPaginationOffByOneOperator(), + NewEmptyResultInjectionOperator(), + NewContentTypeSwapOperator(), + NewHeaderDropOperator(), + NewDateFormatSwapOperator(), + NewNumericPrecisionLossOperator(), + } +} + +// MutationProxy wraps httputil.ReverseProxy and intercepts responses to apply mutations. +type MutationProxy struct { + server *http.Server + listener net.Listener + mu sync.RWMutex + active Operator // nil = passthrough +} + +// NewProxy creates a MutationProxy that forwards to target (e.g. "http://api:8080"). +// Call Close() when done. +func NewProxy(target string) (*MutationProxy, error) { + targetURL, err := url.Parse(target) + if err != nil { + return nil, fmt.Errorf("invalid target URL: %w", err) + } + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("starting proxy listener: %w", err) + } + + mp := &MutationProxy{listener: ln} + + rp := httputil.NewSingleHostReverseProxy(targetURL) + rp.ModifyResponse = mp.modifyResponse + + mp.server = &http.Server{Handler: rp} + go mp.server.Serve(ln) //nolint:errcheck + return mp, nil +} + +// Addr returns the proxy's local listen address (e.g. "127.0.0.1:34521"). +func (mp *MutationProxy) Addr() string { + return mp.listener.Addr().String() +} + +// SetActive sets the operator applied to every response until changed. +// Pass nil for passthrough mode. +func (mp *MutationProxy) SetActive(op Operator) { + mp.mu.Lock() + defer mp.mu.Unlock() + mp.active = op +} + +// Close shuts down the proxy server. +func (mp *MutationProxy) Close() error { + return mp.server.Close() +} + +func (mp *MutationProxy) modifyResponse(resp *http.Response) error { + mp.mu.RLock() + op := mp.active + mp.mu.RUnlock() + + if op == nil { + return nil + } + + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if err != nil { + return err + } + + mutated, err := op.Apply(resp, body) + if err != nil { + mutated = body // operator failed: pass original body through + } + + resp.Body = io.NopCloser(bytes.NewReader(mutated)) + resp.ContentLength = int64(len(mutated)) + resp.Header.Set("Content-Length", fmt.Sprintf("%d", len(mutated))) + return nil +} diff --git a/internal/mutation/proxy_test.go b/internal/mutation/proxy_test.go new file mode 100644 index 0000000..cf397ba --- /dev/null +++ b/internal/mutation/proxy_test.go @@ -0,0 +1,104 @@ +// internal/mutation/proxy_test.go +package mutation_test + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func TestProxyPassthrough(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":1,"name":"test"}`)) + })) + defer backend.Close() + + proxy, err := mutation.NewProxy(backend.URL) + if err != nil { + t.Fatal(err) + } + defer proxy.Close() + + resp, err := http.Get("http://" + proxy.Addr()) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if string(body) != `{"id":1,"name":"test"}` { + t.Fatalf("passthrough: expected original body, got %s", body) + } +} + +func TestProxyAppliesMutation(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":1,"name":"test","email":"a@b.com"}`)) + })) + defer backend.Close() + + proxy, err := mutation.NewProxy(backend.URL) + if err != nil { + t.Fatal(err) + } + defer proxy.Close() + proxy.SetActive(mutation.NewFieldDropOperator()) + + resp, err := http.Get("http://" + proxy.Addr()) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + + var m map[string]any + if err := json.Unmarshal(body, &m); err != nil { + t.Fatalf("invalid JSON: %s", body) + } + if len(m) != 2 { + t.Fatalf("expected 2 fields after field_drop, got %d: %v", len(m), m) + } +} + +func TestProxyStatusMutation(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _, _ = w.Write([]byte(`{}`)) + })) + defer backend.Close() + + proxy, err := mutation.NewProxy(backend.URL) + if err != nil { + t.Fatal(err) + } + defer proxy.Close() + proxy.SetActive(mutation.NewStatusSwap2xxOperator()) + + resp, err := http.Get("http://" + proxy.Addr()) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 201 { + t.Fatalf("expected 201, got %d", resp.StatusCode) + } +} + +func TestRegistry(t *testing.T) { + ops := mutation.Registry() + if len(ops) != 12 { + t.Fatalf("expected 12 operators, got %d", len(ops)) + } + names := map[string]bool{} + for _, op := range ops { + if names[op.Name()] { + t.Fatalf("duplicate operator name: %s", op.Name()) + } + names[op.Name()] = true + } +} diff --git a/internal/mutation/report.go b/internal/mutation/report.go new file mode 100644 index 0000000..7b48f05 --- /dev/null +++ b/internal/mutation/report.go @@ -0,0 +1,91 @@ +// internal/mutation/report.go +package mutation + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// TextSummary returns a multi-line human-readable summary of a MutationRun. +func TextSummary(run MutationRun) string { + var sb strings.Builder + for _, r := range run.Results { + if r.Survived { + fmt.Fprintf(&sb, " ✗ [%s] %s — mutation not caught\n", r.Operator, r.Title) + } else { + fmt.Fprintf(&sb, " ✓ [%s] %s — caught\n", r.Operator, r.Title) + } + } + pct := 0 + if run.TotalRuns > 0 { + pct = int(run.MutationScore * 100) + } + fmt.Fprintf(&sb, "\nMutation Score: %d/%d killed (%d%%)\n", run.Killed, run.TotalRuns, pct) + fmt.Fprintf(&sb, "Survivors: %d", run.Survivors) + return sb.String() +} + +// WriteReport writes mutation-report.json to outputDir. +func WriteReport(outputDir string, run MutationRun) error { + if err := os.MkdirAll(outputDir, 0755); err != nil { + return err + } + data, err := json.MarshalIndent(run, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(outputDir, "mutation-report.json"), data, 0644) +} + +// Persist writes a timestamped run file to baseDir. +// If baseDir is empty, uses the default .caseforge/mutation/runs directory. +func Persist(baseDir string, run MutationRun) error { + dir := baseDir + if dir == "" { + dir = persistDir() + } + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + data, err := json.MarshalIndent(run, "", " ") + if err != nil { + return err + } + name := timestamp() + ".json" + return os.WriteFile(filepath.Join(dir, name), data, 0644) +} + +// ClusterSurvivors groups results by case ID, computes RiskScore, sorts descending. +func ClusterSurvivors(run MutationRun) []SurvivorCluster { + byCase := map[string]*SurvivorCluster{} + for _, r := range run.Results { + if !r.Survived { + continue + } + c, ok := byCase[r.CaseID] + if !ok { + byCase[r.CaseID] = &SurvivorCluster{CaseID: r.CaseID, Title: r.Title} + c = byCase[r.CaseID] + } + c.Operators = append(c.Operators, r.Operator) + } + + totalOps := len(run.Operators) + if totalOps == 0 { + totalOps = 1 + } + + clusters := make([]SurvivorCluster, 0, len(byCase)) + for _, c := range byCase { + c.RiskScore = float64(len(c.Operators)) / float64(totalOps) + clusters = append(clusters, *c) + } + sort.Slice(clusters, func(i, j int) bool { + return clusters[i].RiskScore > clusters[j].RiskScore + }) + return clusters +} diff --git a/internal/mutation/report_test.go b/internal/mutation/report_test.go new file mode 100644 index 0000000..923f364 --- /dev/null +++ b/internal/mutation/report_test.go @@ -0,0 +1,94 @@ +// internal/mutation/report_test.go +package mutation_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func sampleRun() mutation.MutationRun { + return mutation.MutationRun{ + Target: "http://api:8080", + CasesDir: "./cases", + Operators: []string{"field_drop", "status_swap_2xx"}, + TotalCases: 3, + TotalRuns: 6, + Killed: 4, + Survivors: 2, + MutationScore: 4.0 / 6.0, + Results: []mutation.CaseMutationResult{ + {CaseID: "TC-0001", Title: "GET /pets", Operator: "field_drop", Survived: true}, + {CaseID: "TC-0001", Title: "GET /pets", Operator: "status_swap_2xx", Survived: false}, + {CaseID: "TC-0002", Title: "POST /pets", Operator: "field_drop", Survived: false}, + {CaseID: "TC-0002", Title: "POST /pets", Operator: "status_swap_2xx", Survived: false}, + {CaseID: "TC-0003", Title: "DELETE /pets/{id}", Operator: "field_drop", Survived: true}, + {CaseID: "TC-0003", Title: "DELETE /pets/{id}", Operator: "status_swap_2xx", Survived: false}, + }, + GeneratedAt: "2026-05-10T12:00:00Z", + } +} + +func TestTextSummary(t *testing.T) { + run := sampleRun() + summary := mutation.TextSummary(run) + if !strings.Contains(summary, "Mutation Score") { + t.Error("summary must contain 'Mutation Score'") + } + if !strings.Contains(summary, "Survivors: 2") { + t.Error("summary must contain 'Survivors: 2'") + } +} + +func TestWriteReport(t *testing.T) { + dir := t.TempDir() + run := sampleRun() + if err := mutation.WriteReport(dir, run); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(dir, "mutation-report.json")) + if err != nil { + t.Fatal(err) + } + var decoded mutation.MutationRun + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if decoded.Survivors != 2 { + t.Fatalf("expected 2 survivors in report, got %d", decoded.Survivors) + } +} + +func TestPersist(t *testing.T) { + dir := t.TempDir() + run := sampleRun() + if err := mutation.Persist(dir, run); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("expected 1 run file, got %d", len(entries)) + } + if !strings.HasSuffix(entries[0].Name(), ".json") { + t.Fatalf("run file must be .json, got %s", entries[0].Name()) + } +} + +func TestClusterSurvivors(t *testing.T) { + run := sampleRun() + clusters := mutation.ClusterSurvivors(run) + if len(clusters) != 2 { + t.Fatalf("expected 2 clusters (TC-0001 and TC-0003), got %d", len(clusters)) + } + // Clusters sorted by RiskScore descending + if clusters[0].RiskScore < clusters[1].RiskScore { + t.Error("clusters must be sorted by RiskScore descending") + } +} diff --git a/internal/mutation/types.go b/internal/mutation/types.go new file mode 100644 index 0000000..5579260 --- /dev/null +++ b/internal/mutation/types.go @@ -0,0 +1,87 @@ +// internal/mutation/types.go +package mutation + +import ( + "net/http" + "time" +) + +// Operator applies a single mutation to an HTTP response body. +// Header-mutating operators set headers directly on resp inside Apply. +type Operator interface { + Name() string + Apply(resp *http.Response, body []byte) ([]byte, error) +} + +// CaseMutationResult records whether one operator survived against one test case. +type CaseMutationResult struct { + CaseID string `json:"case_id"` + Title string `json:"title"` + Operator string `json:"operator"` + Survived bool `json:"survived"` // true = mutation was not caught by assertions +} + +// SurvivorCluster groups all operators that survived a single test case. +type SurvivorCluster struct { + CaseID string `json:"case_id"` + Title string `json:"title"` + Operators []string `json:"operators"` + RiskScore float64 `json:"risk_score"` // len(Operators) / total operators in run +} + +// SuggestedAssertion is one LLM-suggested assertion to add to an existing test case. +type SuggestedAssertion struct { + Target string `json:"target"` + Operator string `json:"operator"` + Expected any `json:"expected,omitempty"` +} + +// FeedbackItem is one LLM-generated diagnosis for a SurvivorCluster. +type FeedbackItem struct { + CaseID string `json:"case_id"` + Title string `json:"title"` + RiskScore float64 `json:"risk_score"` + Diagnosis string `json:"diagnosis"` + SuggestedAssertions []SuggestedAssertion `json:"suggested_assertions"` +} + +// MutationRun is the top-level report for a complete mutation run. +type MutationRun struct { + Target string `json:"target"` + CasesDir string `json:"cases_dir"` + Operators []string `json:"operators"` + TotalCases int `json:"total_cases"` + TotalRuns int `json:"total_runs"` // cases × operators + Killed int `json:"killed"` + Survivors int `json:"survivors"` + MutationScore float64 `json:"mutation_score"` // killed / total_runs + Results []CaseMutationResult `json:"results"` + Clusters []SurvivorCluster `json:"clusters,omitempty"` // Phase 2 + Feedback []FeedbackItem `json:"feedback,omitempty"` // Phase 2 + GeneratedAt string `json:"generated_at"` +} + +// RunOptions configures a mutation run. +type RunOptions struct { + Target string + CasesDir string + OutputDir string + Operators []Operator + Concurrency int +} + +func (o *RunOptions) setDefaults() { + if o.Concurrency <= 0 { + o.Concurrency = 4 + } +} + +// persistDir returns the directory for run JSON files. +func persistDir() string { + return ".caseforge/mutation/runs" +} + +// timestamp returns a UTC timestamp string suitable for filenames. +func timestamp() string { + return time.Now().UTC().Format("2006-01-02T15-04-05Z") +} diff --git a/internal/mutation/types_test.go b/internal/mutation/types_test.go new file mode 100644 index 0000000..cd51ba6 --- /dev/null +++ b/internal/mutation/types_test.go @@ -0,0 +1,42 @@ +// internal/mutation/types_test.go +package mutation_test + +import ( + "net/http" + "testing" + + "github.com/testmind-hq/caseforge/internal/mutation" +) + +func TestOperatorInterface(t *testing.T) { + op := &noopOperator{} + if op.Name() == "" { + t.Fatal("Name() must not be empty") + } + body := []byte(`{"id":1}`) + out, err := op.Apply(&http.Response{}, body) + if err != nil { + t.Fatal(err) + } + if string(out) != string(body) { + t.Fatalf("noop should return body unchanged, got %s", out) + } +} + +func TestMutationRunScore(t *testing.T) { + run := mutation.MutationRun{ + TotalRuns: 10, + Killed: 7, + Survivors: 3, + } + run.MutationScore = float64(run.Killed) / float64(run.TotalRuns) + if run.MutationScore < 0.69 || run.MutationScore > 0.71 { + t.Fatalf("expected ~0.70, got %f", run.MutationScore) + } +} + +// noopOperator satisfies the Operator interface for tests. +type noopOperator struct{} + +func (n *noopOperator) Name() string { return "noop" } +func (n *noopOperator) Apply(_ *http.Response, body []byte) ([]byte, error) { return body, nil } diff --git a/internal/runner/hurl.go b/internal/runner/hurl.go index 143c7be..22d4b54 100644 --- a/internal/runner/hurl.go +++ b/internal/runner/hurl.go @@ -16,11 +16,10 @@ type HurlRunner struct{} func NewHurlRunner() *HurlRunner { return &HurlRunner{} } -type hurlReport struct { - Entries []struct { - Filename string `json:"filename"` - Success bool `json:"success"` - } `json:"entries"` +// hurlReportFile represents one file entry in the hurl JSON report array. +type hurlReportFile struct { + Filename string `json:"filename"` + Success bool `json:"success"` } // Run executes all .hurl files in casesDir and returns a RunResult. @@ -71,12 +70,12 @@ func (r *HurlRunner) Run(casesDir string, vars map[string]string) (RunResult, er } func buildRunResult(data []byte) RunResult { - var report hurlReport - if err := json.Unmarshal(data, &report); err != nil { + var files []hurlReportFile + if err := json.Unmarshal(data, &files); err != nil { return RunResult{} } result := RunResult{} - for _, e := range report.Entries { + for _, e := range files { id := strings.TrimSuffix(filepath.Base(e.Filename), ".hurl") result.Cases = append(result.Cases, CaseResult{ID: id, Title: id, Passed: e.Success}) if e.Success { diff --git a/internal/runner/hurl_test.go b/internal/runner/hurl_test.go index e9c7628..97c05ea 100644 --- a/internal/runner/hurl_test.go +++ b/internal/runner/hurl_test.go @@ -58,13 +58,11 @@ func TestHurlRunnerNoBinary(t *testing.T) { } func TestBuildRunResult(t *testing.T) { - reportJSON := `{ - "entries": [ - {"filename": "TC-abc.hurl", "success": true}, - {"filename": "TC-def.hurl", "success": false}, - {"filename": "TC-ghi.hurl", "success": true} - ] - }` + reportJSON := `[ + {"filename": "TC-abc.hurl", "success": true}, + {"filename": "TC-def.hurl", "success": false}, + {"filename": "TC-ghi.hurl", "success": true} + ]` result := buildRunResult([]byte(reportJSON)) assert.Equal(t, 2, result.Passed) assert.Equal(t, 1, result.Failed) diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index ef6379e..26d4a1d 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -1623,6 +1623,41 @@ else skip "AT-303" "gen --with-sandbox end-to-end" "hurl not installed" fi +# ───────────────────────────────────────────────────────────────────────────── +# AT-401 – AT-403: mutate command +# ───────────────────────────────────────────────────────────────────────────── + +echo "# AT-401 – AT-403: mutate" + +# AT-401: mutate --help mentions "mutation" +contains "AT-401" "mutate command is registered" "mutation" "'$BIN' mutate --help" + +# AT-402: mutate without required flags returns non-zero +run "AT-402" "mutate requires --cases and --target" \ + "! '$BIN' mutate >/dev/null 2>&1" + +# AT-403: full mutation run against sandbox (skipped when hurl not installed) +if command -v hurl >/dev/null 2>&1; then + AT403_CASES=$(mktemp -d) + AT403_REPORT=$(mktemp -d) + "$BIN" gen --spec "$WORKDIR/petstore.yaml" --no-ai \ + --technique equivalence_partitioning \ + --format hurl --output "$AT403_CASES" >/dev/null 2>&1 + + run "AT-403" "mutate runs against sandbox and writes report" \ + "PORT403=\$(random_port) + '$BIN' sandbox --spec '$WORKDIR/petstore.yaml' --port \$PORT403 >/dev/null 2>&1 & + SBX403_PID=\$! + for i in \$(seq 1 20); do curl -sf http://127.0.0.1:\$PORT403/pets >/dev/null 2>&1 && break; sleep 0.1; done + '$BIN' mutate --cases '$AT403_CASES' --target \"http://127.0.0.1:\$PORT403\" --output '$AT403_REPORT' --operator field_drop || true + kill \$SBX403_PID 2>/dev/null || true + test -f '$AT403_REPORT/mutation-report.json'" + + rm -rf "$AT403_CASES" "$AT403_REPORT" +else + skip "AT-403" "mutate full end-to-end" "hurl not installed" +fi + echo "" # ------------------------------------------------------- diff --git a/skills/caseforge/SKILL.md b/skills/caseforge/SKILL.md index 522ae3e..be89df9 100644 --- a/skills/caseforge/SKILL.md +++ b/skills/caseforge/SKILL.md @@ -39,6 +39,7 @@ caseforge lint --spec openapi.yaml | Command | Purpose | |---------|---------| +| `mutate` | Run HTTP boundary mutations via a reverse proxy to find weak test assertions; survivors = mutations hurl didn't catch | | `rbt` | Risk-based testing: detect which API ops are at risk from recent git changes | | `rbt index` | Auto-generate `caseforge-map.yaml` by analysing source code | | `explore` | Probe a live API to discover implicit validation rules (DEA) | @@ -49,6 +50,7 @@ caseforge lint --spec openapi.yaml | Command | Purpose | |---------|---------| +| `sandbox` | Start a local HTTP mock server from an OpenAPI spec (realistic responses via example → schema → faker chain) | | `watch` | Watch spec file and regenerate on change | | `suite create` | Create `suite.json` for cross-case DAG orchestration | | `suite validate` | Validate `suite.json` against `index.json` |