diff --git a/README.md b/README.md index 84f92cd..181cf15 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,11 @@ nightshift setup # Check environment and config health nightshift doctor +# Suggest SLO/SLA targets from run history +nightshift slo +nightshift slo --window 7d --min-confidence medium +nightshift slo --project myrepo --format json + # Budget status and calibration nightshift budget --provider claude nightshift budget snapshot --local-only diff --git a/cmd/nightshift/commands/slo.go b/cmd/nightshift/commands/slo.go new file mode 100644 index 0000000..385cd35 --- /dev/null +++ b/cmd/nightshift/commands/slo.go @@ -0,0 +1,230 @@ +package commands + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/marcus/nightshift/internal/reporting" + "github.com/marcus/nightshift/internal/slo" +) + +var sloCmd = &cobra.Command{ + Use: "slo", + Short: "Suggest SLO/SLA candidates from nightshift run history", + Long: `Analyze existing nightshift run history and suggest concrete SLO/SLA +targets with rationale and confidence labels. + +Suggestions cover reliability (task success rate), latency (p95 task +duration), throughput (PRs per run), cost (p90 tokens per run), and +per-project availability. The command is read-only: it does not persist +suggestions or modify configuration.`, + RunE: func(cmd *cobra.Command, args []string) error { + windowStr, _ := cmd.Flags().GetString("window") + project, _ := cmd.Flags().GetString("project") + format, _ := cmd.Flags().GetString("format") + minConf, _ := cmd.Flags().GetString("min-confidence") + reportsDir, _ := cmd.Flags().GetString("reports-dir") + + window, err := parseSLOWindow(windowStr) + if err != nil { + return err + } + conf, err := parseSLOConfidence(minConf) + if err != nil { + return err + } + if reportsDir == "" { + reportsDir = reporting.DefaultReportsDir() + } + + runs, err := loadRunResultsFromDir(reportsDir) + if err != nil { + return fmt.Errorf("loading run reports: %w", err) + } + + candidates := slo.Suggest(runs, slo.Options{ + Window: window, + Project: project, + MinConfidence: conf, + }) + + switch strings.ToLower(format) { + case "", "text": + return renderSLOText(os.Stdout, candidates, windowStr, project) + case "json": + return renderSLOJSON(os.Stdout, candidates) + default: + return fmt.Errorf("unsupported --format %q (use text or json)", format) + } + }, +} + +func init() { + sloCmd.Flags().String("window", "30d", "Lookback window: e.g. 7d, 30d, 90d, or 'all'") + sloCmd.Flags().String("project", "", "Filter to a single project (basename of project path)") + sloCmd.Flags().String("format", "text", "Output format: text or json") + sloCmd.Flags().String("min-confidence", "", "Minimum confidence to include: low, medium, high") + sloCmd.Flags().String("reports-dir", "", "Override reports directory (defaults to nightshift's reports dir)") + rootCmd.AddCommand(sloCmd) +} + +func parseSLOWindow(s string) (time.Duration, error) { + s = strings.TrimSpace(strings.ToLower(s)) + if s == "" || s == "all" { + return 0, nil + } + if strings.HasSuffix(s, "d") { + days, err := parseUint(strings.TrimSuffix(s, "d")) + if err != nil { + return 0, fmt.Errorf("invalid --window %q: %w", s, err) + } + return time.Duration(days) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("invalid --window %q: %w", s, err) + } + return d, nil +} + +func parseUint(s string) (int, error) { + if s == "" { + return 0, errors.New("empty") + } + var n int + for _, c := range s { + if c < '0' || c > '9' { + return 0, fmt.Errorf("not a non-negative integer: %q", s) + } + n = n*10 + int(c-'0') + } + return n, nil +} + +func parseSLOConfidence(s string) (slo.Confidence, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return "", nil + case "low": + return slo.ConfidenceLow, nil + case "medium", "med": + return slo.ConfidenceMedium, nil + case "high": + return slo.ConfidenceHigh, nil + default: + return "", fmt.Errorf("invalid --min-confidence %q (use low, medium, high)", s) + } +} + +func loadRunResultsFromDir(dir string) ([]*reporting.RunResults, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var runs []*reporting.RunResults + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + if !strings.HasPrefix(name, "run-") || !strings.HasSuffix(name, ".json") { + continue + } + r, err := reporting.LoadRunResults(filepath.Join(dir, name)) + if err != nil { + continue + } + runs = append(runs, r) + } + return runs, nil +} + +func renderSLOJSON(w io.Writer, candidates []slo.Candidate) error { + payload := struct { + Generated time.Time `json:"generated_at"` + Candidates []slo.Candidate `json:"candidates"` + }{ + Generated: time.Now().UTC(), + Candidates: candidates, + } + if payload.Candidates == nil { + payload.Candidates = []slo.Candidate{} + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(payload) +} + +func renderSLOText(w io.Writer, candidates []slo.Candidate, window, project string) error { + fmt.Fprintln(w, "SLO/SLA Candidate Suggestions") + fmt.Fprintln(w, "================================") + fmt.Fprintf(w, " Window: %s\n", strings.TrimSpace(window)) + if project != "" { + fmt.Fprintf(w, " Project: %s\n", project) + } + fmt.Fprintln(w) + + if len(candidates) == 0 { + fmt.Fprintln(w, "No SLO candidates: insufficient run history in the selected window.") + fmt.Fprintln(w, "Try widening --window or running more nightshift jobs first.") + return nil + } + + groups := map[slo.Category][]slo.Candidate{} + for _, c := range candidates { + groups[c.Category] = append(groups[c.Category], c) + } + + order := []struct { + cat slo.Category + title string + }{ + {slo.CategoryReliability, "Reliability"}, + {slo.CategoryLatency, "Latency"}, + {slo.CategoryThroughput, "Throughput"}, + {slo.CategoryCost, "Cost"}, + {slo.CategoryAvailability, "Per-Project Availability"}, + } + + for _, sec := range order { + items := groups[sec.cat] + if len(items) == 0 { + continue + } + sort.Slice(items, func(i, j int) bool { + if items[i].Project != items[j].Project { + return items[i].Project < items[j].Project + } + return items[i].Name < items[j].Name + }) + fmt.Fprintf(w, "%s\n", sec.title) + for _, c := range items { + head := c.Name + if c.Project != "" { + head = fmt.Sprintf("%s (%s)", c.Name, c.Project) + } + fmt.Fprintf(w, " • %s\n", head) + fmt.Fprintf(w, " target: %s\n", c.Target) + fmt.Fprintf(w, " metric: %s\n", c.Metric) + fmt.Fprintf(w, " window: %s\n", c.Window) + fmt.Fprintf(w, " sample: n=%d (%s confidence)\n", c.SampleSize, c.Confidence) + fmt.Fprintf(w, " rationale: %s\n", c.Rationale) + } + fmt.Fprintln(w) + } + + fmt.Fprintln(w, "Suggestions are informational and based purely on observed history.") + return nil +} diff --git a/internal/slo/suggester.go b/internal/slo/suggester.go new file mode 100644 index 0000000..1950dd3 --- /dev/null +++ b/internal/slo/suggester.go @@ -0,0 +1,534 @@ +// Package slo derives SLO/SLA candidate suggestions from nightshift run history. +// +// The suggester is read-only: it consumes existing reporting.RunResults and +// produces a slice of Candidate values with rationale and a confidence label. +// It does not persist suggestions or change any nightshift configuration. +package slo + +import ( + "fmt" + "math" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/marcus/nightshift/internal/reporting" +) + +// Confidence describes how much sample data the suggestion is built on. +type Confidence string + +const ( + ConfidenceLow Confidence = "low" + ConfidenceMedium Confidence = "medium" + ConfidenceHigh Confidence = "high" +) + +// Category groups candidates for output rendering. +type Category string + +const ( + CategoryReliability Category = "reliability" + CategoryLatency Category = "latency" + CategoryThroughput Category = "throughput" + CategoryCost Category = "cost" + CategoryAvailability Category = "availability" +) + +// Candidate is a single SLO/SLA recommendation. +type Candidate struct { + Name string `json:"name" yaml:"name"` + Category Category `json:"category" yaml:"category"` + Metric string `json:"metric" yaml:"metric"` + Target string `json:"target" yaml:"target"` + Window string `json:"window" yaml:"window"` + Rationale string `json:"rationale" yaml:"rationale"` + Confidence Confidence `json:"confidence" yaml:"confidence"` + SampleSize int `json:"sample_size" yaml:"sample_size"` + Project string `json:"project,omitempty" yaml:"project,omitempty"` +} + +// Options configures a Suggester run. +type Options struct { + // Window is the lookback period applied to RunResults.StartTime. + // Zero means "use everything". + Window time.Duration + + // Project filters to a single project (path basename match) when non-empty. + Project string + + // Now overrides the current time (for tests). Zero means time.Now. + Now time.Time + + // MinConfidence filters candidates below this level. Empty means low+. + MinConfidence Confidence +} + +// Suggest produces SLO/SLA candidates from the supplied run history. +// +// runs may be in any order; the suggester sorts a working copy by start time. +func Suggest(runs []*reporting.RunResults, opts Options) []Candidate { + now := opts.Now + if now.IsZero() { + now = time.Now() + } + + filtered := filterRuns(runs, opts, now) + if len(filtered) == 0 { + return nil + } + + var candidates []Candidate + candidates = appendIfPresent(candidates, successRateCandidate(filtered, opts)) + candidates = appendIfPresent(candidates, taskLatencyCandidate(filtered, opts)) + candidates = appendIfPresent(candidates, prThroughputCandidate(filtered, opts)) + candidates = appendIfPresent(candidates, tokenBudgetCandidate(filtered, opts)) + + if opts.Project == "" { + candidates = append(candidates, perProjectAvailabilityCandidates(filtered, opts)...) + } + + if opts.MinConfidence != "" { + candidates = filterByConfidence(candidates, opts.MinConfidence) + } + + return candidates +} + +func appendIfPresent(out []Candidate, c *Candidate) []Candidate { + if c == nil { + return out + } + return append(out, *c) +} + +func filterRuns(runs []*reporting.RunResults, opts Options, now time.Time) []*reporting.RunResults { + out := make([]*reporting.RunResults, 0, len(runs)) + var cutoff time.Time + if opts.Window > 0 { + cutoff = now.Add(-opts.Window) + } + + for _, r := range runs { + if r == nil { + continue + } + if !cutoff.IsZero() && r.StartTime.Before(cutoff) { + continue + } + if opts.Project != "" && !runTouchesProject(r, opts.Project) { + continue + } + out = append(out, r) + } + + sort.Slice(out, func(i, j int) bool { + return out[i].StartTime.Before(out[j].StartTime) + }) + return out +} + +func runTouchesProject(r *reporting.RunResults, project string) bool { + want := strings.ToLower(project) + for _, task := range r.Tasks { + if task.Project == "" { + continue + } + name := strings.ToLower(filepath.Base(task.Project)) + full := strings.ToLower(task.Project) + if name == want || full == want { + return true + } + } + return false +} + +func windowLabel(opts Options) string { + if opts.Window <= 0 { + return "all-time" + } + days := int(math.Round(opts.Window.Hours() / 24)) + if days <= 1 { + return "last 24h" + } + return fmt.Sprintf("rolling %dd", days) +} + +func classifyConfidence(n int) Confidence { + switch { + case n >= 50: + return ConfidenceHigh + case n >= 10: + return ConfidenceMedium + default: + return ConfidenceLow + } +} + +func confidenceRank(c Confidence) int { + switch c { + case ConfidenceHigh: + return 3 + case ConfidenceMedium: + return 2 + case ConfidenceLow: + return 1 + default: + return 0 + } +} + +func filterByConfidence(in []Candidate, min Confidence) []Candidate { + want := confidenceRank(min) + if want == 0 { + return in + } + out := in[:0] + for _, c := range in { + if confidenceRank(c.Confidence) >= want { + out = append(out, c) + } + } + return out +} + +// successRateCandidate suggests a reliability SLO based on per-run task +// success rate. Target is the p10 of run success rates (i.e., we should beat +// it 90% of the time), floored at 90%. +func successRateCandidate(runs []*reporting.RunResults, opts Options) *Candidate { + rates := make([]float64, 0, len(runs)) + totalCompleted, totalFailed := 0, 0 + + for _, r := range runs { + var completed, failed int + for _, task := range r.Tasks { + switch task.Status { + case "completed": + completed++ + case "failed": + failed++ + } + } + denom := completed + failed + if denom == 0 { + continue + } + totalCompleted += completed + totalFailed += failed + rates = append(rates, float64(completed)/float64(denom)*100) + } + + if len(rates) < 3 { + return nil + } + + overall := float64(totalCompleted) / float64(totalCompleted+totalFailed) * 100 + p10 := percentile(rates, 10) + target := math.Min(p10, overall) + if target < 90 { + target = 90 + } + target = math.Floor(target) + + return &Candidate{ + Name: "task-success-rate", + Category: CategoryReliability, + Metric: "completed_tasks / (completed_tasks + failed_tasks)", + Target: fmt.Sprintf(">= %.0f%%", target), + Window: windowLabel(opts), + Rationale: fmt.Sprintf( + "observed success rate is %.1f%% (p10 across %d runs: %.1f%%); target floored at 90%%", + overall, len(rates), p10, + ), + Confidence: classifyConfidence(len(rates)), + SampleSize: len(rates), + } +} + +// taskLatencyCandidate suggests a latency SLO based on p95 task duration. +func taskLatencyCandidate(runs []*reporting.RunResults, opts Options) *Candidate { + durations := make([]float64, 0, 128) + for _, r := range runs { + for _, task := range r.Tasks { + if task.Status != "completed" { + continue + } + if task.Duration <= 0 { + continue + } + durations = append(durations, task.Duration.Seconds()) + } + } + + if len(durations) < 5 { + return nil + } + + p95 := percentile(durations, 95) + target := time.Duration(math.Ceil(p95)) * time.Second + target = roundUpDuration(target) + + return &Candidate{ + Name: "task-completion-latency", + Category: CategoryLatency, + Metric: "completed task wall-clock duration", + Target: fmt.Sprintf("p95 <= %s", formatDurationShort(target)), + Window: windowLabel(opts), + Rationale: fmt.Sprintf( + "observed p95 across %d completed tasks is %s; target rounded up to next round value", + len(durations), formatDurationShort(time.Duration(math.Round(p95))*time.Second), + ), + Confidence: classifyConfidence(len(durations)), + SampleSize: len(durations), + } +} + +// prThroughputCandidate suggests an SLA on PRs per run. +func prThroughputCandidate(runs []*reporting.RunResults, opts Options) *Candidate { + if len(runs) < 5 { + return nil + } + + counts := make([]float64, 0, len(runs)) + runsWithPR := 0 + for _, r := range runs { + var prs int + for _, task := range r.Tasks { + if strings.EqualFold(task.OutputType, "pr") && task.OutputRef != "" { + prs++ + } + } + counts = append(counts, float64(prs)) + if prs > 0 { + runsWithPR++ + } + } + + if runsWithPR == 0 { + return nil + } + + median := percentile(counts, 50) + target := math.Floor(median) + if target < 1 { + target = 1 + } + + return &Candidate{ + Name: "pr-throughput", + Category: CategoryThroughput, + Metric: "PRs created per run", + Target: fmt.Sprintf(">= %.0f PR(s) per run", target), + Window: windowLabel(opts), + Rationale: fmt.Sprintf( + "%d/%d runs produced a PR; median PRs/run = %.1f", + runsWithPR, len(runs), median, + ), + Confidence: classifyConfidence(len(runs)), + SampleSize: len(runs), + } +} + +// tokenBudgetCandidate suggests a cost SLO using p90 tokens-per-run as a ceiling. +func tokenBudgetCandidate(runs []*reporting.RunResults, opts Options) *Candidate { + usage := make([]float64, 0, len(runs)) + for _, r := range runs { + tokens := r.UsedBudget + if tokens == 0 { + for _, task := range r.Tasks { + tokens += task.TokensUsed + } + } + if tokens <= 0 { + continue + } + usage = append(usage, float64(tokens)) + } + + if len(usage) < 5 { + return nil + } + + p90 := percentile(usage, 90) + target := roundUpTokens(int64(math.Ceil(p90))) + + return &Candidate{ + Name: "token-budget-per-run", + Category: CategoryCost, + Metric: "tokens used per run", + Target: fmt.Sprintf("<= %s tokens/run (p90)", formatTokens(target)), + Window: windowLabel(opts), + Rationale: fmt.Sprintf( + "observed p90 across %d runs with token data is %s", + len(usage), formatTokens(int64(math.Round(p90))), + ), + Confidence: classifyConfidence(len(usage)), + SampleSize: len(usage), + } +} + +// perProjectAvailabilityCandidates produces one availability SLO per project +// that has at least 3 runs in the window. +func perProjectAvailabilityCandidates(runs []*reporting.RunResults, opts Options) []Candidate { + type projectState struct { + days map[string]struct{} + successes map[string]struct{} + runs int + } + + projects := map[string]*projectState{} + for _, r := range runs { + seenInRun := map[string]bool{} + for _, task := range r.Tasks { + if task.Project == "" { + continue + } + name := filepath.Base(task.Project) + st, ok := projects[name] + if !ok { + st = &projectState{ + days: map[string]struct{}{}, + successes: map[string]struct{}{}, + } + projects[name] = st + } + day := r.StartTime.UTC().Format("2006-01-02") + st.days[day] = struct{}{} + if !seenInRun[name] { + st.runs++ + seenInRun[name] = true + } + if task.Status == "completed" { + st.successes[day] = struct{}{} + } + } + } + + names := make([]string, 0, len(projects)) + for n := range projects { + names = append(names, n) + } + sort.Strings(names) + + var out []Candidate + for _, name := range names { + st := projects[name] + nights := len(st.days) + if nights < 3 { + continue + } + good := len(st.successes) + pct := float64(good) / float64(nights) * 100 + target := math.Floor(pct) + if target < 70 { + target = 70 + } + + out = append(out, Candidate{ + Name: "project-availability", + Category: CategoryAvailability, + Project: name, + Metric: "% of run-days with >= 1 successful task", + Target: fmt.Sprintf(">= %.0f%%", target), + Window: windowLabel(opts), + Rationale: fmt.Sprintf( + "%d/%d run-days produced at least one successful task for %s", + good, nights, name, + ), + Confidence: classifyConfidence(nights), + SampleSize: nights, + }) + } + return out +} + +func percentile(values []float64, p float64) float64 { + if len(values) == 0 { + return 0 + } + sorted := append([]float64(nil), values...) + sort.Float64s(sorted) + if len(sorted) == 1 { + return sorted[0] + } + rank := (p / 100) * float64(len(sorted)-1) + low := int(math.Floor(rank)) + high := int(math.Ceil(rank)) + if low == high { + return sorted[low] + } + weight := rank - float64(low) + return sorted[low]*(1-weight) + sorted[high]*weight +} + +func roundUpDuration(d time.Duration) time.Duration { + switch { + case d <= 30*time.Second: + return ((d + 4*time.Second) / (5 * time.Second)) * (5 * time.Second) + case d <= 5*time.Minute: + return ((d + 29*time.Second) / (30 * time.Second)) * (30 * time.Second) + case d <= time.Hour: + return ((d + 59*time.Second) / time.Minute) * time.Minute + default: + return ((d + 4*time.Minute + 59*time.Second) / (5 * time.Minute)) * (5 * time.Minute) + } +} + +func roundUpTokens(n int64) int64 { + if n <= 0 { + return 0 + } + switch { + case n < 1_000: + return ((n + 99) / 100) * 100 + case n < 10_000: + return ((n + 499) / 500) * 500 + case n < 100_000: + return ((n + 999) / 1_000) * 1_000 + default: + return ((n + 4_999) / 5_000) * 5_000 + } +} + +func formatTokens(n int64) string { + if n < 1000 { + return fmt.Sprintf("%d", n) + } + sign := "" + if n < 0 { + sign = "-" + n = -n + } + str := fmt.Sprintf("%d", n) + var groups []string + for len(str) > 3 { + groups = append([]string{str[len(str)-3:]}, groups...) + str = str[:len(str)-3] + } + groups = append([]string{str}, groups...) + return sign + strings.Join(groups, ",") +} + +func formatDurationShort(d time.Duration) string { + if d <= 0 { + return "0s" + } + if d < time.Minute { + return fmt.Sprintf("%ds", int(math.Round(d.Seconds()))) + } + if d < time.Hour { + mins := int(d / time.Minute) + secs := int((d % time.Minute) / time.Second) + if secs == 0 { + return fmt.Sprintf("%dm", mins) + } + return fmt.Sprintf("%dm%ds", mins, secs) + } + hours := int(d / time.Hour) + mins := int((d % time.Hour) / time.Minute) + if mins == 0 { + return fmt.Sprintf("%dh", hours) + } + return fmt.Sprintf("%dh%dm", hours, mins) +} diff --git a/internal/slo/suggester_test.go b/internal/slo/suggester_test.go new file mode 100644 index 0000000..fd49b72 --- /dev/null +++ b/internal/slo/suggester_test.go @@ -0,0 +1,277 @@ +package slo + +import ( + "strings" + "testing" + "time" + + "github.com/marcus/nightshift/internal/reporting" +) + +func mkTask(project, status, outputType, outputRef string, tokens int, dur time.Duration) reporting.TaskResult { + return reporting.TaskResult{ + Project: project, + TaskType: "lint-fix", + Title: "test", + Status: status, + OutputType: outputType, + OutputRef: outputRef, + TokensUsed: tokens, + Duration: dur, + } +} + +func mkRun(start time.Time, tasks ...reporting.TaskResult) *reporting.RunResults { + end := start.Add(30 * time.Minute) + used := 0 + for _, t := range tasks { + used += t.TokensUsed + } + return &reporting.RunResults{ + Date: start, + StartTime: start, + EndTime: end, + UsedBudget: used, + Tasks: append([]reporting.TaskResult(nil), tasks...), + } +} + +func TestSuggest_EmptyHistory(t *testing.T) { + out := Suggest(nil, Options{}) + if len(out) != 0 { + t.Fatalf("expected no candidates from nil runs, got %d", len(out)) + } + out = Suggest([]*reporting.RunResults{}, Options{}) + if len(out) != 0 { + t.Fatalf("expected no candidates from empty runs, got %d", len(out)) + } +} + +func TestSuggest_SmallSample_LowConfidence(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := []*reporting.RunResults{ + mkRun(now.Add(-120*time.Hour), + mkTask("/proj/a", "completed", "PR", "https://example/1", 1000, 2*time.Minute), + mkTask("/proj/a", "completed", "", "", 500, time.Minute), + ), + mkRun(now.Add(-96*time.Hour), + mkTask("/proj/a", "completed", "PR", "https://example/2", 800, time.Minute), + mkTask("/proj/a", "failed", "", "", 100, 30*time.Second), + ), + mkRun(now.Add(-72*time.Hour), + mkTask("/proj/a", "completed", "PR", "https://example/3", 1200, 90*time.Second), + mkTask("/proj/a", "completed", "", "", 700, 75*time.Second), + ), + mkRun(now.Add(-48*time.Hour), + mkTask("/proj/a", "completed", "PR", "https://example/4", 1100, 80*time.Second), + ), + mkRun(now.Add(-24*time.Hour), + mkTask("/proj/a", "completed", "PR", "https://example/5", 900, 70*time.Second), + ), + } + + got := Suggest(runs, Options{Now: now, Window: 30 * 24 * time.Hour}) + if len(got) == 0 { + t.Fatalf("expected at least one candidate from 5 runs, got none") + } + for _, c := range got { + if c.Confidence != ConfidenceLow { + t.Errorf("candidate %q: confidence=%s want low (n<10)", c.Name, c.Confidence) + } + } + + if !hasCandidate(got, "task-success-rate") { + t.Errorf("expected task-success-rate candidate") + } + if !hasCandidate(got, "task-completion-latency") { + t.Errorf("expected task-completion-latency candidate") + } + if !hasCandidate(got, "pr-throughput") { + t.Errorf("expected pr-throughput candidate") + } + if !hasCandidate(got, "token-budget-per-run") { + t.Errorf("expected token-budget-per-run candidate") + } +} + +func TestSuggest_LargeSample_HighConfidence(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := make([]*reporting.RunResults, 0, 60) + for i := 0; i < 60; i++ { + start := now.Add(-time.Duration(i) * 24 * time.Hour) + runs = append(runs, mkRun(start, + mkTask("/proj/a", "completed", "PR", "https://example/p", 1500, 2*time.Minute), + mkTask("/proj/a", "completed", "", "", 500, time.Minute), + )) + } + + got := Suggest(runs, Options{Now: now, Window: 0}) + if len(got) == 0 { + t.Fatal("expected candidates with 60 runs") + } + for _, c := range got { + if c.Name == "project-availability" { + continue + } + if c.Confidence != ConfidenceHigh { + t.Errorf("candidate %q: confidence=%s want high (n>=50)", c.Name, c.Confidence) + } + } +} + +func TestSuggest_AllFailedRuns(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := make([]*reporting.RunResults, 0, 6) + for i := 0; i < 6; i++ { + start := now.Add(-time.Duration(i) * 24 * time.Hour) + runs = append(runs, mkRun(start, + mkTask("/proj/a", "failed", "", "", 100, 30*time.Second), + )) + } + + got := Suggest(runs, Options{Now: now}) + // Success rate should still emit but be floored at 90% + for _, c := range got { + if c.Name == "task-success-rate" { + if !strings.Contains(c.Target, "90%") { + t.Errorf("all-failed: success-rate target should floor at 90%%, got %q", c.Target) + } + return + } + } + t.Errorf("expected a task-success-rate candidate even with all failed runs") +} + +func TestSuggest_NoPRs(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := make([]*reporting.RunResults, 0, 8) + for i := 0; i < 8; i++ { + start := now.Add(-time.Duration(i) * 24 * time.Hour) + runs = append(runs, mkRun(start, + mkTask("/proj/a", "completed", "Report", "/tmp/x", 500, 90*time.Second), + )) + } + + got := Suggest(runs, Options{Now: now}) + if hasCandidate(got, "pr-throughput") { + t.Errorf("expected no pr-throughput candidate when no PRs were created") + } +} + +func TestSuggest_MissingTokenData(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := make([]*reporting.RunResults, 0, 8) + for i := 0; i < 8; i++ { + start := now.Add(-time.Duration(i) * 24 * time.Hour) + r := mkRun(start, + mkTask("/proj/a", "completed", "", "", 0, time.Minute), + ) + r.UsedBudget = 0 + runs = append(runs, r) + } + + got := Suggest(runs, Options{Now: now}) + if hasCandidate(got, "token-budget-per-run") { + t.Errorf("expected no token-budget candidate when token data is missing") + } +} + +func TestSuggest_ProjectFilter(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := []*reporting.RunResults{ + mkRun(now.Add(-48*time.Hour), + mkTask("/code/alpha", "completed", "PR", "https://x/1", 500, time.Minute), + ), + mkRun(now.Add(-24*time.Hour), + mkTask("/code/beta", "completed", "PR", "https://x/2", 500, time.Minute), + ), + mkRun(now.Add(-12*time.Hour), + mkTask("/code/alpha", "completed", "PR", "https://x/3", 500, time.Minute), + ), + } + + got := Suggest(runs, Options{Now: now, Project: "alpha"}) + for _, c := range got { + if c.Name == "project-availability" { + t.Errorf("per-project availability should not be emitted when filtering to a single project: %+v", c) + } + } +} + +func TestSuggest_MinConfidenceFilter(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := []*reporting.RunResults{ + mkRun(now.Add(-72*time.Hour), + mkTask("/p/a", "completed", "PR", "https://x/1", 1000, time.Minute), + ), + mkRun(now.Add(-48*time.Hour), + mkTask("/p/a", "completed", "PR", "https://x/2", 1000, time.Minute), + ), + mkRun(now.Add(-24*time.Hour), + mkTask("/p/a", "completed", "PR", "https://x/3", 1000, time.Minute), + ), + } + got := Suggest(runs, Options{Now: now, MinConfidence: ConfidenceMedium}) + if len(got) != 0 { + t.Errorf("expected zero candidates after filtering to medium+, got %d", len(got)) + } +} + +func TestSuggest_WindowFilter(t *testing.T) { + now := time.Date(2026, 5, 16, 12, 0, 0, 0, time.UTC) + runs := []*reporting.RunResults{ + mkRun(now.Add(-90*24*time.Hour), + mkTask("/p/a", "completed", "PR", "https://x/1", 1000, time.Minute), + ), + mkRun(now.Add(-3*24*time.Hour), + mkTask("/p/a", "completed", "PR", "https://x/2", 1000, time.Minute), + ), + } + got := Suggest(runs, Options{Now: now, Window: 7 * 24 * time.Hour}) + // Only 1 run in window → not enough for any candidate (all gates require >=3-5) + if len(got) != 0 { + t.Errorf("expected zero candidates with single in-window run, got %d", len(got)) + } +} + +func TestPercentile(t *testing.T) { + values := []float64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + got := percentile(values, 90) + if got < 89 || got > 96 { + t.Errorf("p90: got %.2f, want ~90-96", got) + } + if v := percentile(nil, 50); v != 0 { + t.Errorf("p50 of empty: got %.2f want 0", v) + } + if v := percentile([]float64{42}, 50); v != 42 { + t.Errorf("p50 single: got %.2f want 42", v) + } +} + +func TestRoundUpTokens(t *testing.T) { + cases := []struct { + in, want int64 + }{ + {0, 0}, + {50, 100}, + {510, 600}, + {1_200, 1_500}, + {12_000, 12_000}, + {12_345, 13_000}, + {120_001, 125_000}, + } + for _, c := range cases { + if got := roundUpTokens(c.in); got != c.want { + t.Errorf("roundUpTokens(%d)=%d want %d", c.in, got, c.want) + } + } +} + +func hasCandidate(cs []Candidate, name string) bool { + for _, c := range cs { + if c.Name == name { + return true + } + } + return false +}