From b574ada41f8fd446fb9b2928b1d07f1fcba1a67b Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Wed, 22 Apr 2026 02:19:52 -0700 Subject: [PATCH 1/2] docs: draft v0.3.4 release notes Nightshift-Task: release-notes Nightshift-Ref: https://github.com/marcus/nightshift --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59afa7d..cb536f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to nightshift are documented in this file. +## [v0.3.4] - 2026-02-28 + +### Features +- **Configurable agent timeouts** — add `--timeout` to `nightshift run` and `nightshift daemon`, with daemon re-exec forwarding the flag (#27) +- **Expanded PII scanner guidance** — add detailed instructions for detecting hardcoded PII, leaked env files, unsafe storage, and related exposure patterns in the built-in task (#34) + +### Fixes +- **Timeout handling and diagnostics** — preserve partial output on timeout, terminate full process groups, and surface partial logs from failed plan/implement/review steps (#33) +- **Copilot CLI integration** — improve binary resolution, permission gating, and CLI flag handling for Copilot runs (#39) +- **Provider config YAML serialization** — write provider settings with the correct YAML keys during setup (#43) +- **Configured run limits and budget fallback** — honor `schedule.max_projects` and `schedule.max_tasks`, improve budget calibration at day and week boundaries, and preserve Codex fallback permissions in headless runs (#42) + +### Other +- **Task reference docs** — add a comprehensive reference page for all 59 built-in tasks and refresh related task documentation (#30) +- **Docs cleanup** — remove auto-generated implementation docs from the repository (#40) +- **Low-risk cleanup** — resolve Copilot helper lint warnings and replace `WriteString(fmt.Sprintf(...))` patterns with `fmt.Fprintf` in reporting and setup code (#38, #41) + ## [v0.3.3] - 2026-02-19 ### Features From e14bc447f6f17a27a99c2f3a2630661cd82ebea8 Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Wed, 13 May 2026 02:40:10 -0700 Subject: [PATCH 2/2] feat: add metrics-coverage analyzer and CLI subcommand Add internal/analysis/coverage package that walks Go source via go/ast and tallies functions containing instrumentation calls (logging/stats/metrics/ telemetry) versus those that don't, with sensible default patterns plus --pattern and --exclude overrides. Expose it as `nightshift metrics-coverage` with text/JSON output and a --min-coverage CI gate. Nightshift-Task: metrics-coverage Nightshift-Ref: https://github.com/marcus/nightshift Co-Authored-By: Claude Opus 4.7 (1M context) --- cmd/nightshift/commands/metrics_coverage.go | 110 +++++++ internal/analysis/coverage/coverage.go | 324 ++++++++++++++++++++ internal/analysis/coverage/coverage_test.go | 241 +++++++++++++++ internal/analysis/coverage/report.go | 70 +++++ website/docs/cli-reference.md | 1 + website/docs/metrics-coverage.md | 105 +++++++ 6 files changed, 851 insertions(+) create mode 100644 cmd/nightshift/commands/metrics_coverage.go create mode 100644 internal/analysis/coverage/coverage.go create mode 100644 internal/analysis/coverage/coverage_test.go create mode 100644 internal/analysis/coverage/report.go create mode 100644 website/docs/metrics-coverage.md diff --git a/cmd/nightshift/commands/metrics_coverage.go b/cmd/nightshift/commands/metrics_coverage.go new file mode 100644 index 0000000..32ede62 --- /dev/null +++ b/cmd/nightshift/commands/metrics_coverage.go @@ -0,0 +1,110 @@ +package commands + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + + "github.com/marcus/nightshift/internal/analysis/coverage" +) + +var metricsCoverageCmd = &cobra.Command{ + Use: "metrics-coverage [path]", + Short: "Analyze metrics instrumentation coverage", + Long: `Statically scan a Go codebase for instrumentation calls (logging, +stats, metrics, telemetry) and report per-package coverage. + +A function is considered "instrumented" if its body contains at least one +call expression matching the configured patterns. The default pattern set +covers log/logging/stats/metrics/otel/telemetry packages and common method +names like .Info, .Error, .Record, .Observe, and .Inc. + +Use --min-coverage in CI to fail the build when overall coverage drops +below a threshold.`, + Example: ` nightshift metrics-coverage + nightshift metrics-coverage --path ./internal --format json + nightshift metrics-coverage --exclude 'vendor/**' --exclude '**/mocks/**' + nightshift metrics-coverage --min-coverage 60`, + RunE: func(cmd *cobra.Command, args []string) error { + path, _ := cmd.Flags().GetString("path") + if path == "" && len(args) > 0 { + path = args[0] + } + if path == "" { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current directory: %w", err) + } + path = cwd + } + + format, _ := cmd.Flags().GetString("format") + minCov, _ := cmd.Flags().GetFloat64("min-coverage") + excludes, _ := cmd.Flags().GetStringArray("exclude") + output, _ := cmd.Flags().GetString("output") + includeTests, _ := cmd.Flags().GetBool("include-tests") + patterns, _ := cmd.Flags().GetStringArray("pattern") + maxGaps, _ := cmd.Flags().GetInt("max-gaps") + + absPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolving path: %w", err) + } + + opts := coverage.Options{ + Root: absPath, + Patterns: patterns, + Excludes: excludes, + IncludeTests: includeTests, + } + + report, err := coverage.New(opts).Analyze() + if err != nil { + return fmt.Errorf("running coverage analysis: %w", err) + } + + var out io.Writer = os.Stdout + if output != "" { + f, err := os.Create(output) + if err != nil { + return fmt.Errorf("opening output file: %w", err) + } + defer func() { _ = f.Close() }() + out = f + } + + switch strings.ToLower(format) { + case "json": + if err := coverage.RenderJSON(out, report); err != nil { + return fmt.Errorf("rendering JSON: %w", err) + } + case "", "text": + if err := coverage.RenderText(out, report, maxGaps); err != nil { + return fmt.Errorf("rendering text: %w", err) + } + default: + return fmt.Errorf("unknown format %q (expected text or json)", format) + } + + if minCov > 0 && report.Percent < minCov { + return fmt.Errorf("metrics coverage %.1f%% is below threshold %.1f%%", report.Percent, minCov) + } + return nil + }, +} + +func init() { + metricsCoverageCmd.Flags().StringP("path", "p", "", "Directory to analyze (default: current dir)") + metricsCoverageCmd.Flags().String("format", "text", "Output format: text or json") + metricsCoverageCmd.Flags().Float64("min-coverage", 0, "Fail if overall coverage is below this percent (0 disables)") + metricsCoverageCmd.Flags().StringArray("exclude", nil, "Glob patterns of relative paths to exclude (repeatable)") + metricsCoverageCmd.Flags().StringArray("pattern", nil, "Override default instrumentation call-site patterns (repeatable)") + metricsCoverageCmd.Flags().StringP("output", "o", "", "Write output to file instead of stdout") + metricsCoverageCmd.Flags().Bool("include-tests", false, "Include *_test.go files in analysis") + metricsCoverageCmd.Flags().Int("max-gaps", 10, "Max uninstrumented functions to list per package in text output (0 to hide)") + rootCmd.AddCommand(metricsCoverageCmd) +} diff --git a/internal/analysis/coverage/coverage.go b/internal/analysis/coverage/coverage.go new file mode 100644 index 0000000..56edce9 --- /dev/null +++ b/internal/analysis/coverage/coverage.go @@ -0,0 +1,324 @@ +// Package coverage provides static analysis of metrics instrumentation +// coverage across a Go codebase. It walks .go source files, parses them with +// go/ast, and reports which functions contain instrumentation calls +// (logging, stats, metrics, telemetry) and which do not. +package coverage + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" +) + +// DefaultPatterns are the call-site patterns that count as "instrumentation" +// for the purposes of this analyzer. They are matched against the textual +// form of a CallExpr's function expression (e.g. "logging.Component", +// "log.Printf", "logger.Infof", "stats.Record", "metrics.Inc", +// "otel.Tracer", "telemetry.Emit"). +var DefaultPatterns = []string{ + "log.", + "logger.", + "logging.", + "zerolog.", + "zlog.", + "stats.", + "metrics.", + "otel.", + "telemetry.", + "trace.", + "prometheus.", + "observability.", + ".Info", + ".Infof", + ".Warn", + ".Warnf", + ".Error", + ".Errorf", + ".Debug", + ".Debugf", + ".Trace", + ".Tracef", + ".Record", + ".Emit", + ".Observe", + ".Inc", +} + +// Options configures the analyzer. +type Options struct { + // Root is the directory to analyze (recursively). + Root string + // Patterns are the substrings used to identify instrumentation calls. + // If empty, DefaultPatterns is used. + Patterns []string + // Excludes are glob patterns matched against the relative path of each + // candidate file (e.g. "vendor/**", "**/*_generated.go"). Files matching + // any glob are skipped. + Excludes []string + // IncludeTests, when false, skips *_test.go files. + IncludeTests bool +} + +// FunctionGap describes an uninstrumented function. +type FunctionGap struct { + Name string `json:"name"` + File string `json:"file"` + Line int `json:"line"` +} + +// PackageCoverage holds the per-package coverage tally. +type PackageCoverage struct { + Package string `json:"package"` + Dir string `json:"dir"` + TotalFuncs int `json:"total_funcs"` + InstrumentedFuncs int `json:"instrumented_funcs"` + Percent float64 `json:"percent"` + UninstrumentedFuncs []FunctionGap `json:"uninstrumented_funcs,omitempty"` +} + +// OverallCoverage aggregates per-package results. +type OverallCoverage struct { + Root string `json:"root"` + Patterns []string `json:"patterns"` + Packages []PackageCoverage `json:"packages"` + TotalFuncs int `json:"total_funcs"` + InstrumentedFuncs int `json:"instrumented_funcs"` + Percent float64 `json:"percent"` +} + +// Analyzer performs metrics instrumentation coverage analysis. +type Analyzer struct { + opts Options +} + +// New creates an analyzer with the supplied options. +func New(opts Options) *Analyzer { + if len(opts.Patterns) == 0 { + opts.Patterns = DefaultPatterns + } + return &Analyzer{opts: opts} +} + +// Analyze walks the configured root and returns the overall coverage report. +func (a *Analyzer) Analyze() (*OverallCoverage, error) { + root, err := filepath.Abs(a.opts.Root) + if err != nil { + return nil, fmt.Errorf("resolving root: %w", err) + } + + pkgs := make(map[string]*PackageCoverage) + + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + name := d.Name() + // Skip common directories that don't contain hand-written source. + if path != root && (name == "vendor" || name == "node_modules" || name == ".git" || name == "testdata") { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if !a.opts.IncludeTests && strings.HasSuffix(path, "_test.go") { + return nil + } + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + rel = path + } + if a.isExcluded(rel) { + return nil + } + if isGenerated(path) { + return nil + } + return a.analyzeFile(path, rel, pkgs) + }) + if err != nil { + return nil, err + } + + result := &OverallCoverage{ + Root: root, + Patterns: append([]string(nil), a.opts.Patterns...), + Packages: make([]PackageCoverage, 0, len(pkgs)), + } + + for _, pc := range pkgs { + sort.Slice(pc.UninstrumentedFuncs, func(i, j int) bool { + if pc.UninstrumentedFuncs[i].File == pc.UninstrumentedFuncs[j].File { + return pc.UninstrumentedFuncs[i].Line < pc.UninstrumentedFuncs[j].Line + } + return pc.UninstrumentedFuncs[i].File < pc.UninstrumentedFuncs[j].File + }) + if pc.TotalFuncs > 0 { + pc.Percent = float64(pc.InstrumentedFuncs) / float64(pc.TotalFuncs) * 100 + } + result.Packages = append(result.Packages, *pc) + result.TotalFuncs += pc.TotalFuncs + result.InstrumentedFuncs += pc.InstrumentedFuncs + } + + sort.Slice(result.Packages, func(i, j int) bool { + return result.Packages[i].Package < result.Packages[j].Package + }) + + if result.TotalFuncs > 0 { + result.Percent = float64(result.InstrumentedFuncs) / float64(result.TotalFuncs) * 100 + } + + return result, nil +} + +func (a *Analyzer) analyzeFile(path, rel string, pkgs map[string]*PackageCoverage) error { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return fmt.Errorf("parsing %s: %w", path, err) + } + + if hasGeneratedComment(file) { + return nil + } + + pkgName := file.Name.Name + dir := filepath.Dir(rel) + key := dir + "::" + pkgName + pc, ok := pkgs[key] + if !ok { + pc = &PackageCoverage{Package: pkgName, Dir: dir} + pkgs[key] = pc + } + + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + pc.TotalFuncs++ + if a.functionInstrumented(fn) { + pc.InstrumentedFuncs++ + } else { + pos := fset.Position(fn.Pos()) + pc.UninstrumentedFuncs = append(pc.UninstrumentedFuncs, FunctionGap{ + Name: functionDisplayName(fn), + File: rel, + Line: pos.Line, + }) + } + } + + return nil +} + +func (a *Analyzer) functionInstrumented(fn *ast.FuncDecl) bool { + found := false + ast.Inspect(fn.Body, func(n ast.Node) bool { + if found { + return false + } + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + expr := exprString(call.Fun) + if a.matchesPattern(expr) { + found = true + return false + } + return true + }) + return found +} + +func (a *Analyzer) matchesPattern(expr string) bool { + for _, p := range a.opts.Patterns { + if p == "" { + continue + } + if strings.Contains(expr, p) { + return true + } + } + return false +} + +func (a *Analyzer) isExcluded(rel string) bool { + rel = filepath.ToSlash(rel) + for _, pat := range a.opts.Excludes { + if pat == "" { + continue + } + pat = filepath.ToSlash(pat) + if matched, err := filepath.Match(pat, rel); err == nil && matched { + return true + } + // Allow "dir/**" style patterns by checking prefix when "**" suffix. + if strings.HasSuffix(pat, "/**") { + prefix := strings.TrimSuffix(pat, "/**") + if strings.HasPrefix(rel, prefix+"/") || rel == prefix { + return true + } + } + // Plain substring match as fallback for convenience. + if strings.Contains(rel, pat) { + return true + } + } + return false +} + +func exprString(e ast.Expr) string { + switch v := e.(type) { + case *ast.Ident: + return v.Name + case *ast.SelectorExpr: + return exprString(v.X) + "." + v.Sel.Name + case *ast.CallExpr: + return exprString(v.Fun) + "()" + case *ast.IndexExpr: + return exprString(v.X) + case *ast.IndexListExpr: + return exprString(v.X) + case *ast.ParenExpr: + return exprString(v.X) + case *ast.StarExpr: + return exprString(v.X) + } + return "" +} + +func functionDisplayName(fn *ast.FuncDecl) string { + if fn.Recv != nil && len(fn.Recv.List) > 0 { + recv := exprString(fn.Recv.List[0].Type) + recv = strings.TrimPrefix(recv, "*") + return fmt.Sprintf("(%s).%s", recv, fn.Name.Name) + } + return fn.Name.Name +} + +func hasGeneratedComment(file *ast.File) bool { + for _, cg := range file.Comments { + for _, c := range cg.List { + line := strings.TrimSpace(strings.TrimPrefix(c.Text, "//")) + if strings.HasPrefix(line, "Code generated") && strings.Contains(line, "DO NOT EDIT") { + return true + } + } + } + return false +} + +func isGenerated(path string) bool { + base := filepath.Base(path) + return strings.HasSuffix(base, "_generated.go") || strings.HasSuffix(base, ".pb.go") +} diff --git a/internal/analysis/coverage/coverage_test.go b/internal/analysis/coverage/coverage_test.go new file mode 100644 index 0000000..8c28111 --- /dev/null +++ b/internal/analysis/coverage/coverage_test.go @@ -0,0 +1,241 @@ +package coverage + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +func findPkg(r *OverallCoverage, dir string) *PackageCoverage { + for i := range r.Packages { + if r.Packages[i].Dir == dir { + return &r.Packages[i] + } + } + return nil +} + +func TestAnalyzer_EmptyPackage(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "empty", "doc.go"), `// Package empty has no functions. +package empty +`) + + r, err := New(Options{Root: root}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + if r.TotalFuncs != 0 { + t.Fatalf("expected 0 total funcs, got %d", r.TotalFuncs) + } + if r.Percent != 0 { + t.Fatalf("expected 0%% coverage, got %.1f", r.Percent) + } +} + +func TestAnalyzer_FullyInstrumented(t *testing.T) { + root := t.TempDir() + src := `package full + +import "log" + +func A() { log.Printf("a") } +func B() { log.Println("b") } +` + writeFile(t, filepath.Join(root, "full", "f.go"), src) + + r, err := New(Options{Root: root}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + pc := findPkg(r, "full") + if pc == nil { + t.Fatalf("package not found in %+v", r.Packages) + } + if pc.TotalFuncs != 2 || pc.InstrumentedFuncs != 2 { + t.Fatalf("want 2/2, got %d/%d", pc.InstrumentedFuncs, pc.TotalFuncs) + } + if pc.Percent != 100 { + t.Fatalf("want 100%%, got %.1f", pc.Percent) + } + if len(pc.UninstrumentedFuncs) != 0 { + t.Fatalf("unexpected gaps: %+v", pc.UninstrumentedFuncs) + } +} + +func TestAnalyzer_PartiallyInstrumented(t *testing.T) { + root := t.TempDir() + src := `package partial + +import "log" + +func A() { log.Printf("a") } +func B() int { return 1 } +func C(x int) int { return x + 1 } +` + writeFile(t, filepath.Join(root, "partial", "p.go"), src) + + r, err := New(Options{Root: root}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + pc := findPkg(r, "partial") + if pc == nil { + t.Fatalf("package not found") + } + if pc.TotalFuncs != 3 || pc.InstrumentedFuncs != 1 { + t.Fatalf("want 1/3, got %d/%d", pc.InstrumentedFuncs, pc.TotalFuncs) + } + if len(pc.UninstrumentedFuncs) != 2 { + t.Fatalf("want 2 gaps, got %d", len(pc.UninstrumentedFuncs)) + } + names := []string{pc.UninstrumentedFuncs[0].Name, pc.UninstrumentedFuncs[1].Name} + if names[0] != "B" || names[1] != "C" { + t.Fatalf("unexpected gap names: %v", names) + } +} + +func TestAnalyzer_TestFilesExcluded(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "x", "x.go"), `package x +func A() {} +`) + writeFile(t, filepath.Join(root, "x", "x_test.go"), `package x +import "testing" +func TestA(t *testing.T) {} +`) + + r, err := New(Options{Root: root}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + pc := findPkg(r, "x") + if pc == nil || pc.TotalFuncs != 1 { + t.Fatalf("want 1 func, got %+v", pc) + } + + // Re-run including tests. + r2, err := New(Options{Root: root, IncludeTests: true}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + pc2 := findPkg(r2, "x") + if pc2 == nil || pc2.TotalFuncs != 2 { + t.Fatalf("want 2 funcs with tests, got %+v", pc2) + } +} + +func TestAnalyzer_VendoredAndGeneratedExcluded(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "vendor", "lib", "lib.go"), `package lib +func V() {} +`) + writeFile(t, filepath.Join(root, "gen", "x_generated.go"), `package gen +func G() {} +`) + writeFile(t, filepath.Join(root, "gen", "marker.go"), `// Code generated by tool; DO NOT EDIT. +package gen +func M() {} +`) + writeFile(t, filepath.Join(root, "good", "g.go"), `package good +import "log" +func H() { log.Print("ok") } +`) + + r, err := New(Options{Root: root}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + if findPkg(r, filepath.Join("vendor", "lib")) != nil { + t.Fatalf("vendor/lib should be skipped") + } + gen := findPkg(r, "gen") + if gen != nil && gen.TotalFuncs > 0 { + t.Fatalf("generated files should be skipped, got %+v", gen) + } + good := findPkg(r, "good") + if good == nil || good.TotalFuncs != 1 || good.InstrumentedFuncs != 1 { + t.Fatalf("want 1/1 in good, got %+v", good) + } +} + +func TestAnalyzer_ExcludePatterns(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "skipme", "s.go"), `package skipme +func S() {} +`) + writeFile(t, filepath.Join(root, "keep", "k.go"), `package keep +func K() {} +`) + + r, err := New(Options{Root: root, Excludes: []string{"skipme/**"}}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + if findPkg(r, "skipme") != nil { + t.Fatalf("skipme should be excluded") + } + if findPkg(r, "keep") == nil { + t.Fatalf("keep should remain") + } +} + +func TestAnalyzer_CustomPatterns(t *testing.T) { + root := t.TempDir() + src := `package c +func A() { myMetric("a") } +func B() { other("b") } +func myMetric(s string) {} +func other(s string) {} +` + writeFile(t, filepath.Join(root, "c", "c.go"), src) + r, err := New(Options{Root: root, Patterns: []string{"myMetric"}}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + pc := findPkg(r, "c") + if pc == nil { + t.Fatalf("package missing") + } + if pc.InstrumentedFuncs != 1 { + t.Fatalf("want 1 instrumented, got %d", pc.InstrumentedFuncs) + } +} + +func TestAnalyzer_MethodReceiverDisplay(t *testing.T) { + root := t.TempDir() + src := `package r +type T struct{} +func (t *T) Do() { x() } +func x() {} +` + writeFile(t, filepath.Join(root, "r", "r.go"), src) + r, err := New(Options{Root: root}).Analyze() + if err != nil { + t.Fatalf("analyze: %v", err) + } + pc := findPkg(r, "r") + if pc == nil { + t.Fatalf("package missing") + } + foundMethod := false + for _, g := range pc.UninstrumentedFuncs { + if strings.Contains(g.Name, "(T).Do") { + foundMethod = true + } + } + if !foundMethod { + t.Fatalf("expected method gap with receiver, got %+v", pc.UninstrumentedFuncs) + } +} diff --git a/internal/analysis/coverage/report.go b/internal/analysis/coverage/report.go new file mode 100644 index 0000000..ba0c9d5 --- /dev/null +++ b/internal/analysis/coverage/report.go @@ -0,0 +1,70 @@ +package coverage + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sort" +) + +// RenderText writes a human-readable coverage report to w. +func RenderText(w io.Writer, r *OverallCoverage, showGaps int) error { + var buf bytes.Buffer + + fmt.Fprintf(&buf, "Metrics Coverage Report\n") + fmt.Fprintf(&buf, "Root: %s\n\n", r.Root) + + fmt.Fprintf(&buf, "Overall: %d/%d functions instrumented (%.1f%%)\n\n", + r.InstrumentedFuncs, r.TotalFuncs, r.Percent) + + if len(r.Packages) == 0 { + buf.WriteString("No Go packages analyzed.\n") + _, err := w.Write(buf.Bytes()) + return err + } + + pkgs := make([]PackageCoverage, len(r.Packages)) + copy(pkgs, r.Packages) + sort.Slice(pkgs, func(i, j int) bool { + return pkgs[i].Percent < pkgs[j].Percent + }) + + buf.WriteString("Per-package coverage (lowest first):\n") + for _, pc := range pkgs { + fmt.Fprintf(&buf, " %-50s %3d/%-3d %5.1f%%\n", + pc.Dir+" ("+pc.Package+")", pc.InstrumentedFuncs, pc.TotalFuncs, pc.Percent) + } + buf.WriteString("\n") + + if showGaps > 0 { + buf.WriteString("Uninstrumented functions:\n") + for _, pc := range pkgs { + if len(pc.UninstrumentedFuncs) == 0 { + continue + } + fmt.Fprintf(&buf, " %s (%s):\n", pc.Dir, pc.Package) + limit := len(pc.UninstrumentedFuncs) + if showGaps > 0 && limit > showGaps { + limit = showGaps + } + for i := 0; i < limit; i++ { + g := pc.UninstrumentedFuncs[i] + fmt.Fprintf(&buf, " %s:%d %s\n", g.File, g.Line, g.Name) + } + if limit < len(pc.UninstrumentedFuncs) { + fmt.Fprintf(&buf, " ... and %d more\n", len(pc.UninstrumentedFuncs)-limit) + } + } + } + + _, err := w.Write(buf.Bytes()) + return err +} + +// RenderJSON writes the coverage report as indented JSON. +func RenderJSON(w io.Writer, r *OverallCoverage) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(r) +} diff --git a/website/docs/cli-reference.md b/website/docs/cli-reference.md index d5a2cd4..931f8b4 100644 --- a/website/docs/cli-reference.md +++ b/website/docs/cli-reference.md @@ -19,6 +19,7 @@ title: CLI Reference | `nightshift logs` | Stream or export logs | | `nightshift stats` | Token usage statistics | | `nightshift daemon` | Background scheduler | +| `nightshift metrics-coverage` | Analyze metrics instrumentation coverage | ## Run Options diff --git a/website/docs/metrics-coverage.md b/website/docs/metrics-coverage.md new file mode 100644 index 0000000..099abe5 --- /dev/null +++ b/website/docs/metrics-coverage.md @@ -0,0 +1,105 @@ +--- +sidebar_position: 9 +title: Metrics Coverage +--- + +# Metrics Coverage Analyzer + +`nightshift metrics-coverage` statically scans a Go codebase for +instrumentation calls (logging, stats, metrics, telemetry) and reports +per-package coverage. It's a CI-friendly heuristic for spotting code that +ships with no observability. + +## What counts as "instrumented" + +A function is considered instrumented if its body contains at least one +call whose textual form matches one of the configured patterns. The +default pattern set looks for common call sites: + +- Package prefixes: `log.`, `logger.`, `logging.`, `zerolog.`, `stats.`, + `metrics.`, `otel.`, `telemetry.`, `trace.`, `prometheus.`, + `observability.` +- Method suffixes: `.Info`, `.Infof`, `.Warn`, `.Warnf`, `.Error`, + `.Errorf`, `.Debug`, `.Debugf`, `.Trace`, `.Tracef`, `.Record`, + `.Emit`, `.Observe`, `.Inc` + +Override the set entirely with one or more `--pattern` flags. + +Test files, `vendor/`, `node_modules/`, `.git/`, and files marked with the +standard `Code generated ... DO NOT EDIT` comment are skipped by default. +Use `--include-tests` to include `*_test.go`. + +## Usage + +```bash +# Analyze the current directory +nightshift metrics-coverage + +# Limit to a subtree, output JSON +nightshift metrics-coverage --path ./internal --format json + +# CI gate: fail if overall coverage drops below 60% +nightshift metrics-coverage --min-coverage 60 + +# Exclude directories with glob patterns +nightshift metrics-coverage --exclude 'vendor/**' --exclude '**/mocks/**' + +# Use custom patterns (e.g. an in-house metrics package) +nightshift metrics-coverage --pattern 'myco/obs.' --pattern '.RecordMetric' +``` + +### Flags + +| Flag | Default | Description | +|------|---------|-------------| +| `--path`, `-p` | current dir | Directory to analyze (recursively) | +| `--format` | `text` | Output format: `text` or `json` | +| `--min-coverage` | `0` | Fail if overall coverage is below this percent | +| `--exclude` | | Glob pattern of relative paths to skip (repeatable) | +| `--pattern` | _defaults_ | Override default instrumentation patterns (repeatable) | +| `--include-tests` | `false` | Include `*_test.go` files | +| `--max-gaps` | `10` | Cap on gaps listed per package (text only; `0` hides) | +| `--output`, `-o` | stdout | Write report to file | + +## Output + +Text output sorts packages from lowest to highest coverage and lists the +first uninstrumented functions per package with file and line: + +``` +Metrics Coverage Report +Root: /repo/internal + +Overall: 142/210 functions instrumented (67.6%) + +Per-package coverage (lowest first): + budget (budget) 12/30 40.0% + scheduler (scheduler) 18/25 72.0% + ... + +Uninstrumented functions: + budget (budget): + budget.go:88 (Manager).snapshot + budget.go:124 normalize + ... +``` + +JSON output is the full `OverallCoverage` struct — suitable for piping into +`jq` or attaching as a CI artifact. + +## Interpretation + +Coverage percentage is a rough signal, not a target. A low number on a +pure-data package (constants, struct definitions, getters) is fine; a low +number on a request-handling or job-orchestration package usually warrants +attention. + +Useful workflows: + +- **Spot dark code paths**: sort by lowest coverage; eyeball the gap list + for any function that owns I/O, retries, or state transitions but has no + logs. +- **Prevent regressions**: run with `--min-coverage` in CI on packages + where you've already invested in observability. +- **Tune patterns**: if you have an in-house wrapper (e.g. + `obs.RecordLatency`), pass `--pattern` so it counts.