diff --git a/CHANGELOG.md b/CHANGELOG.md index 788a61d..8665760 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,12 @@ All notable changes to DebtLens are documented here. This project adheres to ### Added - `debtlens history record` / `history show` with `.debtlens/history.jsonl` ledger and timeline reports. -- Payoff ranking via `--sort payoff`, JSON `payoffScore`, and top-payoff report sections. -- `debtlens calibrate` for percentile-based threshold suggestions with optional `--write`. -- Interactive `debtlens triage` for keep/baseline/suppress workflows. +- Payoff ranking via `--sort payoff`, JSON `payoffScore`, and top-payoff report sections + ([#251](https://github.com/ColumbusLabs/DebtLens/issues/251)). +- `debtlens calibrate` for percentile-based threshold suggestions with optional `--write` + ([#256](https://github.com/ColumbusLabs/DebtLens/issues/256)). +- Interactive `debtlens triage` for keep/baseline/suppress workflows + ([#255](https://github.com/ColumbusLabs/DebtLens/issues/255)). - `debtlens fix` dry-run autofix allowlist for duplicated literals. - Opt-in `feature-flags` pack with `stale-feature-flag` detector. - `--concurrency` and `--cache-dir` scan controls for large-repo performance. @@ -18,7 +21,8 @@ All notable changes to DebtLens are documented here. This project adheres to - HTML reports now include import-graph SVG and directory debt treemap views. - Core rules: `long-parameter-list`, `god-file`, and `cognitive-complexity` for function/module design smells beyond single-axis size checks. -- Config `budgets` block and `debtlens scan --budget-report` for per-area debt SLO gating. +- Config `budgets` block and `debtlens scan --budget-report` for per-area debt SLO gating + ([#252](https://github.com/ColumbusLabs/DebtLens/issues/252)). - `debtlens scan --format badge` emits a self-contained SVG badge plus shields.io endpoint JSON. - Scan results now warn when matched files exceed `maxFiles`; terminal output prints the advisory and JSON reports include it in `summary.warnings` while keeping diff --git a/README.md b/README.md index 996d38a..a465c53 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,8 @@ debtlens init --pack core # starter config using the core rule pack preset debtlens init --policy @org/debtlens-policy # starter config from a shared policy package debtlens init --from-eslint eslint.config.json # print a migration suggestion without writing debtlens adopt # adoption report (dry run; recommends minSeverity) +debtlens calibrate . # suggest percentile-based threshold overrides +debtlens triage . # interactively keep, baseline, or suppress findings debtlens watch examples/react --rules todo-comment # rescan on file changes debtlens completions zsh # print shell completions debtlens mcp # stdio MCP server for Cursor/Claude-style agents @@ -343,6 +345,39 @@ The second command writes `debtlens.config.json` and `debtlens-baseline.json` (b For serious open-source repositories and broad monorepos, treat adoption as a scoped rollout instead of a whole-repo gate on day one. Run `adopt` first, then start with `--changed origin/main` or a maintained source subdirectory, add `--package` for workspaces, keep generated/dependency outputs excluded, and narrow expensive or noisy rules with `--rules`, thresholds, baselines, or confidence floors before widening coverage. If the default 2,000-file cap appears, either raise it with `--max-files` for a deliberate full scan or make the target more precise with `--package`, `--include`, `--exclude`, `--changed`, or `--respect-gitignore`. +### Tune, prioritize, triage, and budget + +After `adopt` picks rules, use these commands to finish a low-noise rollout: + +```bash +# 1. Tune numeric thresholds to your repo's distributions +debtlens calibrate . --percentile 90 +debtlens calibrate . --percentile 90 --write # merge into debtlens.config.json + +# 2. Review highest-ROI findings first (enable hotspots when git history is available) +debtlens scan . --sort payoff --hotspots + +# 3. Walk the backlog interactively (requires a TTY; use --dry-run to preview) +debtlens triage . + +# 4. Protect cleaned areas with per-path budgets in debtlens.config.json +debtlens scan . --budget-report +debtlens scan . --fail-on high # budget breaches also fail the gate +``` + +Example `budgets` block: + +```json +{ + "budgets": { + "src/payments": { "maxIssues": 20, "maxHigh": 0 }, + "src/legacy": { "maxIssues": 250 } + } +} +``` + +See [`docs/prioritization.md`](./docs/prioritization.md) for the payoff formula, `priority` weights, and budget glob matching. + Named quality-gate presets give teams a shared rollout vocabulary: | Preset | Use when | Default behavior | diff --git a/docs/false-positives.md b/docs/false-positives.md index 6381c34..f0402d0 100644 --- a/docs/false-positives.md +++ b/docs/false-positives.md @@ -14,6 +14,30 @@ adding inline suppressions. | Low-confidence finding family | `ruleConfidenceFloors` or `--fail-on-confidence` | Keeps findings visible while preventing weak gates. | | One documented exception | Inline suppression with a reason | Auditable, local, and visible in JSON/SARIF output. | +## Calibrate thresholds to your repo + +Default thresholds are deliberately generic. On a large legacy repo they can be noisy; on a small repo they can miss real debt. After `debtlens adopt` picks rules and severity, run calibration to tune the numbers: + +```bash +# Preview percentile-based threshold suggestions (default p90) +debtlens calibrate . + +# Tune aggressiveness and merge suggestions into debtlens.config.json +debtlens calibrate . --percentile 85 --write +``` + +Calibration scans the target, collects observed metrics for threshold-driven rules (function length, branch counts, and similar), and suggests values at the chosen percentile so roughly the worst N% is flagged. Review the printed config snippet before using `--write`; unrelated config keys are preserved. + +Pair calibration with payoff ranking and triage for a low-noise first rollout: + +```bash +debtlens calibrate . --percentile 90 +debtlens scan . --sort payoff --hotspots +debtlens triage . +``` + +See [`docs/prioritization.md`](./prioritization.md) for payoff scoring and per-area budgets. + ## Baseline before suppressing ```bash diff --git a/docs/prioritization.md b/docs/prioritization.md new file mode 100644 index 0000000..e1c5789 --- /dev/null +++ b/docs/prioritization.md @@ -0,0 +1,85 @@ +# Payoff prioritization + +DebtLens can rank findings by **payoff score** so teams fix the debt that costs the most first instead of working through a flat severity list. + +## When scores appear + +Each issue gets a `payoffScore` in JSON output when payoff ranking is enabled: + +- pass `--sort payoff` on `debtlens scan`, or +- enable git churn hotspots (`hotspots: true` in config or `--hotspots` on the CLI), or +- pass `--blame-age` to include age in the score. + +Terminal, Markdown, and HTML reports include a **Top payoff targets** section when any issue carries a score. + +## Scoring formula + +Payoff combines signals already present on each finding: + +``` +payoffScore = severityWeight × confidence × churnFactor × ageFactor +``` + +| Factor | Default behavior | +| --- | --- | +| `severityWeight` | high 16, medium 8, low 3, info 1 | +| `confidence` | issue confidence, floored at 0.35 | +| `churnFactor` | `1 + log2(1 + churnMetric) × churnWeight` when hotspot data exists; otherwise 1 | +| `ageFactor` | `1 + min(introducedDaysAgo / 365, 2) × ageWeight` when blame age is available; otherwise 1 | + +Churn metrics come from [`src/core/hotspots.ts`](../src/core/hotspots.ts). Age uses optional `introducedDaysAgo` from `--blame-age`. + +## CLI usage + +```bash +# Sort findings by payoff and show top targets in the terminal report +debtlens scan . --sort payoff --hotspots + +# JSON consumers read payoffScore on each issue +debtlens scan . --sort payoff --format json +``` + +Use `--hotspots` (or config `hotspots: true`) when git history is available so churn boosts files that change often. In CI, check out enough history (`fetch-depth: 0` or a bounded `--churn-range`). + +## Config weights + +Tune weights under the `priority` block in `debtlens.config.json`: + +```json +{ + "priority": { + "severity": { "high": 20, "medium": 10, "low": 4, "info": 1 }, + "churn": 1.2, + "age": 0.75 + } +} +``` + +Higher `churn` and `age` weights amplify those factors. Severity weights follow the same keys as built-in severities. + +## Adoption workflow + +Payoff ranking fits between calibration and triage: + +1. `debtlens calibrate` — tune thresholds to your repo (see [`docs/false-positives.md`](./false-positives.md)). +2. `debtlens scan . --sort payoff --hotspots` — review the highest-ROI findings first. +3. `debtlens triage` — baseline or suppress the backlog interactively. +4. Add `budgets` — cap debt per directory after cleanup (see below). + +## Per-area budgets + +After triage, protect cleaned areas with path budgets: + +```json +{ + "budgets": { + "src/payments": { "maxIssues": 20, "maxHigh": 0 }, + "src/legacy": { "maxIssues": 250 } + } +} +``` + +- Budget breaches fail the scan gate (exit code 1) like `--fail-on`. +- `debtlens scan --budget-report` prints used/budget/headroom without failing — useful in CI dashboards. + +Keys are path globs (`src/payments/**`, `src/**/*.ts`, or exact prefixes). diff --git a/src/cli/commands/calibrate.ts b/src/cli/commands/calibrate.ts index 8b4cb9a..5da0b10 100644 --- a/src/cli/commands/calibrate.ts +++ b/src/cli/commands/calibrate.ts @@ -9,6 +9,9 @@ export function registerCalibrateCommand(program: Command): void { .argument("[target]", "directory or file to scan", ".") .option("--cwd ", "working directory", process.cwd()) .option("--config ", "path to debtlens.config.json") + .option("--pack ", "rule pack preset to scan with") + .option("--rules ", "comma-separated rule ids to run") + .option("--threshold ", "comma-separated key=value threshold overrides") .option("--percentile ", "percentile used for suggestions (50-99)", parseInteger) .option("--write", "merge suggested thresholds into debtlens.config.json") .action(async (target: string, rawOptions: Record) => { diff --git a/src/cli/triage.ts b/src/cli/triage.ts index a2b3637..b1ea20c 100644 --- a/src/cli/triage.ts +++ b/src/cli/triage.ts @@ -20,6 +20,8 @@ export interface TriageInput { cliOptions?: Record; input?: NodeJS.ReadableStream; output?: NodeJS.WritableStream; + /** Injectable prompt for tests; defaults to readline when omitted. */ + ask?: (message: string) => Promise; } export interface TriageActionResult { @@ -56,18 +58,22 @@ export async function runTriage(input: TriageInput): Promise const suppressions: string[] = []; const counts: TriageActionResult = { kept: 0, baselined: 0, suppressed: 0, skipped: 0 }; - const rl = createInterface({ - input: input.input ?? process.stdin, - output: input.output ?? process.stdout, - }); + const output = input.output ?? process.stdout; + const rl = input.ask + ? undefined + : createInterface({ + input: input.input ?? process.stdin, + output, + }); + const ask = input.ask ?? ((message: string) => rl!.question(message)); try { for (let index = 0; index < issues.length; index += 1) { const issue = issues[index]; if (!issue) continue; const rendered = formatIssue(issue, index + 1, issues.length); - process.stdout.write(`\n${rendered}\n`); - const rawAnswer = (await rl.question("Action [k]eep [b]aseline [s]uppress [n]ext [q]uit [B]atch rule: ")).trim(); + output.write(`\n${rendered}\n`); + const rawAnswer = (await ask("Action [k]eep [b]aseline [s]uppress [n]ext [q]uit [B]atch rule: ")).trim(); const answer = rawAnswer.toLowerCase(); if (answer === "q" || answer === "quit") break; @@ -76,7 +82,10 @@ export async function runTriage(input: TriageInput): Promise continue; } if (rawAnswer === "B" || answer === "batch" || answer === "b-rule") { - const batchAction = (await rl.question("Apply to all remaining findings of this rule with [k]eep [b]aseline [s]uppress? ")).trim().toLowerCase(); + const batchAction = (await ask("Apply to all remaining findings of this rule with [k]eep [b]aseline [s]uppress? ")).trim().toLowerCase(); + const suppressReason = isSuppressAction(batchAction) + ? await promptSuppressReason(ask, output) + : undefined; for (let cursor = index; cursor < issues.length; cursor += 1) { const candidate = issues[cursor]; if (!candidate || candidate.ruleId !== issue.ruleId) continue; @@ -86,29 +95,37 @@ export async function runTriage(input: TriageInput): Promise fingerprints, suppressions, counts, + suppressReason, }); } break; } + const suppressReason = isSuppressAction(answer) + ? await promptSuppressReason(ask, output) + : undefined; applyTriageAction(issue, answer, { dryRun: input.dryRun, baseline, fingerprints, suppressions, counts, + suppressReason, }); } } finally { - rl.close(); + rl?.close(); } if (!input.dryRun) { writeBaseline(cwd, baselinePath, baseline); if (suppressions.length > 0) { - process.stdout.write("\nSuggested suppression directives:\n"); - for (const directive of suppressions) process.stdout.write(directive); + output.write("\nSuggested suppression directives:\n"); + for (const directive of suppressions) output.write(directive); } + } else if (suppressions.length > 0) { + output.write("\nSuggested suppression directives (dry run):\n"); + for (const directive of suppressions) output.write(directive); } return counts; @@ -123,6 +140,7 @@ function applyTriageAction( fingerprints: Set; suppressions: string[]; counts: TriageActionResult; + suppressReason?: string; }, ): void { const fingerprint = issue.fingerprint ?? issue.id; @@ -133,7 +151,10 @@ function applyTriageAction( return; } if (action === "s" || action === "suppress") { - const reason = "triaged via debtlens triage"; + const reason = context.suppressReason?.trim(); + if (!reason) { + throw new Error("Suppression reason is required."); + } context.suppressions.push(runSuppress({ ruleId: issue.ruleId, reason })); context.counts.suppressed += 1; return; @@ -141,6 +162,21 @@ function applyTriageAction( context.counts.kept += 1; } +function isSuppressAction(action: string): boolean { + return action === "s" || action === "suppress"; +} + +async function promptSuppressReason( + ask: (message: string) => Promise, + output: NodeJS.WritableStream, +): Promise { + while (true) { + const reason = (await ask("Suppression reason (required): ")).trim(); + if (reason) return reason; + output.write("A reason is required for suppressions.\n"); + } +} + function formatIssue(issue: DebtIssue, index: number, total: number): string { const location = issue.location ? `${issue.file}:${issue.location.startLine}` : issue.file; return [ diff --git a/tests/cli/calibrate.test.ts b/tests/cli/calibrate.test.ts new file mode 100644 index 0000000..7c9f18c --- /dev/null +++ b/tests/cli/calibrate.test.ts @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "node:test"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const cliEntrypoint = join(repoRoot, "src", "cli", "index.ts"); +const localRequire = createRequire(import.meta.url); +const tsxLoader = localRequire.resolve("tsx"); + +const lowReactThresholds = [ + "--threshold", + "large-component.maxLines=50,large-component.maxHooks=3,large-component.maxBranches=5", +]; + +function runCli(args: string[], options: { cwd?: string } = {}) { + return spawnSync(process.execPath, ["--import", tsxLoader, cliEntrypoint, ...args], { + cwd: options.cwd ?? repoRoot, + encoding: "utf8", + }); +} + +describe("debtlens calibrate", () => { + it("prints percentile-based threshold suggestions", () => { + const result = runCli([ + "calibrate", + "examples/react", + "--pack", + "react", + ...lowReactThresholds, + "--percentile", + "90", + ]); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /DebtLens calibrate \(p90\)/); + assert.match(result.stdout, /large-component\.maxBranches/); + assert.match(result.stdout, /Suggested config snippet/); + }); + + it("merges suggested thresholds with --write", () => { + const dir = mkdtempSync(join(tmpdir(), "debtlens-calibrate-write-")); + try { + const configPath = join(dir, "debtlens.config.json"); + writeFileSync(configPath, JSON.stringify({ + pack: "react", + rules: ["large-component"], + minSeverity: "low", + thresholds: { + "large-component.maxLines": 50, + "large-component.maxHooks": 3, + "large-component.maxBranches": 5, + }, + })); + + const result = runCli([ + "calibrate", + "examples/react", + "--cwd", + repoRoot, + "--config", + configPath, + "--pack", + "react", + ...lowReactThresholds, + "--write", + ]); + assert.equal(result.status, 0, result.stderr); + + const config = JSON.parse(readFileSync(configPath, "utf8")) as { + minSeverity: string; + thresholds: Record; + }; + assert.equal(config.minSeverity, "low"); + assert.ok(config.thresholds["large-component.maxLines"] > 50); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/cli/scan.test.ts b/tests/cli/scan.test.ts index 8f477d0..510ab59 100644 --- a/tests/cli/scan.test.ts +++ b/tests/cli/scan.test.ts @@ -427,6 +427,38 @@ describe("debtlens scan output formats", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("sorts findings by payoff and prints top payoff targets", () => { + const result = runScan([ + "examples/react", + "--rules", + "todo-comment,prop-drilling", + "--sort", + "payoff", + "--format", + "json", + ]); + + assert.equal(result.status, 0); + const parsed = JSON.parse(result.stdout) as { issues: Array<{ payoffScore?: number }> }; + assert.ok(parsed.issues.length > 1); + assert.ok(parsed.issues.every((issue) => issue.payoffScore !== undefined)); + for (let index = 1; index < parsed.issues.length; index += 1) { + const previous = parsed.issues[index - 1]?.payoffScore ?? 0; + const current = parsed.issues[index]?.payoffScore ?? 0; + assert.ok(previous >= current); + } + + const terminal = runScan([ + "examples/react", + "--rules", + "todo-comment", + "--sort", + "payoff", + ]); + assert.equal(terminal.status, 0); + assert.match(terminal.stdout, /Top payoff targets/); + }); }); describe("debtlens scan fail-on confidence", () => { diff --git a/tests/cli/triage.test.ts b/tests/cli/triage.test.ts index 7e96225..797f757 100644 --- a/tests/cli/triage.test.ts +++ b/tests/cli/triage.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Readable, Writable } from "node:stream"; @@ -27,4 +27,58 @@ describe("debtlens triage", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("baselines a finding when requested", async () => { + const dir = mkdtempSync(join(tmpdir(), "debtlens-triage-baseline-")); + try { + mkdirSync(join(dir, "src")); + writeFileSync(join(dir, "src", "app.ts"), "// TODO triage me\nexport const value = 1;\n"); + const baselinePath = join(dir, "debtlens-baseline.json"); + + const counts = await runTriage({ + target: ".", + cwd: dir, + baselinePath, + cliOptions: { rules: "todo-comment" }, + input: Readable.from(["b\n", "q\n"]), + output: new Writable({ write(_chunk, _encoding, callback) { callback(); } }), + }); + + assert.deepEqual(counts, { kept: 0, baselined: 1, suppressed: 0, skipped: 0 }); + const baseline = JSON.parse(readFileSync(baselinePath, "utf8")) as { fingerprints: Record }; + assert.equal(Object.keys(baseline.fingerprints).length, 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("prompts for a suppression reason", async () => { + const dir = mkdtempSync(join(tmpdir(), "debtlens-triage-suppress-")); + try { + mkdirSync(join(dir, "src")); + writeFileSync(join(dir, "src", "app.ts"), "// TODO triage me\nexport const value = 1;\n"); + const output: string[] = []; + const answers = ["s", "tracked in PROJ-42"]; + const out = new Writable({ + write(chunk, _encoding, callback) { + output.push(String(chunk)); + callback(); + }, + }); + + const counts = await runTriage({ + target: ".", + cwd: dir, + dryRun: true, + cliOptions: { rules: "todo-comment" }, + output: out, + ask: async () => answers.shift() ?? "", + }); + + assert.deepEqual(counts, { kept: 0, baselined: 0, suppressed: 1, skipped: 0 }); + assert.match(output.join(""), /debtlens-disable-next-line todo-comment -- tracked in PROJ-42/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); });