diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index 867a339..93869f1 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -12,3 +12,13 @@ {"id":"int-5369ba6b","kind":"field_change","created_at":"2026-04-07T13:33:11.931574Z","actor":"Ken Judy","issue_id":"code-quality-metrics-a6p","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"pr-metrics.yml refactored: inline helpers removed, require() added, isTestFile() replaces 4x regex, CONFIG.* thresholds, classifyDoraArchetype signature fixed"}} {"id":"int-7a66b645","kind":"field_change","created_at":"2026-04-07T13:34:14.142186Z","actor":"Ken Judy","issue_id":"code-quality-metrics-ac0","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"CLAUDE.md updated: architecture section reflects shared lib/, config section notes single source of truth, 'update both places' warning removed, net additions ratio threshold updated"}} {"id":"int-6b197fa6","kind":"field_change","created_at":"2026-04-07T13:34:39.195467Z","actor":"Ken Judy","issue_id":"code-quality-metrics-22h","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"lint clean, typecheck clean, 119/119, 96.21%/96.42% coverage โ€” all thresholds met"}} +{"id":"int-bfbb744e","kind":"field_change","created_at":"2026-04-07T14:19:17.960966Z","actor":"Ken Judy","issue_id":"code-quality-metrics-136","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}} +{"id":"int-ec619666","kind":"field_change","created_at":"2026-04-07T14:19:18.535756Z","actor":"Ken Judy","issue_id":"code-quality-metrics-0jd","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-4c8966f4","kind":"field_change","created_at":"2026-04-07T14:19:19.087859Z","actor":"Ken Judy","issue_id":"code-quality-metrics-9ji","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-d3f400f4","kind":"field_change","created_at":"2026-04-07T15:03:20.883507Z","actor":"Ken Judy","issue_id":"code-quality-metrics-4j6","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-0f17aa50","kind":"field_change","created_at":"2026-04-07T15:07:12.425052Z","actor":"Ken Judy","issue_id":"code-quality-metrics-g0a","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-3ad9c87f","kind":"field_change","created_at":"2026-04-07T15:11:38.627913Z","actor":"Ken Judy","issue_id":"code-quality-metrics-84l","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-8359fc75","kind":"field_change","created_at":"2026-04-07T15:14:28.801592Z","actor":"Ken Judy","issue_id":"code-quality-metrics-yuf","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-67abf2c2","kind":"field_change","created_at":"2026-04-07T15:19:31.62249Z","actor":"Ken Judy","issue_id":"code-quality-metrics-38c","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-1177d69a","kind":"field_change","created_at":"2026-04-07T15:23:05.721325Z","actor":"Ken Judy","issue_id":"code-quality-metrics-moo","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} +{"id":"int-ac0d33c7","kind":"field_change","created_at":"2026-04-07T15:23:06.286044Z","actor":"Ken Judy","issue_id":"code-quality-metrics-cek","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}} diff --git a/.github/workflows/code-metrics.yml b/.github/workflows/code-metrics.yml index 5b68ae2..539a968 100644 --- a/.github/workflows/code-metrics.yml +++ b/.github/workflows/code-metrics.yml @@ -106,7 +106,9 @@ jobs: branch_commit_counts: branchCommitCounts, large_commits_pct: "0.00", sprawling_commits_pct: "0.00", - test_first_pct: "0.00", + test_coverage_rate: "0.00", + test_isolation_rate: "0.00", + uncovered_prod_rate: "0.00", avg_files_changed: "0.00", avg_lines_changed: "0.00", avg_prod_lines_changed: "0.00", @@ -156,6 +158,8 @@ jobs: test_files_count: testFiles.length, prod_files_count: prodFiles.length, test_first_indicator: testFiles.length > 0 && prodFiles.length > 0, + test_only_commit: testFiles.length > 0 && prodFiles.length === 0, + uncovered_prod_commit: testFiles.length === 0 && prodFiles.length > 0 && (prodFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0)) > CONFIG.LARGE_COMMIT_THRESHOLD, prod_additions: prodFiles.reduce((sum, f) => sum + f.additions, 0), prod_deletions: prodFiles.reduce((sum, f) => sum + f.deletions, 0), large_commit: (prodFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0)) > CONFIG.LARGE_COMMIT_THRESHOLD, @@ -192,7 +196,9 @@ jobs: const largePct = parseFloat((metrics.filter(m => m.large_commit).length / metrics.length * 100).toFixed(2)); const sprawlingPct = parseFloat((metrics.filter(m => m.sprawling_commit).length / metrics.length * 100).toFixed(2)); - const testFirstPct = parseFloat((metrics.filter(m => m.test_first_indicator).length / metrics.length * 100).toFixed(2)); + const testCoveragePct = parseFloat((metrics.filter(m => m.test_first_indicator).length / metrics.length * 100).toFixed(2)); + const testIsolationPct = parseFloat((metrics.filter(m => m.test_only_commit).length / metrics.length * 100).toFixed(2)); + const uncoveredProdPct = parseFloat((metrics.filter(m => m.uncovered_prod_commit).length / metrics.length * 100).toFixed(2)); const summary = { total_commits: metrics.length, @@ -201,7 +207,9 @@ jobs: branch_commit_counts: branchCommitCounts, large_commits_pct: largePct.toFixed(2), sprawling_commits_pct: sprawlingPct.toFixed(2), - test_first_pct: testFirstPct.toFixed(2), + test_coverage_rate: testCoveragePct.toFixed(2), + test_isolation_rate: testIsolationPct.toFixed(2), + uncovered_prod_rate: uncoveredProdPct.toFixed(2), avg_files_changed: (metrics.reduce((sum, m) => sum + m.files_changed, 0) / metrics.length).toFixed(2), avg_lines_changed: (metrics.reduce((sum, m) => sum + (m.prod_additions || 0) + (m.prod_deletions || 0), 0) / metrics.length).toFixed(2), avg_prod_lines_changed: (metrics.reduce((sum, m) => sum + (m.prod_additions || 0) + (m.prod_deletions || 0), 0) / metrics.length).toFixed(2), @@ -216,7 +224,7 @@ jobs: net_additions_ratio_median: parseFloat(ratioStats.p50.toFixed(2)), net_additions_ratio_p90: parseFloat(ratioStats.p90.toFixed(2)), message_quality_pct: msgQualityPct, - dora_archetype: classifyDoraArchetype({ large_commits_pct: largePct.toFixed(2), sprawling_commits_pct: sprawlingPct.toFixed(2), test_first_pct: testFirstPct.toFixed(2), message_quality_pct: msgQualityPct }), + dora_archetype: classifyDoraArchetype({ large_commits_pct: largePct.toFixed(2), sprawling_commits_pct: sprawlingPct.toFixed(2), test_coverage_rate: testCoveragePct.toFixed(2), uncovered_prod_rate: uncoveredProdPct.toFixed(2), message_quality_pct: msgQualityPct }), note: "Feature branches only - main/master excluded. Large commit threshold applied to production code only." }; @@ -281,7 +289,9 @@ jobs: '|--------|-------|--------|--------|', `| Large Commits (>100 prod lines) | ${summary.large_commits_pct}% | <20% | ${statusMark(summary.large_commits_pct, 20)} |`, `| Sprawling Commits (>5 files) | ${summary.sprawling_commits_pct}% | <10% | ${statusMark(summary.sprawling_commits_pct, 10)} |`, - `| Test-First Discipline | ${summary.test_first_pct}% | >50% | ${statusMark(summary.test_first_pct, 50, 'above')} |`, + `| Test Coverage (test+prod commits) | ${summary.test_coverage_rate}% | >50% | ${statusMark(summary.test_coverage_rate, 50, 'above')} |`, + `| Test Isolation (test-only commits) | ${summary.test_isolation_rate}% | >10% | ${statusMark(summary.test_isolation_rate, 10, 'above')} |`, + `| Uncovered Prod (large, no tests) | ${summary.uncovered_prod_rate}% | <10% | ${statusMark(summary.uncovered_prod_rate, 10)} |`, `| Message Quality | ${summary.message_quality_pct}% | >60% | ${statusMark(summary.message_quality_pct, 60, 'above')} |`, `| Net Additions Ratio (median) | ${summary.net_additions_ratio_median} | <0.50 | ${parseFloat(summary.net_additions_ratio_median) < 0.50 ? 'OK' : 'Warning'} |`, '', diff --git a/.github/workflows/pr-metrics.yml b/.github/workflows/pr-metrics.yml index 36bf056..540c7d9 100644 --- a/.github/workflows/pr-metrics.yml +++ b/.github/workflows/pr-metrics.yml @@ -5,6 +5,7 @@ on: jobs: analyze-pr: runs-on: ubuntu-latest + if: github.actor != 'dependabot[bot]' steps: - name: Checkout with full history uses: actions/checkout@v4 @@ -81,6 +82,8 @@ jobs: large_commit: prodChanges > CONFIG.LARGE_COMMIT_THRESHOLD, sprawling_commit: (detail.data.files?.length || 0) > CONFIG.SPRAWLING_COMMIT_THRESHOLD, test_first_indicator: commitTestFiles.length > 0 && commitProdFiles.length > 0, + test_only_commit: commitTestFiles.length > 0 && commitProdFiles.length === 0, + uncovered_prod_commit: commitTestFiles.length === 0 && commitProdFiles.length > 0 && prodChanges > CONFIG.LARGE_COMMIT_THRESHOLD, test_only: commitTestFiles.length > 0 && commitProdFiles.length === 0, prod_only: commitProdFiles.length > 0 && commitTestFiles.length === 0, test_ratio: prodChanges > 0 ? (testChanges / prodChanges) : 0, @@ -131,6 +134,8 @@ jobs: const largePct = commitMetrics.length > 0 ? (largeCommits / commitMetrics.length * 100) : 0; const sprawlingPct = commitMetrics.length > 0 ? (sprawlingCommits / commitMetrics.length * 100) : 0; const testFirstPct = commitMetrics.length > 0 ? (testFirstCommits / commitMetrics.length * 100) : 0; + const testCoveragePct = testFirstPct; + const uncoveredProdPct = commitMetrics.length > 0 ? (commitMetrics.filter(c => c.uncovered_prod_commit).length / commitMetrics.length * 100) : 0; const msgQualityPct = commitMetrics.length > 0 ? (qualityCommits / commitMetrics.length * 100) : 0; const medianAdditionsRatio = (() => { @@ -250,7 +255,7 @@ jobs: // --------------------------------------------------------------- const archetype = commitMetrics.length > 0 - ? classifyDoraArchetype({ large_commits_pct: largePct.toFixed(2), sprawling_commits_pct: sprawlingPct.toFixed(2), test_first_pct: testFirstPct.toFixed(2), message_quality_pct: msgQualityPct.toFixed(2) }) + ? classifyDoraArchetype({ large_commits_pct: largePct.toFixed(2), sprawling_commits_pct: sprawlingPct.toFixed(2), test_coverage_rate: testCoveragePct.toFixed(2), uncovered_prod_rate: uncoveredProdPct.toFixed(2), message_quality_pct: msgQualityPct.toFixed(2) }) : 'mixed-signals'; const archetypeDescriptions = { diff --git a/AGENTS.md b/AGENTS.md index 224e831..b4cdc8f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -125,6 +125,8 @@ For more details, see README.md and docs/QUICKSTART.md. Run these after each implementation step, not only at the end of a session: +**When renaming a field that appears in another file's JSDoc `@param` type**, update the JSDoc in the same step as the rename โ€” do not defer to a later step. A transient typecheck failure between steps is a process violation. Run `npm run typecheck` immediately after any rename before proceeding. + ```bash npm test # all tests must pass before moving to next step npm run lint # lint must be clean before moving to next step diff --git a/__tests__/analyzeCommit.test.js b/__tests__/analyzeCommit.test.js index b76f98a..3424591 100644 --- a/__tests__/analyzeCommit.test.js +++ b/__tests__/analyzeCommit.test.js @@ -102,4 +102,53 @@ describe('analyzeCommit', () => { execSync.mockReturnValue(numstatLine(5, 2, 'src/app.js')); expect(analyzeCommit(MOCK_SHA, MOCK_BRANCH).source_branch).toBe(MOCK_BRANCH); }); + + // --- test_only_commit --- + test('sets test_only_commit true when only test files changed', () => { + execSync.mockReturnValue(numstatLine(10, 2, 'src/app.test.js')); + const result = analyzeCommit(MOCK_SHA, MOCK_BRANCH); + expect(result.test_only_commit).toBe(true); + }); + + test('sets test_only_commit false when both test and prod files changed', () => { + execSync.mockReturnValue([ + numstatLine(10, 2, 'src/app.js'), + numstatLine(5, 1, 'src/app.test.js') + ].join('\n')); + const result = analyzeCommit(MOCK_SHA, MOCK_BRANCH); + expect(result.test_only_commit).toBe(false); + }); + + test('sets test_only_commit false when only prod files changed', () => { + execSync.mockReturnValue(numstatLine(20, 5, 'src/app.js')); + expect(analyzeCommit(MOCK_SHA, MOCK_BRANCH).test_only_commit).toBe(false); + }); + + // --- uncovered_prod_commit --- + test('sets uncovered_prod_commit true when only large prod commit with no tests', () => { + const lines = CONFIG.LARGE_COMMIT_THRESHOLD + 1; + execSync.mockReturnValue(numstatLine(lines, 0, 'src/app.js')); + const result = analyzeCommit(MOCK_SHA, MOCK_BRANCH); + expect(result.uncovered_prod_commit).toBe(true); + }); + + test('sets uncovered_prod_commit false when prod-only commit is not large', () => { + execSync.mockReturnValue(numstatLine(10, 0, 'src/app.js')); + expect(analyzeCommit(MOCK_SHA, MOCK_BRANCH).uncovered_prod_commit).toBe(false); + }); + + test('sets uncovered_prod_commit false when large commit includes test files', () => { + const lines = CONFIG.LARGE_COMMIT_THRESHOLD + 1; + execSync.mockReturnValue([ + numstatLine(lines, 0, 'src/app.js'), + numstatLine(5, 0, 'src/app.test.js') + ].join('\n')); + expect(analyzeCommit(MOCK_SHA, MOCK_BRANCH).uncovered_prod_commit).toBe(false); + }); + + test('sets uncovered_prod_commit false when large commit is test-only', () => { + const lines = CONFIG.LARGE_COMMIT_THRESHOLD + 1; + execSync.mockReturnValue(numstatLine(lines, 0, 'src/app.test.js')); + expect(analyzeCommit(MOCK_SHA, MOCK_BRANCH).uncovered_prod_commit).toBe(false); + }); }); diff --git a/__tests__/collectLocalMetrics.test.js b/__tests__/collectLocalMetrics.test.js index 59dd214..2c3b139 100644 --- a/__tests__/collectLocalMetrics.test.js +++ b/__tests__/collectLocalMetrics.test.js @@ -148,6 +148,17 @@ describe('collectLocalMetrics โ€” successful run', () => { .toContain(summary.dora_archetype); }); + test('writes local_metrics_summary.json with three-way test classification rates', async () => { + await collectLocalMetrics(); + + const summaryCall = fs.writeFileSync.mock.calls.find(c => c[0].includes('local_metrics_summary')); + const summary = JSON.parse(summaryCall[1]); + expect(typeof summary.test_coverage_rate).toBe('string'); + expect(typeof summary.test_isolation_rate).toBe('string'); + expect(typeof summary.uncovered_prod_rate).toBe('string'); + expect(summary.test_first_pct).toBeUndefined(); + }); + test('writes local_metrics_summary.json with statistical distribution fields', async () => { await collectLocalMetrics(); diff --git a/__tests__/doraArchetype.test.js b/__tests__/doraArchetype.test.js index e6de7ff..c625536 100644 --- a/__tests__/doraArchetype.test.js +++ b/__tests__/doraArchetype.test.js @@ -7,16 +7,28 @@ describe('classifyDoraArchetype', () => { expect(classifyDoraArchetype({ large_commits_pct: '10.00', sprawling_commits_pct: '5.00', - test_first_pct: '70.00', + test_coverage_rate: '70.00', + uncovered_prod_rate: '5.00', message_quality_pct: '80.00' })).toBe('harmonious-high-achiever'); }); + it('returns "mixed-signals" (not harmonious) when uncovered_prod_rate is at or above 10', () => { + expect(classifyDoraArchetype({ + large_commits_pct: '10.00', + sprawling_commits_pct: '5.00', + test_coverage_rate: '70.00', + uncovered_prod_rate: '10.00', + message_quality_pct: '80.00' + })).not.toBe('harmonious-high-achiever'); + }); + it('returns "legacy-bottleneck" for high sprawl combined with high large commits', () => { expect(classifyDoraArchetype({ large_commits_pct: '35.00', sprawling_commits_pct: '30.00', - test_first_pct: '40.00', + test_coverage_rate: '40.00', + uncovered_prod_rate: '5.00', message_quality_pct: '50.00' })).toBe('legacy-bottleneck'); }); @@ -25,16 +37,18 @@ describe('classifyDoraArchetype', () => { expect(classifyDoraArchetype({ large_commits_pct: '45.00', sprawling_commits_pct: '8.00', - test_first_pct: '55.00', + test_coverage_rate: '55.00', + uncovered_prod_rate: '5.00', message_quality_pct: '65.00' })).toBe('foundational-challenges'); }); - it('returns "foundational-challenges" for low test discipline with elevated large commits', () => { + it('returns "foundational-challenges" when uncovered_prod_rate exceeds 20%', () => { expect(classifyDoraArchetype({ large_commits_pct: '25.00', sprawling_commits_pct: '8.00', - test_first_pct: '25.00', + test_coverage_rate: '40.00', + uncovered_prod_rate: '25.00', message_quality_pct: '50.00' })).toBe('foundational-challenges'); }); @@ -43,7 +57,8 @@ describe('classifyDoraArchetype', () => { expect(classifyDoraArchetype({ large_commits_pct: '25.00', sprawling_commits_pct: '12.00', - test_first_pct: '40.00', + test_coverage_rate: '40.00', + uncovered_prod_rate: '5.00', message_quality_pct: '55.00' })).toBe('mixed-signals'); }); diff --git a/__tests__/generateInsights.test.js b/__tests__/generateInsights.test.js index 4fed9db..d5430bf 100644 --- a/__tests__/generateInsights.test.js +++ b/__tests__/generateInsights.test.js @@ -7,7 +7,9 @@ function makeSummary(overrides = {}) { return { large_commits_pct: '0.00', sprawling_commits_pct: '0.00', - test_first_pct: '60.00', + test_coverage_rate: '60.00', + test_isolation_rate: '10.00', + uncovered_prod_rate: '0.00', avg_lines_changed: '50.00', ...overrides }; @@ -41,12 +43,12 @@ describe('generateInsights', () => { // --- degenerate / zero case --- test('returns empty arrays when metrics list is empty and values are zero', () => { const { insights, warnings, recommendations } = generateInsights( - makeSummary({ large_commits_pct: '0.00', sprawling_commits_pct: '0.00', test_first_pct: '60.00' }), + makeSummary({ large_commits_pct: '0.00', sprawling_commits_pct: '0.00', test_coverage_rate: '60.00', uncovered_prod_rate: '0.00' }), [] ); expect(warnings).toEqual([]); expect(recommendations).toEqual([]); - expect(insights).toHaveLength(3); // healthy large + healthy sprawling + strong test-first (60% > 50) + expect(insights).toHaveLength(3); // healthy large + healthy sprawling + strong test coverage (60% > 50) }); // --- healthy thresholds produce positive insights, no warnings --- @@ -62,9 +64,9 @@ describe('generateInsights', () => { expect(insights.some(i => i.includes('Good sprawling commit control'))).toBe(true); }); - test('emits positive insight for test_first_pct above 50', () => { - const { insights } = generateInsights(makeSummary({ test_first_pct: '55.00' }), []); - expect(insights.some(i => i.includes('Strong test-first discipline'))).toBe(true); + test('emits positive insight for test_coverage_rate above 50', () => { + const { insights } = generateInsights(makeSummary({ test_coverage_rate: '55.00' }), []); + expect(insights.some(i => i.includes('Strong test coverage'))).toBe(true); }); // --- warning thresholds --- @@ -92,9 +94,38 @@ describe('generateInsights', () => { expect(recommendations.length).toBeGreaterThan(0); }); - test('emits warning for test_first_pct below 30', () => { - const { warnings } = generateInsights(makeSummary({ test_first_pct: '20.00' }), []); - expect(warnings.some(w => w.includes('Low test-first discipline'))).toBe(true); + test('emits warning for test_coverage_rate below 30', () => { + const { warnings } = generateInsights(makeSummary({ test_coverage_rate: '20.00' }), []); + expect(warnings.some(w => w.includes('Low test coverage'))).toBe(true); + }); + + // --- uncovered_prod_rate --- + test('emits critical warning for uncovered_prod_rate above 20', () => { + const { warnings, recommendations } = generateInsights(makeSummary({ uncovered_prod_rate: '25.00' }), []); + expect(warnings.some(w => w.includes('uncovered production commits'))).toBe(true); + expect(recommendations.length).toBeGreaterThan(0); + }); + + test('emits warning (not critical) for uncovered_prod_rate between 10 and 20', () => { + const { warnings } = generateInsights(makeSummary({ uncovered_prod_rate: '15.00' }), []); + expect(warnings.some(w => w.includes('uncovered production commits'))).toBe(true); + expect(warnings.some(w => w.includes('๐Ÿšจ'))).toBe(false); + }); + + test('emits no uncovered_prod warning when rate is below 10', () => { + const { warnings } = generateInsights(makeSummary({ uncovered_prod_rate: '5.00' }), []); + expect(warnings.some(w => w.includes('uncovered production commits'))).toBe(false); + }); + + // --- test_isolation_rate --- + test('emits positive insight for test_isolation_rate above 10', () => { + const { insights } = generateInsights(makeSummary({ test_isolation_rate: '15.00' }), []); + expect(insights.some(i => i.includes('test-only commits'))).toBe(true); + }); + + test('emits no test isolation insight when rate is below 10', () => { + const { insights } = generateInsights(makeSummary({ test_isolation_rate: '5.00' }), []); + expect(insights.some(i => i.includes('test-only commits'))).toBe(false); }); test('emits warning for avg_lines_changed above 500', () => { diff --git a/lib/git.js b/lib/git.js index c329a8b..ef780b9 100644 --- a/lib/git.js +++ b/lib/git.js @@ -109,6 +109,8 @@ function analyzeCommit(sha, branch) { test_files_count: testFiles, prod_files_count: prodFiles, test_first_indicator: testFiles > 0 && prodFiles > 0, + test_only_commit: testFiles > 0 && prodFiles === 0, + uncovered_prod_commit: testFiles === 0 && prodFiles > 0 && totalLines > CONFIG.LARGE_COMMIT_THRESHOLD, large_commit: totalLines > CONFIG.LARGE_COMMIT_THRESHOLD, sprawling_commit: filesChanged > CONFIG.SPRAWLING_COMMIT_THRESHOLD, outlier: false, diff --git a/lib/metrics.js b/lib/metrics.js index 70b3a10..c2b946b 100644 --- a/lib/metrics.js +++ b/lib/metrics.js @@ -32,24 +32,25 @@ function scoreMessageQuality(message) { /** * Classify a repo into a DORA team archetype based on summary metrics. * Evaluated in priority order: harmonious-high-achiever โ†’ legacy-bottleneck โ†’ foundational-challenges โ†’ mixed-signals - * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_first_pct: string, message_quality_pct: string }} summary + * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_coverage_rate: string, uncovered_prod_rate: string, message_quality_pct: string }} summary * @returns {string} */ function classifyDoraArchetype(summary) { const large = parseFloat(summary.large_commits_pct); const sprawling = parseFloat(summary.sprawling_commits_pct); - const testFirst = parseFloat(summary.test_first_pct); + const testCoverage = parseFloat(summary.test_coverage_rate); + const uncoveredProd = parseFloat(summary.uncovered_prod_rate); const msgQuality = parseFloat(summary.message_quality_pct); - if (large < 20 && sprawling < 10 && testFirst > 50 && msgQuality > 60) return 'harmonious-high-achiever'; + if (large < 20 && sprawling < 10 && testCoverage > 50 && uncoveredProd < 10 && msgQuality > 60) return 'harmonious-high-achiever'; if (sprawling > 25 && large > 30) return 'legacy-bottleneck'; - if (large > 40 || (testFirst < 30 && large > 20)) return 'foundational-challenges'; + if (large > 40 || uncoveredProd > 20) return 'foundational-challenges'; return 'mixed-signals'; } /** * Generate insights based on metrics - * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_first_pct: string, avg_lines_changed: string }} summary + * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_coverage_rate: string, test_isolation_rate: string, uncovered_prod_rate: string, avg_lines_changed: string }} summary * @param {Array} metrics * @returns {{ insights: string[], warnings: string[], recommendations: string[] }} */ @@ -60,7 +61,9 @@ function generateInsights(summary, metrics) { const largePct = parseFloat(summary.large_commits_pct); const sprawlingPct = parseFloat(summary.sprawling_commits_pct); - const testFirstPct = parseFloat(summary.test_first_pct); + const testCoveragePct = parseFloat(summary.test_coverage_rate); + const testIsolationPct = parseFloat(summary.test_isolation_rate); + const uncoveredProdPct = parseFloat(summary.uncovered_prod_rate); const avgLines = parseFloat(summary.avg_lines_changed); if (largePct > 40) { @@ -81,13 +84,24 @@ function generateInsights(summary, metrics) { insights.push(`โœ… Good sprawling commit control (${sprawlingPct}%)`); } - if (testFirstPct > 50) { - insights.push(`โœ… Strong test-first discipline (${testFirstPct}%)`); - } else if (testFirstPct < 30) { - warnings.push(`โš ๏ธ Low test-first discipline (${testFirstPct}%) - AI tools may be bypassing TDD`); + if (uncoveredProdPct > 20) { + warnings.push(`๐Ÿšจ High rate of uncovered production commits (${uncoveredProdPct}%) - Large prod commits with no tests`); + recommendations.push('Write tests before accepting AI-generated production code'); + } else if (uncoveredProdPct >= 10) { + warnings.push(`โš ๏ธ Elevated uncovered production commits (${uncoveredProdPct}%) - Monitor test discipline`); + } + + if (testCoveragePct > 50) { + insights.push(`โœ… Strong test coverage discipline (${testCoveragePct}%)`); + } else if (testCoveragePct < 30) { + warnings.push(`โš ๏ธ Low test coverage rate (${testCoveragePct}%) - AI tools may be bypassing TDD`); recommendations.push('Ensure test coverage when accepting AI-generated code'); } + if (testIsolationPct > 10) { + insights.push(`โœ… Healthy rate of test-only commits (${testIsolationPct}%) - TDD red-phase or test improvement commits visible`); + } + if (avgLines > 1000) { warnings.push(`๐Ÿšจ Very high average lines per commit (${avgLines}) - Extreme batch coding`); recommendations.push('Implement strict commit size limits when using AI tools'); diff --git a/local-code-metrics.js b/local-code-metrics.js index 7b898bd..5611387 100644 --- a/local-code-metrics.js +++ b/local-code-metrics.js @@ -27,7 +27,7 @@ const { CLAUDE_SYSTEM_PROMPT, getAnthropicClient, selectClaudeCommits, analyzeWi /** * @typedef {{ sha: string, full_sha: string, date: string, author: string, message: string, source_branch?: string }} CommitInfo - * @typedef {{ total_additions: number, total_deletions: number, files_changed: number, binary_files: number, test_files_count: number, prod_files_count: number, test_first_indicator: boolean, large_commit: boolean, sprawling_commit: boolean, outlier: boolean, source_branch: string, change_ratio: string, ai_confidence?: number, risk_score?: number, patterns?: string[], architectural_concerns?: string[], claude_summary?: string }} CommitStats + * @typedef {{ total_additions: number, total_deletions: number, files_changed: number, binary_files: number, test_files_count: number, prod_files_count: number, test_first_indicator: boolean, test_only_commit: boolean, uncovered_prod_commit: boolean, large_commit: boolean, sprawling_commit: boolean, outlier: boolean, source_branch: string, change_ratio: string, ai_confidence?: number, risk_score?: number, patterns?: string[], architectural_concerns?: string[], claude_summary?: string }} CommitStats * @typedef {CommitInfo & CommitStats & { commit_type: string }} CommitMetric */ @@ -224,7 +224,9 @@ async function collectLocalMetrics() { // Pre-compute pct fields once โ€” reused in both summary object and classifyDoraArchetype call const large_commits_pct = metrics.length > 0 ? ((metrics.filter(m => m.large_commit).length / metrics.length) * 100).toFixed(2) : '0.00'; const sprawling_commits_pct = metrics.length > 0 ? ((metrics.filter(m => m.sprawling_commit).length / metrics.length) * 100).toFixed(2) : '0.00'; - const test_first_pct = metrics.length > 0 ? ((metrics.filter(m => m.test_first_indicator).length / metrics.length) * 100).toFixed(2) : '0.00'; + const test_coverage_rate = metrics.length > 0 ? ((metrics.filter(m => m.test_first_indicator).length / metrics.length) * 100).toFixed(2) : '0.00'; + const test_isolation_rate = metrics.length > 0 ? ((metrics.filter(m => m.test_only_commit).length / metrics.length) * 100).toFixed(2) : '0.00'; + const uncovered_prod_rate = metrics.length > 0 ? ((metrics.filter(m => m.uncovered_prod_commit).length / metrics.length) * 100).toFixed(2) : '0.00'; // Generate summary statistics const summary = { @@ -236,7 +238,9 @@ async function collectLocalMetrics() { branch_commit_counts: branchCommitCounts, large_commits_pct, sprawling_commits_pct, - test_first_pct, + test_coverage_rate, + test_isolation_rate, + uncovered_prod_rate, avg_files_changed: metrics.length > 0 ? (metrics.reduce((sum, m) => sum + m.files_changed, 0) / metrics.length).toFixed(2) : "0.00", avg_lines_changed: metrics.length > 0 ? (metrics.reduce((sum, m) => sum + m.total_additions + m.total_deletions, 0) / metrics.length).toFixed(2) : "0.00", p50_lines_changed: lineStats.p50, @@ -251,7 +255,7 @@ async function collectLocalMetrics() { net_additions_ratio_median: ratioStats.p50, net_additions_ratio_p90: ratioStats.p90, message_quality_pct, - dora_archetype: classifyDoraArchetype({ large_commits_pct, sprawling_commits_pct, test_first_pct, message_quality_pct }), + dora_archetype: classifyDoraArchetype({ large_commits_pct, sprawling_commits_pct, test_coverage_rate, uncovered_prod_rate, message_quality_pct }), config: CONFIG, note: "Local feature branches analysis - shows actual development patterns before merge squashing" }; @@ -286,7 +290,9 @@ async function collectLocalMetrics() { console.log(`๐Ÿ“ˆ Total commits analyzed: ${summary.total_commits}`); console.log(`๐Ÿ“ Large commits (>${CONFIG.LARGE_COMMIT_THRESHOLD} lines): ${summary.large_commits_pct}%`); console.log(`๐Ÿ“ Sprawling commits (>${CONFIG.SPRAWLING_COMMIT_THRESHOLD} files): ${summary.sprawling_commits_pct}%`); - console.log(`๐Ÿงช Test-first discipline: ${summary.test_first_pct}%`); + console.log(`๐Ÿงช Test coverage (test+prod): ${summary.test_coverage_rate}%`); + console.log(`๐Ÿงช Test isolation (test-only): ${summary.test_isolation_rate}%`); + console.log(`๐Ÿšจ Uncovered prod (large, no tests): ${summary.uncovered_prod_rate}%`); console.log(`๐Ÿ“‚ Average files changed: ${summary.avg_files_changed}`); console.log(`๐Ÿ“ Average lines changed: ${summary.avg_lines_changed}`); console.log(''); diff --git a/metrics-specification.md b/metrics-specification.md index a76d90b..ad2ee5d 100644 --- a/metrics-specification.md +++ b/metrics-specification.md @@ -94,29 +94,38 @@ sprawling_commit_pct = (commits where files_changed > SPRAWLING_COMMIT_THRESHOLD --- -### Metric 3: Test-First Discipline Rate +### Metric 3: Three-Way Test Coverage Classification -**What it measures**: The proportion of commits that modify both test files and production files in the same commit, used as a proxy for test discipline under AI-assisted development. +**What it measures**: Replaces the binary `test_first_indicator` with three distinct commit categories, each carrying different signal quality for AI drift detection. -**Formula**: +| Category | Formula | Per-commit flag | Summary field | +|----------|---------|----------------|---------------| +| Test Coverage | test AND prod files in same commit | `test_first_indicator` | `test_coverage_rate` | +| Test Isolation | test files only (no prod files) | `test_only_commit` | `test_isolation_rate` | +| Uncovered Prod | prod files only AND large commit | `uncovered_prod_commit` | `uncovered_prod_rate` | + +**Formulas**: ``` -test_first_pct = (commits where test_files_count > 0 AND prod_files_count > 0) / total_commits ร— 100 +test_coverage_rate = (commits where test_files_count > 0 AND prod_files_count > 0) / total_commits ร— 100 +test_isolation_rate = (commits where test_files_count > 0 AND prod_files_count = 0) / total_commits ร— 100 +uncovered_prod_rate = (commits where test_files_count = 0 AND prod_files_count > 0 AND large_commit = true) / total_commits ร— 100 ``` -**Per-commit flag**: `test_first_indicator: boolean` - **Data source**: `git show --numstat {sha}` (file paths matched against `TEST_FILE_PATTERNS`) **CONFIG key**: `TEST_FILE_PATTERNS` (array of 8 regex patterns; covers JS/TS, Python, Go, Java, C#) **Thresholds**: -| Range | Signal | -|-------|--------| -| > 50% | Healthy: strong test discipline | -| 30โ€“50% | Warning: monitor AI tool usage patterns | -| < 30% | Critical: AI tools may be bypassing TDD practices | +| Metric | Range | Signal | +|--------|-------|--------| +| `test_coverage_rate` | > 50% | Healthy: test discipline present | +| `test_coverage_rate` | < 30% | Warning: low paired test+prod commits | +| `test_isolation_rate` | > 10% | Positive: TDD red-phase or test-improvement commits visible | +| `uncovered_prod_rate` | < 10% | Healthy | +| `uncovered_prod_rate` | 10โ€“20% | Warning: elevated AI drift risk | +| `uncovered_prod_rate` | > 20% | Critical: "accepted AI output without writing tests" pattern | -**Limitations**: This metric cannot distinguish test-first (TDD) from test-after. It measures co-occurrence, not ordering. A commit that adds production code and test code written afterward scores the same as one where tests were written first. +**Why `uncovered_prod_rate` matters**: A commit that is both prod-only and large is the clearest AI drift signal in this toolkit โ€” it matches the pattern of a developer accepting a large AI-generated code block without writing any tests. `test_isolation_rate` is a positive signal: test-only commits indicate TDD red-phase work or deliberate test improvements, both of which the binary metric incorrectly classified as "bad". **DORA connection**: DORA's research identifies automated testing as the single strongest predictor of whether AI tools help or hurt a team. Teams without it when they adopt AI see the fastest debt accumulation. @@ -315,7 +324,8 @@ outlier = (total_additions + total_deletions) > (mean_lines + 2 ร— stddev_l harmonious-high-achiever: large_commits_pct < 20 AND sprawling_commits_pct < 10 - AND test_first_pct > 50 + AND test_coverage_rate > 50 + AND uncovered_prod_rate < 10 AND message_quality_pct > 60 legacy-bottleneck: @@ -324,7 +334,7 @@ legacy-bottleneck: foundational-challenges: large_commits_pct > 40 - OR (test_first_pct < 30 AND large_commits_pct > 20) + OR uncovered_prod_rate > 20 mixed-signals: (all other combinations) @@ -558,10 +568,12 @@ Single summary object for the analysis run: branches_analyzed: string[], branch_commit_counts: Record, - // Original 5 metrics (preserved for backwards compatibility) + // Core metrics large_commits_pct: string, // "XX.XX" sprawling_commits_pct: string, - test_first_pct: string, + test_coverage_rate: string, // commits with test AND prod files / total + test_isolation_rate: string, // commits with test files only / total + uncovered_prod_rate: string, // large commits with prod files only / total avg_files_changed: string, avg_lines_changed: string,