diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..25d872b --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +# Copy this file to .env and fill in your values. +# .env is gitignored — never commit real API keys. + +# Optional: enables Claude diff-level analysis of high-risk commits. +# Get a key at https://console.anthropic.com/ +# Also requires: npm install @anthropic-ai/sdk +ANTHROPIC_API_KEY=sk-ant-... diff --git a/.github/workflows/code-metrics.yml b/.github/workflows/code-metrics.yml index 33980f4..c14c470 100644 --- a/.github/workflows/code-metrics.yml +++ b/.github/workflows/code-metrics.yml @@ -30,37 +30,92 @@ jobs: with: script: | const fs = require('fs'); - + + // --------------------------------------------------------------- + // Inline helpers (mirrors local-code-metrics.js logic) + // --------------------------------------------------------------- + + const CONVENTIONAL_COMMIT_RE = /^(feat|fix|refactor|test|chore|docs|perf|ci|build|revert)(\(.+\))?:/i; + function scoreMessageQuality(message) { + if (!message) return false; + if (CONVENTIONAL_COMMIT_RE.test(message)) return true; + return message.split(/\s+/).filter(Boolean).length >= 10; + } + + function quantile(sorted, p) { + if (sorted.length === 0) return 0; + if (sorted.length === 1) return sorted[0]; + const idx = p * (sorted.length - 1); + const lo = Math.floor(idx); + const hi = Math.ceil(idx); + return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo); + } + + function computeStats(values) { + if (values.length === 0) return { p50: 0, p90: 0, p95: 0, mean: 0, stddev: 0 }; + const sorted = [...values].sort((a, b) => a - b); + const mean = values.reduce((s, v) => s + v, 0) / values.length; + const variance = values.reduce((s, v) => s + (v - mean) ** 2, 0) / values.length; + return { + p50: quantile(sorted, 0.5), + p90: quantile(sorted, 0.9), + p95: quantile(sorted, 0.95), + mean, + stddev: Math.sqrt(variance) + }; + } + + function computeVelocityTrend(timestamps) { + if (timestamps.length < 2) return 'stable'; + const ms = timestamps.map(t => new Date(t).getTime()).sort((a, b) => a - b); + const spanDays = (ms[ms.length - 1] - ms[0]) / 86400000 || 1; + const midMs = (ms[0] + ms[ms.length - 1]) / 2; + const halfSpan = spanDays / 2 || 1; + const firstRate = ms.filter(t => t <= midMs).length / halfSpan; + const secondRate = ms.filter(t => t > midMs).length / halfSpan; + if (secondRate > firstRate * 1.25) return 'accelerating'; + if (secondRate < firstRate * 0.75) return 'decelerating'; + return 'stable'; + } + + function classifyDoraArchetype(large, sprawling, testFirst, msgQuality) { + if (large < 20 && sprawling < 10 && testFirst > 50 && msgQuality > 60) return 'harmonious-high-achiever'; + if (sprawling > 25 && large > 30) return 'legacy-bottleneck'; + if (large > 40 || (testFirst < 30 && large > 20)) return 'foundational-challenges'; + return 'mixed-signals'; + } + + // --------------------------------------------------------------- + // Main collection + // --------------------------------------------------------------- + async function collectMetrics() { const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/'); const since = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000).toISOString(); - + console.log(`Repository: ${owner}/${repo}`); console.log(`Looking for commits since: ${since}`); - + try { - const repoInfo = await github.rest.repos.get({ - owner, - repo - }); + const repoInfo = await github.rest.repos.get({ owner, repo }); console.log(`Repository access confirmed: ${repoInfo.data.full_name}`); - + const branches = await github.paginate(github.rest.repos.listBranches, { owner, repo, per_page: 100 }); - - const featureBranches = branches.filter(branch => + + const featureBranches = branches.filter(branch => !['main', 'master'].includes(branch.name.toLowerCase()) ); - + console.log(`Found ${branches.length} total branches`); console.log(`Analyzing ${featureBranches.length} feature branches: ${featureBranches.map(b => b.name).join(', ')}`); - + let allCommits = []; let branchCommitCounts = {}; - + for (const branch of featureBranches) { try { console.log(`Fetching commits from branch: ${branch.name}`); @@ -71,28 +126,27 @@ jobs: since, per_page: 100 }); - + branchCommitCounts[branch.name] = branchCommits.length; allCommits = allCommits.concat(branchCommits.map(commit => ({ ...commit, source_branch: branch.name }))); - + await new Promise(resolve => setTimeout(resolve, 200)); - } catch (error) { console.log(`Error fetching commits from branch ${branch.name}: ${error.message}`); branchCommitCounts[branch.name] = 0; } } - - const uniqueCommits = allCommits.filter((commit, index, self) => + + const uniqueCommits = allCommits.filter((commit, index, self) => index === self.findIndex(c => c.sha === commit.sha) ); - + console.log(`Found ${allCommits.length} total commits, ${uniqueCommits.length} unique commits`); console.log('Commits per branch:', branchCommitCounts); - + if (uniqueCommits.length === 0) { console.log('No commits found in feature branches in the last 30 days.'); const emptyMetrics = []; @@ -101,22 +155,31 @@ jobs: branches_analyzed: featureBranches.map(b => b.name), branch_commit_counts: branchCommitCounts, large_commits_pct: "0.00", - sprawling_commits_pct: "0.00", + sprawling_commits_pct: "0.00", test_first_pct: "0.00", avg_files_changed: "0.00", avg_lines_changed: "0.00", avg_prod_lines_changed: "0.00", + message_quality_pct: "0.00", + p50_lines_changed: 0, + p90_lines_changed: 0, + p95_lines_changed: 0, + stddev_lines_changed: 0, + velocity_trend: "stable", + additions_ratio_median: 0, + additions_ratio_p90: 0, + dora_archetype: "mixed-signals", note: "Feature branches only - main/master excluded. Large commit threshold applied to production code only." }; - + fs.writeFileSync('commit_metrics.json', JSON.stringify(emptyMetrics, null, 2)); fs.writeFileSync('metrics_summary.json', JSON.stringify(summary, null, 2)); return; } - + const metrics = []; console.log(`Processing ${Math.min(uniqueCommits.length, 50)} commits...`); - + for (const commit of uniqueCommits.slice(0, 50)) { try { const detail = await github.rest.repos.getCommit({ @@ -124,7 +187,7 @@ jobs: repo, ref: commit.sha }); - + const testFiles = detail.data.files?.filter(f => /\.(test|spec)\.|Tests?\.cs$|Test\.cs$|__tests__|\/tests?\//i.test(f.filename) ) || []; @@ -132,11 +195,13 @@ jobs: const prodFiles = detail.data.files?.filter(f => !/\.(test|spec)\.|Tests?\.cs$|Test\.cs$|__tests__|\/tests?\//i.test(f.filename) ) || []; - + + const message = commit.commit.message.split('\n')[0]; + metrics.push({ sha: commit.sha, date: commit.commit.committer.date, - message: commit.commit.message.split('\n')[0], + message, author: commit.commit.author.name, source_branch: commit.source_branch, total_additions: detail.data.stats.additions, @@ -145,54 +210,74 @@ jobs: test_files_count: testFiles.length, prod_files_count: prodFiles.length, test_first_indicator: testFiles.length > 0 && prodFiles.length > 0, - - // Calculate production code changes only prod_additions: prodFiles.reduce((sum, f) => sum + f.additions, 0), prod_deletions: prodFiles.reduce((sum, f) => sum + f.deletions, 0), - - // Base "large commit" on production code only large_commit: (prodFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0)) > 100, - sprawling_commit: (detail.data.files?.length || 0) > 5, + message_quality: scoreMessageQuality(message), commit_type: "feature_branch" }); - + await new Promise(resolve => setTimeout(resolve, 100)); } catch (error) { console.log(`Error processing commit ${commit.sha}: ${error.message}`); } } - + console.log(`Saving metrics for ${metrics.length} commits`); fs.writeFileSync('commit_metrics.json', JSON.stringify(metrics, null, 2)); - - // Calculate summary AFTER all commits are processed + + // Statistical distributions + const lineSizes = metrics.map(m => m.total_additions + m.total_deletions); + const lineStats = computeStats(lineSizes); + const fileStats = computeStats(metrics.map(m => m.files_changed)); + const ratios = metrics.map(m => m.total_additions / (m.total_deletions || 1)); + const ratioStats = computeStats([...ratios].sort((a, b) => a - b).map((_, i, a) => a[i])); + const velocityTrend = computeVelocityTrend(metrics.map(m => m.date)); + + const qualityCount = metrics.filter(m => m.message_quality).length; + const msgQualityPct = metrics.length > 0 + ? ((qualityCount / metrics.length) * 100).toFixed(2) + : "0.00"; + + const largePct = parseFloat((metrics.filter(m => m.large_commit).length / metrics.length * 100).toFixed(2)); + const sprawlingPct = parseFloat((metrics.filter(m => m.sprawling_commit).length / metrics.length * 100).toFixed(2)); + const testFirstPct = parseFloat((metrics.filter(m => m.test_first_indicator).length / metrics.length * 100).toFixed(2)); + const summary = { total_commits: metrics.length, filtered_from: uniqueCommits.length, branches_analyzed: featureBranches.map(b => b.name), branch_commit_counts: branchCommitCounts, - large_commits_pct: (metrics.filter(m => m.large_commit).length / metrics.length * 100).toFixed(2), - sprawling_commits_pct: (metrics.filter(m => m.sprawling_commit).length / metrics.length * 100).toFixed(2), - test_first_pct: (metrics.filter(m => m.test_first_indicator).length / metrics.length * 100).toFixed(2), + large_commits_pct: largePct.toFixed(2), + sprawling_commits_pct: sprawlingPct.toFixed(2), + test_first_pct: testFirstPct.toFixed(2), avg_files_changed: (metrics.reduce((sum, m) => sum + m.files_changed, 0) / metrics.length).toFixed(2), - - // Use production code lines for average (both total and prod-only metrics) avg_lines_changed: (metrics.reduce((sum, m) => sum + (m.prod_additions || 0) + (m.prod_deletions || 0), 0) / metrics.length).toFixed(2), avg_prod_lines_changed: (metrics.reduce((sum, m) => sum + (m.prod_additions || 0) + (m.prod_deletions || 0), 0) / metrics.length).toFixed(2), - + p50_lines_changed: Math.round(lineStats.p50), + p90_lines_changed: Math.round(lineStats.p90), + p95_lines_changed: Math.round(lineStats.p95), + stddev_lines_changed: Math.round(lineStats.stddev), + p50_files_changed: Math.round(fileStats.p50), + p90_files_changed: Math.round(fileStats.p90), + velocity_trend: velocityTrend, + additions_ratio_median: parseFloat(ratioStats.p50.toFixed(2)), + additions_ratio_p90: parseFloat(ratioStats.p90.toFixed(2)), + message_quality_pct: msgQualityPct, + dora_archetype: classifyDoraArchetype(largePct, sprawlingPct, testFirstPct, parseFloat(msgQualityPct)), note: "Feature branches only - main/master excluded. Large commit threshold applied to production code only." }; - + fs.writeFileSync('metrics_summary.json', JSON.stringify(summary, null, 2)); console.log('Summary:', summary); - + } catch (error) { console.error('Error in collectMetrics:', error); throw error; } } - + collectMetrics(); - name: Upload metrics artifacts @@ -217,32 +302,68 @@ jobs: .map(([branch, count]) => `- ${branch}: ${count} commits`) .join('\n'); + function statusMark(value, threshold, direction = 'below') { + const ok = direction === 'below' ? parseFloat(value) < threshold : parseFloat(value) > threshold; + return ok ? 'OK' : 'Warning'; + } + + const archetypeDescriptions = { + 'harmonious-high-achiever': 'Strong foundational practices; AI tools likely amplifying positive outcomes.', + 'foundational-challenges': 'Weak testing or batch discipline; AI tools may be accelerating debt accumulation.', + 'legacy-bottleneck': 'Architectural scatter combined with large commits; AI making cross-cutting changes worse.', + 'mixed-signals': 'Inconsistent patterns across metrics; investigate specific outlier commits.' + }; + + const archetype = summary.dora_archetype || 'mixed-signals'; + const archetypeDesc = archetypeDescriptions[archetype] || ''; + const bodyLines = [ '## Weekly Code Metrics Report', '', - '**Period:** Last 30 days', + `**Period:** Last 30 days`, `**Commits Analyzed:** ${summary.total_commits}`, `**Branches Analyzed:** ${summary.branches_analyzed?.join(', ') || 'N/A'}`, '', '### Key Metrics', '', - `**Large Commits (>100 production lines):** ${summary.large_commits_pct}%`, - `**Sprawling Commits (>5 files):** ${summary.sprawling_commits_pct}%`, - `**Test-First Discipline:** ${summary.test_first_pct}%`, - `**Avg Files Changed:** ${summary.avg_files_changed}`, - `**Avg Production Lines Changed:** ${summary.avg_prod_lines_changed}`, + '| Metric | Value | Target | Status |', + '|--------|-------|--------|--------|', + `| Large Commits (>100 prod lines) | ${summary.large_commits_pct}% | <20% | ${statusMark(summary.large_commits_pct, 20)} |`, + `| Sprawling Commits (>5 files) | ${summary.sprawling_commits_pct}% | <10% | ${statusMark(summary.sprawling_commits_pct, 10)} |`, + `| Test-First Discipline | ${summary.test_first_pct}% | >50% | ${statusMark(summary.test_first_pct, 50, 'above')} |`, + `| Message Quality | ${summary.message_quality_pct}% | >60% | ${statusMark(summary.message_quality_pct, 60, 'above')} |`, + `| Additions Ratio (median) | ${summary.additions_ratio_median} | <3.0 | ${parseFloat(summary.additions_ratio_median) < 3.0 ? 'OK' : 'Warning'} |`, + '', + '### Commit Size Distribution', + '', + '| Percentile | Lines Changed |', + '|------------|---------------|', + `| p50 (median) | ${summary.p50_lines_changed} |`, + `| p90 | ${summary.p90_lines_changed} |`, + `| p95 | ${summary.p95_lines_changed} |`, + `| Std Dev | ${summary.stddev_lines_changed} |`, + '', + `**Commit size trend:** ${summary.velocity_trend === 'accelerating' ? 'Accelerating (monitor closely)' : summary.velocity_trend === 'decelerating' ? 'Decelerating' : 'Stable'}`, + '', + '### DORA Team Archetype', + '', + `**${archetype}**`, + '', + archetypeDesc, '', '### Branch Activity', + '', branchActivity, '', '### Interpretation', '', - 'Target: <20% large commits, <10% sprawling commits', - 'Test-first discipline should trend upward', - 'Large commits now measured by production code only (excludes test files)', - 'Consider investigation if metrics worsen significantly', + '- Target: <20% large commits, <10% sprawling commits', + '- Test-first discipline should remain above 50%', + '- Message quality above 60% indicates disciplined commit practices', + '- Large commits measured by production code only (excludes test files)', + '- p90 > 500 lines warrants investigation of review burden', '', - '_Generated by PDCA Code Metrics Workflow_' + '_Generated by Code Metrics Workflow_' ]; const body = bodyLines.join('\n'); diff --git a/.github/workflows/pr-metrics.yml b/.github/workflows/pr-metrics.yml index 9ae0979..bfa045a 100644 --- a/.github/workflows/pr-metrics.yml +++ b/.github/workflows/pr-metrics.yml @@ -16,35 +16,57 @@ jobs: with: script: | const pr = context.payload.pull_request; - - // Get PR files for size analysis + + // --------------------------------------------------------------- + // Inline helpers (mirrors local-code-metrics.js logic) + // --------------------------------------------------------------- + + const CONVENTIONAL_COMMIT_RE = /^(feat|fix|refactor|test|chore|docs|perf|ci|build|revert)(\(.+\))?:/i; + function scoreMessageQuality(message) { + if (!message) return false; + if (CONVENTIONAL_COMMIT_RE.test(message)) return true; + return message.split(/\s+/).filter(Boolean).length >= 10; + } + + function classifyDoraArchetype(large, sprawling, testFirst, msgQuality) { + if (large < 20 && sprawling < 10 && testFirst > 50 && msgQuality > 60) return 'harmonious-high-achiever'; + if (sprawling > 25 && large > 30) return 'legacy-bottleneck'; + if (large > 40 || (testFirst < 30 && large > 20)) return 'foundational-challenges'; + return 'mixed-signals'; + } + + // --------------------------------------------------------------- + // Fetch PR data + // --------------------------------------------------------------- + const files = await github.paginate(github.rest.pulls.listFiles, { owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, per_page: 100 }); - - // Get commits in this PR for detailed analysis + const commits = await github.paginate(github.rest.pulls.listCommits, { owner: context.repo.owner, repo: context.repo.repo, pull_number: context.issue.number, per_page: 100 }); - + console.log(`Analyzing ${commits.length} commits and ${files.length} files in PR`); - - // Separate test and production files + const testFiles = files.filter(f => /\.(test|spec)\.|Tests?\.cs$|Test\.cs$|__tests__|\/tests?\//i.test(f.filename) ); - + const prodFiles = files.filter(f => !/\.(test|spec)\.|Tests?\.cs$|Test\.cs$|__tests__|\/tests?\//i.test(f.filename) ); - - // Analyze each commit in detail + + // --------------------------------------------------------------- + // Per-commit analysis + // --------------------------------------------------------------- + const commitMetrics = []; for (const commit of commits) { try { @@ -53,7 +75,7 @@ jobs: repo: context.repo.repo, ref: commit.sha }); - + const commitTestFiles = detail.data.files?.filter(f => /\.(test|spec)\.|Tests?\.cs$|Test\.cs$|__tests__|\/tests?\//i.test(f.filename) ) || []; @@ -61,13 +83,16 @@ jobs: const commitProdFiles = detail.data.files?.filter(f => !/\.(test|spec)\.|Tests?\.cs$|Test\.cs$|__tests__|\/tests?\//i.test(f.filename) ) || []; - + const prodChanges = commitProdFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0); const testChanges = commitTestFiles.reduce((sum, f) => sum + f.additions + f.deletions, 0); - + const additions = detail.data.stats?.additions || 0; + const deletions = detail.data.stats?.deletions || 1; + const message = commit.commit.message.split('\n')[0]; + commitMetrics.push({ sha: commit.sha.substring(0, 7), - message: commit.commit.message.split('\n')[0], + message, author: commit.commit.author.name, date: new Date(commit.commit.author.date).toLocaleDateString(), files_changed: detail.data.files?.length || 0, @@ -81,206 +106,271 @@ jobs: test_first_indicator: commitTestFiles.length > 0 && commitProdFiles.length > 0, test_only: commitTestFiles.length > 0 && commitProdFiles.length === 0, prod_only: commitProdFiles.length > 0 && commitTestFiles.length === 0, - test_ratio: prodChanges > 0 ? (testChanges / prodChanges) : 0 + test_ratio: prodChanges > 0 ? (testChanges / prodChanges) : 0, + additions_ratio: (additions / (deletions || 1)).toFixed(2), + message_quality: scoreMessageQuality(message) }); - + await new Promise(resolve => setTimeout(resolve, 100)); } catch (error) { console.log(`Error analyzing commit ${commit.sha}: ${error.message}`); } } - - // Calculate PR-level metrics + + // --------------------------------------------------------------- + // PR-level aggregates + // --------------------------------------------------------------- + const totalAdditions = pr.additions; const totalDeletions = pr.deletions; const totalChangedFiles = pr.changed_files; const totalChanges = totalAdditions + totalDeletions; - + const prodAdditions = prodFiles.reduce((sum, f) => sum + f.additions, 0); const prodDeletions = prodFiles.reduce((sum, f) => sum + f.deletions, 0); const prodChanges = prodAdditions + prodDeletions; const prodFilesCount = prodFiles.length; - + const testAdditions = testFiles.reduce((sum, f) => sum + f.additions, 0); const testDeletions = testFiles.reduce((sum, f) => sum + f.deletions, 0); const testChanges = testAdditions + testDeletions; const testFilesCount = testFiles.length; - - // Commit-level analysis + const largeCommits = commitMetrics.filter(c => c.large_commit).length; const sprawlingCommits = commitMetrics.filter(c => c.sprawling_commit).length; const testFirstCommits = commitMetrics.filter(c => c.test_first_indicator).length; const testOnlyCommits = commitMetrics.filter(c => c.test_only).length; const prodOnlyCommits = commitMetrics.filter(c => c.prod_only).length; - - const avgCommitSize = commitMetrics.length > 0 ? - (commitMetrics.reduce((sum, c) => sum + c.prod_changes, 0) / commitMetrics.length) : 0; - - const avgFilesPerCommit = commitMetrics.length > 0 ? - (commitMetrics.reduce((sum, c) => sum + c.files_changed, 0) / commitMetrics.length) : 0; - - // Development pattern analysis - const hasRefactoringPattern = commitMetrics.some(c => + const qualityCommits = commitMetrics.filter(c => c.message_quality).length; + + const avgCommitSize = commitMetrics.length > 0 + ? (commitMetrics.reduce((sum, c) => sum + c.prod_changes, 0) / commitMetrics.length) + : 0; + + const avgFilesPerCommit = commitMetrics.length > 0 + ? (commitMetrics.reduce((sum, c) => sum + c.files_changed, 0) / commitMetrics.length) + : 0; + + const largePct = commitMetrics.length > 0 ? (largeCommits / commitMetrics.length * 100) : 0; + const sprawlingPct = commitMetrics.length > 0 ? (sprawlingCommits / commitMetrics.length * 100) : 0; + const testFirstPct = commitMetrics.length > 0 ? (testFirstCommits / commitMetrics.length * 100) : 0; + const msgQualityPct = commitMetrics.length > 0 ? (qualityCommits / commitMetrics.length * 100) : 0; + + const medianAdditionsRatio = (() => { + const ratios = commitMetrics.map(c => parseFloat(c.additions_ratio)).sort((a, b) => a - b); + if (ratios.length === 0) return 0; + const mid = Math.floor(ratios.length / 2); + return ratios.length % 2 !== 0 ? ratios[mid] : (ratios[mid - 1] + ratios[mid]) / 2; + })(); + + // --------------------------------------------------------------- + // Development pattern detection + // --------------------------------------------------------------- + + const hasRefactoringPattern = commitMetrics.some(c => /refactor|cleanup|reorganize|extract|rename/i.test(c.message) ); - - const hasIncrementalPattern = commitMetrics.length > 1 && + + const hasIncrementalPattern = commitMetrics.length > 1 && commitMetrics.every(c => c.prod_changes < 200); - - const hasTestDiscipline = commitMetrics.length > 0 && + + const hasTestDiscipline = commitMetrics.length > 0 && (testFirstCommits / commitMetrics.length) > 0.5; - - // PR size classification + + // --------------------------------------------------------------- + // Size classification + // --------------------------------------------------------------- + let sizeLabel = 'small'; - let concerns = []; - let strengths = []; - + const concerns = []; + const strengths = []; + if (prodChanges > 500) { sizeLabel = 'extra-large'; - concerns.push(':warning: Very large production changes - consider breaking into smaller PRs'); + concerns.push('Very large production changes - consider breaking into smaller PRs'); } else if (prodChanges > 200) { sizeLabel = 'large'; - concerns.push(':straight_ruler: Large production changes - review carefully'); + concerns.push('Large production changes - review carefully'); } else if (prodChanges > 100) { sizeLabel = 'medium'; } - + if (prodFilesCount > 10) { - concerns.push(':file_folder: Many production files changed - possible scope creep'); + concerns.push('Many production files changed - possible scope creep'); } - - // Commit quality assessment + if (largeCommits > commitMetrics.length * 0.5) { - concerns.push(`:package: ${largeCommits}/${commitMetrics.length} commits are large (>100 production lines)`); + concerns.push(`${largeCommits}/${commitMetrics.length} commits exceed 100 production lines`); } - + if (sprawlingCommits > commitMetrics.length * 0.3) { - concerns.push(`:deciduous_tree: ${sprawlingCommits}/${commitMetrics.length} commits touch many files (>5)`); + concerns.push(`${sprawlingCommits}/${commitMetrics.length} commits touch more than 5 files`); + } + + if (medianAdditionsRatio > 3.0) { + concerns.push(`Median additions ratio ${medianAdditionsRatio.toFixed(2)} exceeds 3.0 - possible batch-acceptance pattern`); } - - // Positive patterns + + if (msgQualityPct < 40) { + concerns.push(`Message quality ${msgQualityPct.toFixed(0)}% is below 40% - consider more descriptive commit messages`); + } + if (hasIncrementalPattern) { - strengths.push(':building_construction: Incremental development pattern detected'); + strengths.push('Incremental development pattern detected'); } - + if (hasTestDiscipline) { - strengths.push(':test_tube: Strong test-first discipline across commits'); + strengths.push('Strong test-first discipline across commits'); } - + if (hasRefactoringPattern) { - strengths.push(':recycle: Includes refactoring/cleanup work'); + strengths.push('Includes refactoring or cleanup work'); } - + if (testOnlyCommits > 0) { - strengths.push(`:white_check_mark: ${testOnlyCommits} test-only commits (good for TDD)`); + strengths.push(`${testOnlyCommits} test-only commits`); } - - // Test adequacy analysis + + if (msgQualityPct >= 60) { + strengths.push(`Message quality ${msgQualityPct.toFixed(0)}% meets discipline threshold`); + } + + // --------------------------------------------------------------- + // Test adequacy + // --------------------------------------------------------------- + let testAdequacy = 'excellent'; - let testConcerns = []; - let testPraise = []; - + const testConcerns = []; + const testPraise = []; const testRatio = prodChanges > 0 ? (testChanges / prodChanges) : 0; - + if (prodChanges === 0) { if (testChanges > 0) { testAdequacy = 'excellent'; - testPraise.push(':white_check_mark: Test-only changes - refactoring or test improvements'); + testPraise.push('Test-only changes - refactoring or test improvements'); } else { testAdequacy = 'n/a'; } - } else if (prodChanges > 0 && testChanges === 0) { + } else if (testChanges === 0) { testAdequacy = 'poor'; - testConcerns.push(':x: No test changes for production code modifications'); + testConcerns.push('No test changes for production code modifications'); if (prodOnlyCommits === commitMetrics.length) { - testConcerns.push(':warning: All commits lack test coverage'); + testConcerns.push('All commits lack test coverage'); } } else if (testRatio < 0.3) { testAdequacy = 'needs-improvement'; - testConcerns.push(':yellow_circle: Low test coverage - consider adding more tests'); + testConcerns.push('Low test coverage ratio - consider adding more tests'); } else if (testRatio >= 0.3 && testRatio <= 2.0) { testAdequacy = 'good'; - testPraise.push(':white_check_mark: Good balance of production and test code'); + testPraise.push('Good balance of production and test code'); } else { testAdequacy = 'excellent'; - testPraise.push(':star2: Excellent test coverage - comprehensive testing'); + testPraise.push('Comprehensive test coverage'); } - - // Create detailed breakdown - const breakdown = ` - **Production Code:** ${prodChanges} lines (${prodFilesCount} files) - **Test Code:** ${testChanges} lines (${testFilesCount} files) - **Total:** ${totalChanges} lines (${totalChangedFiles} files) - ${prodChanges > 0 ? `**Test-to-Production Ratio:** ${testRatio.toFixed(2)}:1` : ''} - `; - - const commitAnalysis = ` - ## Commit Analysis - - **Total Commits:** ${commitMetrics.length} - **Average Commit Size:** ${avgCommitSize.toFixed(0)} production lines - **Average Files per Commit:** ${avgFilesPerCommit.toFixed(1)} - - ### Commit Quality Metrics: - - **Large Commits (>100 prod lines):** ${largeCommits}/${commitMetrics.length} (${(largeCommits/commitMetrics.length*100).toFixed(0)}%) - - **Sprawling Commits (>5 files):** ${sprawlingCommits}/${commitMetrics.length} (${(sprawlingCommits/commitMetrics.length*100).toFixed(0)}%) - - **Test-First Discipline:** ${testFirstCommits}/${commitMetrics.length} (${(testFirstCommits/commitMetrics.length*100).toFixed(0)}%) - - **Test-Only Commits:** ${testOnlyCommits} - - **Production-Only Commits:** ${prodOnlyCommits} - - ### Development Patterns: - ${strengths.length > 0 ? strengths.map(s => `- ${s}`).join('\n') : '- No notable positive patterns detected'} - - ${commitMetrics.length <= 10 ? ` - ### Commit Details: - ${commitMetrics.map(c => ` - **${c.sha}** by ${c.author} (${c.date}) - \`${c.message}\` - - Files: ${c.files_changed} (${c.prod_files} prod, ${c.test_files} test) - - Changes: ${c.prod_changes} prod lines, ${c.test_changes} test lines - ${c.large_commit ? '- :package: Large commit' : ''}${c.sprawling_commit ? '- :deciduous_tree: Sprawling commit' : ''}${c.test_first_indicator ? '- :test_tube: Test-first' : ''} - `).join('\n')}` : ''} - `; - - const testSection = testAdequacy !== 'n/a' ? ` - ## Test Coverage Analysis - - **Test Adequacy:** ${testAdequacy} - - ${testPraise.length > 0 ? '### Strengths:\n' + testPraise.map(p => `- ${p}`).join('\n') + '\n' : ''} - ${testConcerns.length > 0 ? '### Test Coverage Concerns:\n' + testConcerns.map(c => `- ${c}`).join('\n') + '\n' : ''} - - ### Test Coverage Guidelines: - - **Target ratio:** 0.5-2.0 test lines per production line - - **Minimum:** Some test changes for any production code modifications - - **Best practice:** Write tests first (TDD) when adding new features - ` : ''; - - const comment = ` - ## Enhanced PR Analysis - - **Size:** ${sizeLabel} (based on production code) - - ${breakdown} - - ${concerns.length > 0 ? '### Size & Quality Concerns:\n' + concerns.map(c => `- ${c}`).join('\n') + '\n' : ':white_check_mark: Good size and commit quality\n'} - - ${commitAnalysis} - - ${testSection} - - ### PDCA Framework Alignment: - - **Plan:** ${hasIncrementalPattern ? ':white_check_mark: Shows incremental planning' : ':yellow_circle: Consider smaller, planned increments'} - - **Do:** ${commitMetrics.length > 0 ? ':white_check_mark: Implementation tracked in commits' : ':x: No commit history'} - - **Check:** ${testAdequacy === 'good' || testAdequacy === 'excellent' ? ':white_check_mark: Good validation through tests' : ':yellow_circle: Needs better validation'} - - **Act:** ${hasRefactoringPattern ? ':white_check_mark: Shows continuous improvement' : ':information_source: Consider refactoring opportunities'} - - *Automated by Enhanced PDCA Framework - analyzing both PR and commit-level metrics* - `; - - github.rest.issues.createComment({ + + // --------------------------------------------------------------- + // DORA archetype + // --------------------------------------------------------------- + + const archetype = commitMetrics.length > 0 + ? classifyDoraArchetype(largePct, sprawlingPct, testFirstPct, msgQualityPct) + : 'mixed-signals'; + + const archetypeDescriptions = { + 'harmonious-high-achiever': 'Strong batch discipline, test coverage, and message quality. AI tools likely amplifying positive outcomes.', + 'foundational-challenges': 'Weak testing or batch discipline detected. Consider strengthening practices before scaling AI usage.', + 'legacy-bottleneck': 'High sprawl combined with large commits. AI may be making cross-cutting changes worse.', + 'mixed-signals': 'No clear archetype pattern. Review individual metric thresholds.' + }; + + // --------------------------------------------------------------- + // PR comment + // --------------------------------------------------------------- + + const breakdown = [ + `**Production Code:** ${prodChanges} lines (${prodFilesCount} files)`, + `**Test Code:** ${testChanges} lines (${testFilesCount} files)`, + `**Total:** ${totalChanges} lines (${totalChangedFiles} files)`, + prodChanges > 0 ? `**Test-to-Production Ratio:** ${testRatio.toFixed(2)}:1` : '' + ].filter(Boolean).join('\n'); + + const commitRows = commitMetrics.slice(0, 10).map(c => { + const flags = [ + c.large_commit ? 'large' : '', + c.sprawling_commit ? 'sprawling' : '', + c.test_first_indicator ? 'test+prod' : '', + !c.message_quality ? 'vague-msg' : '' + ].filter(Boolean).join(', '); + return `**${c.sha}** ${c.author} (${c.date})\n\`${c.message}\`\n${c.prod_changes} prod lines, ${c.files_changed} files${flags ? ` [${flags}]` : ''}`; + }).join('\n\n'); + + const doraSection = commitMetrics.length > 0 ? [ + '### DORA Capability Assessment', + '', + `**Archetype:** ${archetype}`, + archetypeDescriptions[archetype], + '', + '| Capability | Metric | Value | Target |', + '|------------|--------|-------|--------|', + `| Small Batches | Large commit % | ${largePct.toFixed(0)}% | <20% |`, + `| Small Batches | Sprawling commit % | ${sprawlingPct.toFixed(0)}% | <10% |`, + `| Version Control | Test-first discipline | ${testFirstPct.toFixed(0)}% | >50% |`, + `| Version Control | Message quality | ${msgQualityPct.toFixed(0)}% | >60% |`, + `| AI Risk Signal | Additions ratio (median) | ${medianAdditionsRatio.toFixed(2)} | <3.0 |` + ].join('\n') : ''; + + const commentParts = [ + '## PR Analysis', + '', + `**Size:** ${sizeLabel} (based on production code)`, + '', + breakdown, + '', + concerns.length > 0 + ? '### Concerns\n\n' + concerns.map(c => `- ${c}`).join('\n') + : 'No size or quality concerns.', + '', + strengths.length > 0 + ? '### Strengths\n\n' + strengths.map(s => `- ${s}`).join('\n') + : '', + '', + '## Commit Analysis', + '', + `**Total Commits:** ${commitMetrics.length}`, + `**Average Commit Size:** ${avgCommitSize.toFixed(0)} production lines`, + `**Average Files per Commit:** ${avgFilesPerCommit.toFixed(1)}`, + '', + '| Metric | Value |', + '|--------|-------|', + `| Large commits (>100 prod lines) | ${largeCommits}/${commitMetrics.length} (${largePct.toFixed(0)}%) |`, + `| Sprawling commits (>5 files) | ${sprawlingCommits}/${commitMetrics.length} (${sprawlingPct.toFixed(0)}%) |`, + `| Test-first discipline | ${testFirstCommits}/${commitMetrics.length} (${testFirstPct.toFixed(0)}%) |`, + `| Message quality | ${qualityCommits}/${commitMetrics.length} (${msgQualityPct.toFixed(0)}%) |`, + `| Median additions ratio | ${medianAdditionsRatio.toFixed(2)} |`, + `| Test-only commits | ${testOnlyCommits} |`, + `| Production-only commits | ${prodOnlyCommits} |`, + '', + commitMetrics.length <= 10 ? '### Commit Details\n\n' + commitRows : '', + '', + testAdequacy !== 'n/a' ? [ + '## Test Coverage', + '', + `**Test Adequacy:** ${testAdequacy}`, + '', + testPraise.length > 0 ? testPraise.map(p => `- ${p}`).join('\n') : '', + testConcerns.length > 0 ? testConcerns.map(c => `- ${c}`).join('\n') : '', + '', + '**Target ratio:** 0.5-2.0 test lines per production line' + ].join('\n') : '', + '', + doraSection, + '', + '_Automated by Code Metrics Workflow_' + ].filter(s => s !== '').join('\n'); + + await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: comment - }); \ No newline at end of file + body: commentParts + }); diff --git a/.gitignore b/.gitignore index 7decb47..afd32dc 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,14 @@ !.beads/issues.jsonl !.beads/config.yaml +# Environment variables — never commit API keys +.env +.env.local + # Generated output files local_commit_metrics.json local_metrics_summary.json +RETROSPECTIVE*.md # Test coverage coverage/ diff --git a/CLAUDE.md b/CLAUDE.md index d98c781..2e35596 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Purpose -This toolkit detects **AI code drift** — problematic patterns that emerge when teams adopt AI coding tools. It captures metrics *before* merge squashing destroys the signals, making visible how AI tools actually affect code quality. +This toolkit detects **AI code drift**: problematic patterns that emerge when teams adopt AI coding tools. It captures metrics *before* merge squashing destroys the signals, making visible how AI tools actually affect code quality. Key insight: Local analysis reveals 10x higher drift rates than remote analysis because `git merge --squash` and branch deletion destroy granular commit-level signals. @@ -26,13 +26,13 @@ npm test # run all tests npm run test:coverage # tests with coverage report (thresholds: 80% lines, 90% functions) npm run test:watch # watch mode npx jest __tests__/parseGitLog.test.js # run a single test file -npm run lint # ESLint (flat config, globals.node required — already configured) +npm run lint # ESLint (flat config, globals.node required; already configured) npm run typecheck # tsc --noEmit (checks local-code-metrics.js via @ts-check + tsconfig.json) ``` -All tests mock `child_process` and `fs` — no git repository required to run the suite. +All tests mock `child_process` and `fs`. No git repository is required to run the suite. -A pre-commit hook runs lint → typecheck → test automatically. After cloning, activate it with: +A pre-commit hook runs lint, typecheck, and test automatically. After cloning, activate it with: ```bash npm install # triggers `prepare`, which sets core.hooksPath to .githooks @@ -40,13 +40,20 @@ npm install # triggers `prepare`, which sets core.hooksPath to .githooks ## Architecture -Three components, no shared code between them: +Three public components with no shared code between them: -1. **`local-code-metrics.js`** — Standalone Node.js script. Reads local git history via shell commands, classifies files as test vs. production, computes metrics, writes `local_commit_metrics.json` + `local_metrics_summary.json`, and prints a console report with insights. +1. **`local-code-metrics.js`**: Standalone Node.js script (requires Node ≥18). Orchestration entry point that delegates to focused modules in `lib/`. Reads local git history via shell commands, classifies files as test vs. production, computes metrics, writes `local_commit_metrics.json` + `local_metrics_summary.json` + (optionally) `local_claude_analysis.json`, and prints a console report with insights. -2. **`.github/workflows/code-metrics.yml`** — Weekly GitHub Actions workflow. Uses the GitHub API to analyze feature branches from the past 30 days. Outputs a JSON artifact and creates a GitHub issue with the summary. + The `lib/` directory contains the internal modules (not shared with workflows): + - `lib/config.js` — CONFIG object; single source of truth for all thresholds + - `lib/git.js` — git shell commands, log parsing, per-commit analysis, diff extraction + - `lib/statistics.js` — statistical distributions (p50/p90/p95/stddev), velocity and trend + - `lib/metrics.js` — message quality scoring, DORA archetype classification, insights generation + - `lib/claude.js` — Anthropic client setup, commit pre-filtering, diff-level API analysis -3. **`.github/workflows/pr-metrics.yml`** — Per-PR GitHub Actions workflow. Posts a detailed comment on each PR with commit-by-commit analysis, test adequacy, and development pattern detection. +2. **`.github/workflows/code-metrics.yml`**: Weekly GitHub Actions workflow. Uses the GitHub API to analyze feature branches from the past 30 days. Outputs a JSON artifact and creates a GitHub issue with the summary. + +3. **`.github/workflows/pr-metrics.yml`**: Per-PR GitHub Actions workflow. Posts a detailed comment on each PR with commit-by-commit analysis, test adequacy, and development pattern detection. ## Key Metrics and Thresholds @@ -56,12 +63,48 @@ Three components, no shared code between them: | Sprawling commit % (>5 files) | <10% | | Test-to-production ratio | 0.5–2.0:1 | | Avg files changed per commit | <5 | +| Commit message quality % | >60% | +| Additions-to-deletions ratio (median) | <3.0 | + +Statistical distributions (p50/p90/p95/stddev) are computed for lines changed and files changed. Commit velocity trend and DORA team archetype are included in the summary. + +### DORA Archetype Classification + +The summary includes a `dora_archetype` field classifying the repository into one of four team archetypes based on large commit %, sprawling commit %, test-first %, and message quality %: + +| Archetype | Signal | +|-----------|--------| +| `harmonious-high-achiever` | All four metrics in healthy range | +| `legacy-bottleneck` | High sprawl (>25%) + high large commits (>30%) | +| `foundational-challenges` | Large commits >40%, or low test discipline + elevated large commits | +| `mixed-signals` | No clear archetype threshold breached | + +### Claude API Integration (Optional) + +Set `ANTHROPIC_API_KEY` to enable diff-level analysis of high-risk commits. When active: +- Up to 5 commits are selected (large commits with additions > deletions × 3) +- Each commit is analyzed for AI-generated code patterns and architectural concerns +- Results are written to `local_claude_analysis.json` +- Commit metrics are annotated with `ai_confidence`, `risk_score`, `patterns`, and `architectural_concerns` + +The script degrades gracefully when the key is absent. No SDK install is required to run. ## Configuration -Thresholds are configured in the `CONFIG` object at the top of `local-code-metrics.js`. The GitHub workflows have equivalent values hard-coded in their shell/jq logic. When adjusting thresholds, update both places. +Thresholds are configured in the `CONFIG` object in `lib/config.js`, which is the single source of truth for the local script. Key values: + +| Key | Default | Description | +|-----|---------|-------------| +| `LARGE_COMMIT_THRESHOLD` | 100 | Prod lines changed to flag as large | +| `SPRAWLING_COMMIT_THRESHOLD` | 5 | Files changed to flag as sprawling | +| `MESSAGE_QUALITY_MIN_WORDS` | 10 | Word count threshold for non-conventional messages | +| `AI_ANALYSIS_MAX_COMMITS` | 5 | Max commits sent to Claude per run | +| `AI_DIFF_MAX_CHARS` | 4000 | Diff truncation limit for Claude API calls | +| `AI_RISK_ADDITIONS_RATIO` | 3 | Additions/deletions multiplier for Claude pre-filter | + +The GitHub workflows have equivalent values hard-coded in their shell/jq logic. When adjusting thresholds, update both places. -Test file detection uses patterns for JS, Python, Go, Java, and C# — extend `TEST_FILE_PATTERNS` in the script or the equivalent grep patterns in the workflows for other languages. +Test file detection uses patterns for JS, Python, Go, Java, and C#. Extend `TEST_FILE_PATTERNS` in `lib/config.js` or the equivalent grep patterns in the workflows for other languages. ## Workflow Permissions diff --git a/README.md b/README.md index 8d77d0b..6053ae2 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,21 @@ Research shows that AI coding tools can lead to increased batch sizes, reduced r ## Tools Included -### 1. 📊 Weekly AI Code Drift Metrics (GitHub Actions) -- **File:** `.github/workflows/ai-code-drift-metrics.yml` +### 1. Weekly AI Code Drift Metrics (GitHub Actions) +- **File:** `.github/workflows/code-metrics.yml` - **Purpose:** Automated weekly analysis of feature branches - **Output:** GitHub issues with trend analysis and artifacts -### 2. 🚦 Real-time PR Size Analysis (GitHub Actions) -- **File:** `.github/workflows/pr-size-analysis.yml` +### 2. Real-time PR Size Analysis (GitHub Actions) +- **File:** `.github/workflows/pr-metrics.yml` - **Purpose:** Immediate feedback on every pull request - **Output:** PR comments with size warnings and recommendations -### 3. 🔍 Local Repository Analysis (Node.js Script) -- **File:** `local-ai-drift-analysis.js` +### 3. Local Repository Analysis (Node.js Script) +- **File:** `local-code-metrics.js` - **Purpose:** Immediate analysis of your local development patterns - **Output:** Console report and JSON files with detailed metrics +- **Requires:** Node.js >= 18 ## Quick Start @@ -31,23 +32,24 @@ Research shows that AI coding tools can lead to increased batch sizes, reduced r ```bash # 1. Copy workflows to your repository mkdir -p .github/workflows -curl -o .github/workflows/ai-code-drift-metrics.yml https://raw.githubusercontent.com/yourrepo/toolkit/main/ai-code-drift-metrics.yml -curl -o .github/workflows/pr-size-analysis.yml https://raw.githubusercontent.com/yourrepo/toolkit/main/pr-size-analysis.yml +# Copy .github/workflows/code-metrics.yml and pr-metrics.yml from this repo # 2. Ensure feature branches aren't auto-deleted # Go to repo Settings > General > Pull Requests # Uncheck "Automatically delete head branches" # 3. Run manually or wait for weekly schedule +gh workflow run code-metrics.yml ``` ### Option 2: Local Analysis ```bash -# 1. Download and run the local script -curl -o local-ai-drift-analysis.js https://raw.githubusercontent.com/yourrepo/toolkit/main/local-ai-drift-analysis.js -node local-ai-drift-analysis.js +# 1. Clone and run +npm install +node local-code-metrics.js # 2. Review the generated JSON files and console output +# Optional: set ANTHROPIC_API_KEY for Claude diff analysis ``` ## Key Metrics Tracked @@ -56,20 +58,21 @@ node local-ai-drift-analysis.js |--------|--------|---------| | **Large Commit %** | <20% | Detects batch AI code acceptance | | **Sprawling Commit %** | <10% | Identifies scattered changes across files | -| **Test-First Discipline** | Trending ↗ | Monitors TDD practices with AI tools | +| **Test-First Discipline** | >50% | Monitors TDD practices with AI tools | +| **Message Quality %** | >60% | Conventional commits or descriptive messages | +| **Additions/Deletions Ratio (median)** | <3.0 | Flags batch-acceptance pattern | | **Avg Files Changed** | <5 | Measures development granularity | -| **Avg Lines Changed** | <100 | Detects wholesale AI code acceptance | ## Real-World Example **Remote Repository Analysis (misleading):** - 4 commits over 30 days -- 0% large commits +- 0% large commits - 8 lines average per commit **Local Repository Analysis (reality):** - 50 commits across 4 feature branches -- **46% large commits** +- **46% large commits** - **9,053 lines average per commit** - Clear AI drift patterns hidden by merge squashing @@ -81,7 +84,7 @@ Customize test file patterns for your language: ```javascript // In workflows or local script CONFIG TEST_FILE_PATTERNS: [ - /\.(test|spec)\./i, // JavaScript/TypeScript + /\.(test|spec)\./i, // JavaScript/TypeScript /Tests?\.cs$/i, // C# (FileTests.cs) /Test\.java$/i, // Java (FileTest.java) /_test\.py$/i, // Python (file_test.py) @@ -95,45 +98,52 @@ TEST_FILE_PATTERNS: [ Adjust warning thresholds based on your team: ```javascript -// Large commit threshold (lines) -large_commit: (additions + deletions) > 100, - -// Sprawling commit threshold (files) -sprawling_commit: files_changed > 5, - -// Analysis period (days) -ANALYSIS_DAYS: 30, +LARGE_COMMIT_THRESHOLD: 100, // lines changed +SPRAWLING_COMMIT_THRESHOLD: 5, // files changed +ANALYSIS_DAYS: 30, // lookback window +MESSAGE_QUALITY_MIN_WORDS: 10, // words for non-conventional messages +AI_RISK_ADDITIONS_RATIO: 3, // additions/deletions multiplier for Claude pre-filter +AI_ANALYSIS_MAX_COMMITS: 5, // max commits sent to Claude per run ``` ## Understanding Results -### 🟢 Healthy Patterns +### Healthy Patterns ``` Large commits: <20% Sprawling commits: <10% Test-first discipline: >50% -Avg files changed: <5 -Avg lines changed: <100 +Message quality: >60% +Additions ratio (median): <3.0 ``` -### 🟡 Warning Signs +### Warning Signs ``` Large commits: 20-40% Sprawling commits: 10-25% Test-first discipline: 30-50% -Avg files changed: 5-10 -Avg lines changed: 100-500 +Additions ratio (median): 3.0-6.0 ``` -### 🔴 Critical Issues +### Critical Issues ``` Large commits: >40% -Sprawling commits: >25% +Sprawling commits: >25% Test-first discipline: <30% -Avg files changed: >10 -Avg lines changed: >500 +Additions ratio (median): >6.0 ``` +## DORA Archetype Classification + +The summary includes a `dora_archetype` field: + +| Archetype | Signal | +|-----------|--------| +| `harmonious-high-achiever` | All metrics in healthy range | +| `legacy-bottleneck` | High sprawl + high large commits | +| `foundational-challenges` | Large commits >40% or low test discipline | +| `mixed-signals` | No clear threshold breached | + ## Workflow Outputs ### Weekly Metrics Report (GitHub Issue) @@ -147,50 +157,50 @@ Avg lines changed: >500 ### Key Metrics | Metric | Value | Target | Status | |--------|-------|--------|---------| -| Large Commits | 28% | <20% | ⚠️ | -| Sprawling Commits | 12% | <10% | ⚠️ | -| Test-First Discipline | 64% | Trending ↗ | ✅ | +| Large Commits | 28% | <20% | Warning | +| Sprawling Commits | 12% | <10% | Warning | +| Test-First Discipline | 64% | >50% | OK | ### Interpretation -⚠️ **Large commits above 20% threshold** - Consider breaking down AI-generated code -⚠️ **Sprawling commits above 10% threshold** - Review AI suggestions for scope creep +**Large commits above 20% threshold** - Consider breaking down AI-generated code +**Sprawling commits above 10% threshold** - Review AI suggestions for scope creep ``` ### PR Size Analysis (PR Comment) -```markdown -## 🟠 PR Size Analysis +```markdown +## PR Size Analysis **Size Classification:** large **Total Changes:** 847 lines (+782, -65) **Files Changed:** 12 -### ⚠️ Concerns: -- ⚠️ **Large PR** - May indicate batch acceptance of AI-generated code -- ⚠️ **Multiple files changed** - Ensure changes are cohesive +### Concerns: +- **Large PR** - May indicate batch acceptance of AI-generated code +- **Multiple files changed** - Ensure changes are cohesive -### 💡 Recommendations: +### Recommendations: - Review carefully for AI-generated patterns that should be broken down - Consider splitting into focused, single-responsibility PRs ``` ### Local Script Output -```bash -=== 📊 ANALYSIS RESULTS === - -📈 Total commits analyzed: 50 -📏 Large commits (>100 lines): 46.00% -📁 Sprawling commits (>5 files): 20.00% -🧪 Test-first discipline: 58.00% -📂 Average files changed: 6.42 -📝 Average lines changed: 9,053 - -=== ⚠️ CONCERNS DETECTED === -🚨 Very high large commit rate (46%) - Strong AI drift indicators -⚠️ High sprawling commit rate (20%) - Watch for scope creep - -=== 💡 RECOMMENDATIONS === -• Consider breaking AI-generated code into smaller, focused commits -• Review if AI suggestions are causing scattered changes across files +``` +=== ANALYSIS RESULTS === + +Total commits analyzed: 50 +Large commits (>100 lines): 46.00% +Sprawling commits (>5 files): 20.00% +Test-first discipline: 58.00% +Average files changed: 6.42 +Average lines changed: 9,053 + +=== CONCERNS DETECTED === +[CRITICAL] Very high large commit rate (46%) - Strong AI drift indicators +[WARNING] High sprawling commit rate (20%) - Watch for scope creep + +=== RECOMMENDATIONS === +- Consider breaking AI-generated code into smaller, focused commits +- Review if AI suggestions are causing scattered changes across files ``` ## Prerequisites @@ -200,20 +210,22 @@ Avg lines changed: >500 - Feature branches preserved after merging (disable auto-delete) - Repository permissions for creating issues and PR comments -### For Local Script -- Node.js (v12 or later) +### For Local Script +- Node.js >= 18 - Git repository with local feature branches - Command line access +- Optional: `ANTHROPIC_API_KEY` for Claude diff-level analysis ## File Structure ``` your-repo/ ├── .github/workflows/ -│ ├── ai-code-drift-metrics.yml # Weekly analysis -│ └── pr-size-analysis.yml # Real-time PR feedback -├── local-ai-drift-analysis.js # Local analysis script -├── local_commit_metrics.json # Generated: detailed data -└── local_metrics_summary.json # Generated: summary stats +│ ├── code-metrics.yml # Weekly analysis +│ └── pr-metrics.yml # Real-time PR feedback +├── local-code-metrics.js # Local analysis script +├── local_commit_metrics.json # Generated: detailed data +├── local_metrics_summary.json # Generated: summary stats +└── local_claude_analysis.json # Generated: Claude analysis (optional) ``` ## Integration Examples @@ -254,17 +266,17 @@ your-repo/ ## Why This Matters **The Problem:** Teams adopting AI tools often see: -- ✅ Faster initial coding -- ❌ Larger, harder-to-review commits -- ❌ Reduced refactoring discipline -- ❌ Technical debt accumulation -- ❌ Net productivity loss over time +- Faster initial coding +- Larger, harder-to-review commits +- Reduced refactoring discipline +- Technical debt accumulation +- Net productivity loss over time **The Solution:** Measure development patterns before they're hidden by workflow processing: -- 🔍 **Early detection** of problematic AI usage -- 📊 **Quantified feedback** for development process improvement -- 🚦 **Real-time prevention** through PR size controls -- 📈 **Trend analysis** to track team improvement +- **Early detection** of problematic AI usage +- **Quantified feedback** for development process improvement +- **Real-time prevention** through PR size controls +- **Trend analysis** to track team improvement ## Related Research @@ -273,9 +285,11 @@ This toolkit implements the methodology described in: Key findings: - Merge squashing destroys 90%+ of AI drift signals -- Local analysis reveals 10x higher drift rates than remote analysis +- Local analysis reveals 10x higher drift rates than remote analysis - Teams can maintain quality with proper measurement and discipline +See also: [metrics-specification.md](metrics-specification.md) for the full technical reference. + ## License This work is licensed under [Creative Commons Attribution 4.0 International (CC BY 4.0)](https://creativecommons.org/licenses/by/4.0/). @@ -284,20 +298,20 @@ You are free to share and adapt this material for any purpose, including commerc ## Contributing -Improvements welcome! Particularly valuable: +Improvements welcome. Particularly valuable: - Additional test file patterns for different languages -- Enhanced AI pattern detection algorithms +- Enhanced AI pattern detection algorithms - Better threshold recommendations for different project types - Integration examples with other development tools ## Support -- 📖 **Documentation:** This README covers all common use cases -- 🐛 **Issues:** Report bugs or request features in the GitHub issues -- 💬 **Discussions:** Share your results and insights with the community +- **Documentation:** This README and [metrics-specification.md](metrics-specification.md) cover all common use cases +- **Issues:** Report bugs or request features in the GitHub issues +- **Discussions:** Share your results and insights with the community --- **Attribution:** Based on research by Ken Judy. Please cite when using or adapting these tools. -**Citation:** Judy, K. (2025). Measuring AI Code Drift: Working with GitHub's Available Metrics to Track LLM Impact on Existing Codebases. https://github.com/stride-nyc/code-quality-metrics \ No newline at end of file +**Citation:** Judy, K. (2025). Measuring AI Code Drift: Working with GitHub's Available Metrics to Track LLM Impact on Existing Codebases. https://github.com/stride-nyc/code-quality-metrics diff --git a/__tests__/claudeAnalysis.test.js b/__tests__/claudeAnalysis.test.js new file mode 100644 index 0000000..ece2049 --- /dev/null +++ b/__tests__/claudeAnalysis.test.js @@ -0,0 +1,215 @@ +'use strict'; + +jest.mock('@anthropic-ai/sdk', () => ({ + Anthropic: jest.fn().mockImplementation(() => ({ + messages: { + create: jest.fn().mockResolvedValue({ + content: [{ type: 'text', text: JSON.stringify({ + ai_confidence: 75, + risk_score: 80, + patterns: ['generic variable names'], + architectural_concerns: [], + summary: 'Test summary' + }) }] + }) + } + })) +}), { virtual: true }); + +jest.mock('child_process'); +jest.mock('fs'); + +const { execSync } = require('child_process'); +const { + getAnthropicClient, + selectClaudeCommits, + getCommitDiff, + analyzeWithClaude, + CONFIG, +} = require('../local-code-metrics'); + +beforeEach(() => { + jest.clearAllMocks(); + delete process.env.ANTHROPIC_API_KEY; +}); + +afterEach(() => { + delete process.env.ANTHROPIC_API_KEY; +}); + +// --------------------------------------------------------------------------- +// getAnthropicClient +// --------------------------------------------------------------------------- + +describe('getAnthropicClient', () => { + test('returns null when ANTHROPIC_API_KEY is not set', async () => { + const client = await getAnthropicClient(); + expect(client).toBeNull(); + }); + + test('returns a client object when ANTHROPIC_API_KEY is set', async () => { + process.env.ANTHROPIC_API_KEY = 'test-key'; + const client = await getAnthropicClient(); + expect(client).not.toBeNull(); + expect(typeof client.messages.create).toBe('function'); + }); +}); + +// --------------------------------------------------------------------------- +// getCommitDiff +// --------------------------------------------------------------------------- + +describe('getCommitDiff', () => { + test('combines stat and diff output into a single string', () => { + execSync + .mockReturnValueOnce('src/app.js | 10 ++++') // git show --stat + .mockReturnValueOnce('+const x = 1;'); // git show diff + + const result = getCommitDiff('abc123'); + expect(result).toContain('File Summary'); + expect(result).toContain('src/app.js | 10 ++++'); + expect(result).toContain('Diff'); + expect(result).toContain('+const x = 1;'); + }); + + test('truncates output to AI_DIFF_MAX_CHARS', () => { + const bigOutput = 'x'.repeat(CONFIG.AI_DIFF_MAX_CHARS + 1000); + execSync + .mockReturnValueOnce(bigOutput) + .mockReturnValueOnce(bigOutput); + + const result = getCommitDiff('abc123'); + expect(result.length).toBeLessThanOrEqual(CONFIG.AI_DIFF_MAX_CHARS); + }); + + test('returns valid string even when git commands return empty', () => { + execSync.mockReturnValue(''); + const result = getCommitDiff('abc123'); + expect(typeof result).toBe('string'); + expect(result).toContain('File Summary'); + }); +}); + +// --------------------------------------------------------------------------- +// analyzeWithClaude +// --------------------------------------------------------------------------- + +describe('analyzeWithClaude', () => { + const { Anthropic } = require('@anthropic-ai/sdk'); + + function makeClient() { + return new Anthropic(); + } + + const COMMIT = { + sha: 'abc1234', + full_sha: 'abc1234' + '0'.repeat(33), + message: 'feat: add thing', + author: 'Dev', + date: '2024-01-15T10:00:00Z', + source_branch: 'feature/x', + }; + + beforeEach(() => { + // getCommitDiff calls execSync twice per commit + execSync.mockReturnValue('mock diff content'); + }); + + test('returns result with ai_confidence and risk_score for a qualifying commit', async () => { + const client = makeClient(); + const results = await analyzeWithClaude(client, [COMMIT]); + expect(results).toHaveLength(1); + expect(results[0].sha).toBe(COMMIT.sha); + expect(results[0].ai_confidence).toBe(75); + expect(results[0].risk_score).toBe(80); + }); + + test('returns result with patterns and architectural_concerns arrays', async () => { + const client = makeClient(); + const results = await analyzeWithClaude(client, [COMMIT]); + expect(Array.isArray(results[0].patterns)).toBe(true); + expect(Array.isArray(results[0].architectural_concerns)).toBe(true); + }); + + test('records error and continues when API call throws', async () => { + const client = makeClient(); + client.messages.create.mockRejectedValueOnce(new Error('rate limited')); + + const results = await analyzeWithClaude(client, [COMMIT]); + expect(results).toHaveLength(1); + expect(results[0].error).toMatch(/rate limited/); + }); + + test('records error when response is not valid JSON', async () => { + const client = makeClient(); + client.messages.create.mockResolvedValueOnce({ + content: [{ type: 'text', text: 'not json at all' }] + }); + + const results = await analyzeWithClaude(client, [COMMIT]); + expect(results).toHaveLength(1); + expect(results[0].error).toBeDefined(); + }); + + test('strips markdown code fences from response before parsing', async () => { + const client = makeClient(); + const payload = { ai_confidence: 50, risk_score: 60, patterns: [], architectural_concerns: [], summary: 'ok' }; + client.messages.create.mockResolvedValueOnce({ + content: [{ type: 'text', text: '```json\n' + JSON.stringify(payload) + '\n```' }] + }); + + const results = await analyzeWithClaude(client, [COMMIT]); + expect(results[0].ai_confidence).toBe(50); + }); +}); + +// --------------------------------------------------------------------------- +// selectClaudeCommits +// --------------------------------------------------------------------------- + +describe('selectClaudeCommits', () => { + function makeMetric(overrides) { + return { + sha: 'abc123', + large_commit: false, + total_additions: 10, + total_deletions: 10, + ...overrides + }; + } + + test('returns only large commits where additions exceed deletions * AI_RISK_ADDITIONS_RATIO', () => { + const metrics = [ + makeMetric({ sha: 'aaa', large_commit: true, total_additions: 400, total_deletions: 10 }), // qualifies + makeMetric({ sha: 'bbb', large_commit: false, total_additions: 400, total_deletions: 10 }), // not large + makeMetric({ sha: 'ccc', large_commit: true, total_additions: 10, total_deletions: 10 }), // ratio too low + ]; + const result = selectClaudeCommits(metrics); + expect(result).toHaveLength(1); + expect(result[0].sha).toBe('aaa'); + }); + + test('caps results at AI_ANALYSIS_MAX_COMMITS', () => { + const metrics = Array.from({ length: CONFIG.AI_ANALYSIS_MAX_COMMITS + 3 }, (_, i) => ( + makeMetric({ sha: `sha${i}`, large_commit: true, total_additions: 500, total_deletions: 10 }) + )); + const result = selectClaudeCommits(metrics); + expect(result).toHaveLength(CONFIG.AI_ANALYSIS_MAX_COMMITS); + }); + + test('returns empty array when no commits qualify', () => { + const metrics = [ + makeMetric({ sha: 'zzz', large_commit: false, total_additions: 10, total_deletions: 10 }), + ]; + expect(selectClaudeCommits(metrics)).toHaveLength(0); + }); + + test('sorts by total churn descending before capping', () => { + const metrics = [ + makeMetric({ sha: 'small', large_commit: true, total_additions: 150, total_deletions: 10 }), + makeMetric({ sha: 'large', large_commit: true, total_additions: 900, total_deletions: 10 }), + ]; + const result = selectClaudeCommits(metrics); + expect(result[0].sha).toBe('large'); + }); +}); diff --git a/__tests__/collectLocalMetrics.test.js b/__tests__/collectLocalMetrics.test.js index 322a559..f6277bb 100644 --- a/__tests__/collectLocalMetrics.test.js +++ b/__tests__/collectLocalMetrics.test.js @@ -2,9 +2,11 @@ jest.mock('child_process'); jest.mock('fs'); +jest.mock('../lib/claude'); const { execSync } = require('child_process'); const fs = require('fs'); +const claude = require('../lib/claude'); const { collectLocalMetrics } = require('../local-code-metrics'); const FAKE_ROOT = '/fake/repo'; @@ -17,6 +19,8 @@ beforeEach(() => { jest.spyOn(console, 'log').mockImplementation(() => {}); jest.spyOn(console, 'error').mockImplementation(() => {}); jest.spyOn(console, 'warn').mockImplementation(() => {}); + // Default: Claude skipped — overridden only in Claude-active tests + claude.getAnthropicClient.mockResolvedValue(null); }); afterEach(() => { @@ -130,6 +134,34 @@ describe('collectLocalMetrics — successful run', () => { expect(typeof summary.avg_lines_changed).toBe('string'); }); + test('writes local_metrics_summary.json with DORA metric fields', async () => { + await collectLocalMetrics(); + + const summaryCall = fs.writeFileSync.mock.calls.find(c => c[0].includes('local_metrics_summary')); + const summary = JSON.parse(summaryCall[1]); + expect(typeof summary.velocity_commits_per_day).toBe('number'); + expect(['accelerating', 'stable', 'decelerating']).toContain(summary.velocity_trend); + expect(typeof summary.additions_ratio_median).toBe('number'); + expect(typeof summary.additions_ratio_p90).toBe('number'); + expect(typeof summary.message_quality_pct).toBe('string'); + expect(['harmonious-high-achiever', 'foundational-challenges', 'legacy-bottleneck', 'mixed-signals']) + .toContain(summary.dora_archetype); + }); + + test('writes local_metrics_summary.json with statistical distribution fields', async () => { + await collectLocalMetrics(); + + const summaryCall = fs.writeFileSync.mock.calls.find(c => c[0].includes('local_metrics_summary')); + const summary = JSON.parse(summaryCall[1]); + expect(typeof summary.p50_lines_changed).toBe('number'); + expect(typeof summary.p90_lines_changed).toBe('number'); + expect(typeof summary.p95_lines_changed).toBe('number'); + expect(typeof summary.stddev_lines_changed).toBe('number'); + expect(typeof summary.p50_files_changed).toBe('number'); + expect(typeof summary.p90_files_changed).toBe('number'); + expect(['growing', 'stable', 'shrinking']).toContain(summary.commit_size_trend); + }); + test('logs warnings and recommendations when commits are large', async () => { // 101 added lines → 100% large commit rate → critical warning + recommendation const bigNumstat = `101\t0\tsrc/app.js`; @@ -173,6 +205,14 @@ describe('collectLocalMetrics — successful run', () => { expect(allLogs).toMatch(/and \d+ more commits/); }); + test('logs Claude-skipped message when ANTHROPIC_API_KEY is absent', async () => { + delete process.env.ANTHROPIC_API_KEY; + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + await collectLocalMetrics(); + const allLogs = logSpy.mock.calls.flat().join(' '); + expect(allLogs).toMatch(/Claude analysis skipped/); + }); + test('deduplicates commits with the same SHA across branches', async () => { // Two branches both surface the same commit SHA mockExecSequence( @@ -191,3 +231,70 @@ describe('collectLocalMetrics — successful run', () => { expect(written).toHaveLength(1); }); }); + +describe('collectLocalMetrics — Claude API active', () => { + const SHA = 'a'.repeat(40); + const NUMSTAT = `110\t5\tsrc/app.js`; // 115 prod lines → large_commit = true, additions >> deletions + + beforeEach(() => { + mockExecSequence( + FAKE_ROOT, + FAKE_REMOTE, + ' feature/x', + `${SHA}|2024-01-15T10:00:00Z|Dev|feat: add thing`, + NUMSTAT + ); + fs.writeFileSync.mockImplementation(() => {}); + claude.selectClaudeCommits.mockReturnValue([{ + sha: SHA.substring(0, 8), + full_sha: SHA, + message: 'feat: add thing', + author: 'Dev', + date: '2024-01-15T10:00:00Z', + source_branch: 'feature/x' + }]); + claude.analyzeWithClaude.mockResolvedValue([{ + sha: SHA.substring(0, 8), + ai_confidence: 75, + risk_score: 80, + patterns: ['generic variable names'], + architectural_concerns: [], + summary: 'Possible AI-generated code' + }]); + }); + + test('annotates metrics and writes local_claude_analysis.json when Claude returns results', async () => { + claude.getAnthropicClient.mockResolvedValue({}); + + await collectLocalMetrics(); + + // Three files: metrics, summary, claude analysis + expect(fs.writeFileSync).toHaveBeenCalledTimes(3); + + const claudeCall = fs.writeFileSync.mock.calls.find(c => c[0].includes('local_claude_analysis')); + expect(claudeCall).toBeDefined(); + const claudeOutput = JSON.parse(claudeCall[1]); + expect(claudeOutput.model).toBe('claude-sonnet-4-6'); + expect(claudeOutput.commits_analyzed).toBe(1); + expect(claudeOutput.results).toHaveLength(1); + + // Metric should be annotated with Claude fields + const metricsCall = fs.writeFileSync.mock.calls.find(c => c[0].includes('local_commit_metrics')); + const metrics = JSON.parse(metricsCall[1]); + expect(metrics[0].ai_confidence).toBe(75); + expect(metrics[0].risk_score).toBe(80); + expect(metrics[0].patterns).toEqual(['generic variable names']); + }); + + test('logs Claude analysis section to console when metrics are annotated', async () => { + claude.getAnthropicClient.mockResolvedValue({}); + const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {}); + + await collectLocalMetrics(); + + const allLogs = logSpy.mock.calls.flat().join(' '); + expect(allLogs).toMatch(/CLAUDE AI ANALYSIS/); + expect(allLogs).toMatch(/confidence=75%/); + expect(allLogs).toMatch(/risk=80%/); + }); +}); diff --git a/__tests__/commitVelocity.test.js b/__tests__/commitVelocity.test.js new file mode 100644 index 0000000..58c6307 --- /dev/null +++ b/__tests__/commitVelocity.test.js @@ -0,0 +1,68 @@ +'use strict'; + +const { computeVelocity } = require('../local-code-metrics'); + +describe('computeVelocity', () => { + it('returns stable trend and zero rate for empty array', () => { + const result = computeVelocity([]); + expect(result.commits_per_day).toBe(0); + expect(result.trend).toBe('stable'); + }); + + it('returns stable trend for single commit', () => { + const result = computeVelocity([new Date('2024-01-15T10:00:00Z').toISOString()]); + expect(result.commits_per_day).toBeGreaterThanOrEqual(0); + expect(result.trend).toBe('stable'); + }); + + it('returns "accelerating" when second half has proportionally more commits', () => { + // 10 days: 2 commits in first 5 days, 8 in second 5 days + const base = new Date('2024-01-01T00:00:00Z').getTime(); + const dates = [ + new Date(base + 1 * 86400000).toISOString(), + new Date(base + 2 * 86400000).toISOString(), + new Date(base + 6 * 86400000).toISOString(), + new Date(base + 7 * 86400000).toISOString(), + new Date(base + 8 * 86400000).toISOString(), + new Date(base + 9 * 86400000).toISOString(), + ]; + expect(computeVelocity(dates).trend).toBe('accelerating'); + }); + + it('returns "decelerating" when second half has proportionally fewer commits', () => { + const base = new Date('2024-01-01T00:00:00Z').getTime(); + const dates = [ + new Date(base + 1 * 86400000).toISOString(), + new Date(base + 2 * 86400000).toISOString(), + new Date(base + 3 * 86400000).toISOString(), + new Date(base + 4 * 86400000).toISOString(), + new Date(base + 9 * 86400000).toISOString(), + ]; + expect(computeVelocity(dates).trend).toBe('decelerating'); + }); + + it('returns "stable" when commit rate is roughly even across both halves', () => { + const base = new Date('2024-01-01T00:00:00Z').getTime(); + const dates = [ + new Date(base + 1 * 86400000).toISOString(), + new Date(base + 2 * 86400000).toISOString(), + new Date(base + 8 * 86400000).toISOString(), + new Date(base + 9 * 86400000).toISOString(), + ]; + expect(computeVelocity(dates).trend).toBe('stable'); + }); + + it('returns positive commits_per_day when dates arrive newest-first (git log order)', () => { + // git log outputs commits newest-first; velocity must still be positive + const base = new Date('2024-01-01T00:00:00Z').getTime(); + const datesOldestFirst = [ + new Date(base + 1 * 86400000).toISOString(), + new Date(base + 3 * 86400000).toISOString(), + new Date(base + 5 * 86400000).toISOString(), + new Date(base + 7 * 86400000).toISOString(), + ]; + const datesNewestFirst = [...datesOldestFirst].reverse(); + const result = computeVelocity(datesNewestFirst); + expect(result.commits_per_day).toBeGreaterThan(0); + }); +}); diff --git a/__tests__/doraArchetype.test.js b/__tests__/doraArchetype.test.js new file mode 100644 index 0000000..e6de7ff --- /dev/null +++ b/__tests__/doraArchetype.test.js @@ -0,0 +1,50 @@ +'use strict'; + +const { classifyDoraArchetype } = require('../local-code-metrics'); + +describe('classifyDoraArchetype', () => { + it('returns "harmonious-high-achiever" for all-healthy metrics', () => { + expect(classifyDoraArchetype({ + large_commits_pct: '10.00', + sprawling_commits_pct: '5.00', + test_first_pct: '70.00', + message_quality_pct: '80.00' + })).toBe('harmonious-high-achiever'); + }); + + it('returns "legacy-bottleneck" for high sprawl combined with high large commits', () => { + expect(classifyDoraArchetype({ + large_commits_pct: '35.00', + sprawling_commits_pct: '30.00', + test_first_pct: '40.00', + message_quality_pct: '50.00' + })).toBe('legacy-bottleneck'); + }); + + it('returns "foundational-challenges" when large commit rate exceeds 40%', () => { + expect(classifyDoraArchetype({ + large_commits_pct: '45.00', + sprawling_commits_pct: '8.00', + test_first_pct: '55.00', + message_quality_pct: '65.00' + })).toBe('foundational-challenges'); + }); + + it('returns "foundational-challenges" for low test discipline with elevated large commits', () => { + expect(classifyDoraArchetype({ + large_commits_pct: '25.00', + sprawling_commits_pct: '8.00', + test_first_pct: '25.00', + message_quality_pct: '50.00' + })).toBe('foundational-challenges'); + }); + + it('returns "mixed-signals" when no archetype threshold is clearly breached', () => { + expect(classifyDoraArchetype({ + large_commits_pct: '25.00', + sprawling_commits_pct: '12.00', + test_first_pct: '40.00', + message_quality_pct: '55.00' + })).toBe('mixed-signals'); + }); +}); diff --git a/__tests__/exports.test.js b/__tests__/exports.test.js index 76a8a85..3418c2e 100644 --- a/__tests__/exports.test.js +++ b/__tests__/exports.test.js @@ -29,4 +29,17 @@ describe('module exports', () => { test('collectLocalMetrics is exported', () => { expect(typeof mod.collectLocalMetrics).toBe('function'); }); + + // D1/D2 additions + test('computeStatistics is exported', () => { expect(typeof mod.computeStatistics).toBe('function'); }); + test('computeVelocity is exported', () => { expect(typeof mod.computeVelocity).toBe('function'); }); + test('scoreMessageQuality is exported', () => { expect(typeof mod.scoreMessageQuality).toBe('function'); }); + test('classifyDoraArchetype is exported', () => { expect(typeof mod.classifyDoraArchetype).toBe('function'); }); + + // D3 additions + test('getAnthropicClient is exported', () => { expect(typeof mod.getAnthropicClient).toBe('function'); }); + test('selectClaudeCommits is exported', () => { expect(typeof mod.selectClaudeCommits).toBe('function'); }); + test('getCommitDiff is exported', () => { expect(typeof mod.getCommitDiff).toBe('function'); }); + test('analyzeWithClaude is exported', () => { expect(typeof mod.analyzeWithClaude).toBe('function'); }); + test('CLAUDE_SYSTEM_PROMPT is exported', () => { expect(typeof mod.CLAUDE_SYSTEM_PROMPT).toBe('string'); }); }); diff --git a/__tests__/messageQuality.test.js b/__tests__/messageQuality.test.js new file mode 100644 index 0000000..87f1d4b --- /dev/null +++ b/__tests__/messageQuality.test.js @@ -0,0 +1,36 @@ +'use strict'; + +const { scoreMessageQuality, CONFIG } = require('../local-code-metrics'); + +describe('scoreMessageQuality', () => { + it('returns false for empty string', () => { + expect(scoreMessageQuality('')).toBe(false); + }); + + it('returns true for conventional commit prefix', () => { + expect(scoreMessageQuality('feat: add login')).toBe(true); + expect(scoreMessageQuality('fix: resolve null pointer')).toBe(true); + expect(scoreMessageQuality('refactor(auth): extract token validation')).toBe(true); + }); + + it('returns true for long specific message without conventional prefix', () => { + const long = 'update the payment processing pipeline to handle declined cards gracefully'; + expect(long.split(/\s+/).length).toBeGreaterThanOrEqual(CONFIG.MESSAGE_QUALITY_MIN_WORDS); + expect(scoreMessageQuality(long)).toBe(true); + }); + + it('returns false for short vague message without conventional prefix', () => { + expect(scoreMessageQuality('fix issue')).toBe(false); + expect(scoreMessageQuality('wip')).toBe(false); + expect(scoreMessageQuality('update stuff')).toBe(false); + }); + + it('returns true for conventional prefix regardless of word count', () => { + expect(scoreMessageQuality('chore: x')).toBe(true); + }); + + it('is case-insensitive for conventional prefix', () => { + expect(scoreMessageQuality('FEAT: add thing')).toBe(true); + expect(scoreMessageQuality('Fix: resolve bug')).toBe(true); + }); +}); diff --git a/__tests__/statisticalDistribution.test.js b/__tests__/statisticalDistribution.test.js new file mode 100644 index 0000000..534e212 --- /dev/null +++ b/__tests__/statisticalDistribution.test.js @@ -0,0 +1,75 @@ +'use strict'; + +const { computeStatistics } = require('../local-code-metrics'); + +describe('computeStatistics', () => { + it('returns zero-value result for empty array', () => { + const result = computeStatistics([], []); + expect(result.p50).toBe(0); + expect(result.p90).toBe(0); + expect(result.p95).toBe(0); + expect(result.mean).toBe(0); + expect(result.stddev).toBe(0); + expect(result.trend).toBe('stable'); + }); + + it('returns correct values for single-element array', () => { + const result = computeStatistics([100], [Date.now()]); + expect(result.p50).toBe(100); + expect(result.p90).toBe(100); + expect(result.p95).toBe(100); + expect(result.mean).toBe(100); + expect(result.stddev).toBe(0); + expect(result.trend).toBe('stable'); + }); + + it('returns correct percentiles for known 10-element dataset', () => { + const sizes = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]; + const now = Date.now(); + const timestamps = sizes.map((_, i) => now + i * 1000); + const result = computeStatistics(sizes, timestamps); + expect(result.p50).toBe(55); + expect(result.mean).toBe(55); + expect(result.p90).toBeGreaterThan(result.p50); + expect(result.p95).toBeGreaterThan(result.p90); + expect(result.stddev).toBeGreaterThan(0); + }); + + it('returns "growing" when sizes increase over time', () => { + const sizes = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]; + const now = Date.now(); + const timestamps = sizes.map((_, i) => now + i * 86400000); + expect(computeStatistics(sizes, timestamps).trend).toBe('growing'); + }); + + it('returns "shrinking" when sizes decrease over time', () => { + const sizes = [100, 90, 80, 70, 60, 50, 40, 30, 20, 10]; + const now = Date.now(); + const timestamps = sizes.map((_, i) => now + i * 86400000); + expect(computeStatistics(sizes, timestamps).trend).toBe('shrinking'); + }); + + it('returns "stable" when sizes are flat', () => { + const sizes = [50, 50, 50, 50, 50, 50]; + const now = Date.now(); + const timestamps = sizes.map((_, i) => now + i * 86400000); + expect(computeStatistics(sizes, timestamps).trend).toBe('stable'); + }); + + it('marks an extreme value as outlier when it exceeds mean + 2*stddev', () => { + const sizes = [10, 10, 10, 10, 10, 10, 10, 10, 10, 1000]; + const now = Date.now(); + const timestamps = sizes.map((_, i) => now + i * 1000); + const result = computeStatistics(sizes, timestamps); + expect(result.isOutlier(1000)).toBe(true); + expect(result.isOutlier(10)).toBe(false); + }); + + it('no values are outliers when distribution is uniform', () => { + const sizes = [50, 50, 50, 50]; + const now = Date.now(); + const timestamps = sizes.map((_, i) => now + i * 1000); + const result = computeStatistics(sizes, timestamps); + expect(result.isOutlier(50)).toBe(false); + }); +}); diff --git a/ai_drift_metrics_coverage_map.html b/ai_drift_metrics_coverage_map.html new file mode 100644 index 0000000..291cd3b --- /dev/null +++ b/ai_drift_metrics_coverage_map.html @@ -0,0 +1,279 @@ + + + +
+ +
+
Fully covered by this toolkit
+
Partial / proxy only
+
Gap — needs external tool
+
Out of scope (organizational)
+
+ +
DORA AI Capabilities (7 total)
+
+
+ +
DORA Delivery Metrics
+
+
+ +
Git-Based Signals — fully measured
+
+
+ +
+ +
Analysis layers
+
+ +
Commercial tools for gaps
+
+ +
+ + diff --git a/lib/claude.js b/lib/claude.js new file mode 100644 index 0000000..316fb2b --- /dev/null +++ b/lib/claude.js @@ -0,0 +1,101 @@ +// @ts-nocheck +'use strict'; + +const { CONFIG } = require('./config'); +const { getCommitDiff } = require('./git'); + +const CLAUDE_SYSTEM_PROMPT = `You are a code quality analyst specializing in detecting AI-generated code patterns and architectural concerns. Analyze the provided git commit diff and return a JSON assessment. + +Detect these AI-generated code patterns: +- Generic variable names (data, result, item, temp, obj, val, arr, str) without domain context +- Boilerplate CRUD operations without error handling or domain-specific validation +- Identically or near-identically structured adjacent functions differing only in variable names +- Absent domain language — uses generic technical terms instead of business/domain vocabulary +- Import patterns inconsistent with the rest of the file +- Missing edge case handling (no null checks, no boundary conditions, no error paths) + +Detect these architectural concerns: +- Code crossing service/module/layer boundaries based on import paths +- New dependencies on modules not previously used in this area of the codebase +- Structural patterns inconsistent with the surrounding code's style + +Respond ONLY with valid JSON in this exact schema, no other text: +{ + "ai_confidence": , + "risk_score": , + "patterns": ["string", ...], + "architectural_concerns": ["string", ...], + "summary": "" +} + +ai_confidence: likelihood this code was AI-generated without careful human review (0=clearly human-authored, 100=clearly AI-generated) +risk_score: overall code quality risk for this commit considering size, patterns, and architectural issues`; + +/** + * Returns an Anthropic client if ANTHROPIC_API_KEY is set and the SDK is available. + * Returns null otherwise — callers must check before using. + * @returns {Promise} + */ +async function getAnthropicClient() { + if (!process.env.ANTHROPIC_API_KEY) return null; + try { + // @ts-ignore — optional peer dependency; not installed when API key absent + const { Anthropic } = require('@anthropic-ai/sdk'); + return new Anthropic(); + } catch { + console.warn('⚠️ Claude analysis unavailable: @anthropic-ai/sdk not installed or requires Node 18+'); + return null; + } +} + +/** + * Pre-filter metrics to commits worth sending to Claude. + * Selects large commits with high additions ratio, sorted by total churn descending, + * capped at AI_ANALYSIS_MAX_COMMITS. + * @param {Array} metrics + * @returns {Array} + */ +function selectClaudeCommits(metrics) { + return metrics + .filter(m => m.large_commit && m.total_additions > m.total_deletions * CONFIG.AI_RISK_ADDITIONS_RATIO) + .sort((a, b) => (b.total_additions + b.total_deletions) - (a.total_additions + a.total_deletions)) + .slice(0, CONFIG.AI_ANALYSIS_MAX_COMMITS); +} + +/** + * Analyze a list of commits with the Claude API, returning structured results. + * Calls are sequential to avoid rate limits. Errors per-commit are captured, not thrown. + * @param {object} client - Anthropic client instance + * @param {Array} commits + * @returns {Promise>} + */ +async function analyzeWithClaude(client, commits) { + const results = []; + + for (const commit of commits) { + const diff = getCommitDiff(commit.full_sha); + try { + const response = await client.messages.create({ + model: 'claude-sonnet-4-6', + max_tokens: 1024, + system: CLAUDE_SYSTEM_PROMPT, + messages: [{ + role: 'user', + content: `Commit: ${commit.sha}\nMessage: ${commit.message}\nAuthor: ${commit.author}\nDate: ${commit.date}\nBranch: ${commit.source_branch}\n\n${diff}` + }] + }); + + const raw = response.content[0].type === 'text' ? response.content[0].text : ''; + const json = raw.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim(); + const parsed = JSON.parse(json); + results.push({ sha: commit.sha, ...parsed }); + } catch (err) { + console.warn(` ⚠️ Claude analysis failed for ${commit.sha}: ${err.message}`); + results.push({ sha: commit.sha, error: err.message }); + } + } + + return results; +} + +module.exports = { CLAUDE_SYSTEM_PROMPT, getAnthropicClient, selectClaudeCommits, analyzeWithClaude }; diff --git a/lib/config.js b/lib/config.js new file mode 100644 index 0000000..c4d5973 --- /dev/null +++ b/lib/config.js @@ -0,0 +1,28 @@ +// @ts-nocheck +'use strict'; + +// Configuration — adjust these for your project +const CONFIG = { + ANALYSIS_DAYS: 30, + MAX_COMMITS: 50, + LARGE_COMMIT_THRESHOLD: 100, + SPRAWLING_COMMIT_THRESHOLD: 5, + MESSAGE_QUALITY_MIN_WORDS: 10, + AI_ANALYSIS_MAX_COMMITS: 5, + AI_DIFF_MAX_CHARS: 4000, + AI_RISK_ADDITIONS_RATIO: 3, + + // Test file patterns — customize for your language/framework + TEST_FILE_PATTERNS: [ + /\.(test|spec)\./i, // file.test.js, file.spec.ts + /Tests?\.cs$/i, // FileTests.cs, FileTest.cs (C#) + /Test\.java$/i, // FileTest.java (Java) + /_test\.py$/i, // file_test.py (Python) + /test_.*\.py$/i, // test_file.py (Python) + /_test\.go$/i, // file_test.go (Go) + /__tests__/i, // __tests__ directory + /\/tests?\//i // /test/ or /tests/ directories + ] +}; + +module.exports = { CONFIG }; diff --git a/lib/git.js b/lib/git.js new file mode 100644 index 0000000..f3c59f7 --- /dev/null +++ b/lib/git.js @@ -0,0 +1,146 @@ +// @ts-nocheck +'use strict'; + +const { execSync } = require('child_process'); +const { CONFIG } = require('./config'); + +/** + * Execute Git command with error handling + * @param {string} command + * @returns {string} + */ +function runGitCommand(command) { + try { + const result = execSync(command, { encoding: 'utf8' }).trim(); + return result; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error(`Error running Git command: ${command}`); + console.error(`Error: ${msg}`); + return ''; + } +} + +/** + * Parse Git log output into structured commit data + * @param {string} logOutput + * @returns {Array<{sha: string, full_sha: string, date: string, author: string, message: string, source_branch?: string}>} + */ +function parseGitLog(logOutput) { + if (!logOutput) return []; + + const commits = []; + const lines = logOutput.split('\n').filter(line => line.trim()); + + for (const line of lines) { + const parts = line.split('|'); + if (parts.length < 4) continue; + + const [sha, date, author, ...messageParts] = parts; + if (sha && sha.length === 40) { + commits.push({ + sha: sha.substring(0, 8), + full_sha: sha, + date, + author, + message: messageParts.join('|') + }); + } + } + return commits; +} + +/** + * Check if a filename matches test file patterns + * @param {string} filename + * @returns {boolean} + */ +function isTestFile(filename) { + return CONFIG.TEST_FILE_PATTERNS.some(pattern => pattern.test(filename)); +} + +/** + * Analyze a single commit for AI drift indicators + * @param {string} sha + * @param {string} branch + * @returns {object|null} + */ +function analyzeCommit(sha, branch) { + try { + const statsOutput = runGitCommand(`git show --numstat --format="" ${sha}`); + if (!statsOutput) { + console.warn(` Warning: No stats found for commit ${sha}`); + return null; + } + + const statsLines = statsOutput.split('\n').filter(line => line.trim()); + + let totalAdditions = 0; + let totalDeletions = 0; + let filesChanged = 0; + let testFiles = 0; + let prodFiles = 0; + let binaryFiles = 0; + + for (const line of statsLines) { + const [additions, deletions, filename] = line.split('\t'); + if (!filename) continue; + + filesChanged++; + + // Handle binary files (marked with '-' in git numstat) + if (additions === '-' && deletions === '-') { + binaryFiles++; + continue; + } + + const addNum = parseInt(additions) || 0; + const delNum = parseInt(deletions) || 0; + + totalAdditions += addNum; + totalDeletions += delNum; + + if (isTestFile(filename)) { + testFiles++; + } else { + prodFiles++; + } + } + + const totalLines = totalAdditions + totalDeletions; + + return { + total_additions: totalAdditions, + total_deletions: totalDeletions, + files_changed: filesChanged, + binary_files: binaryFiles, + test_files_count: testFiles, + prod_files_count: prodFiles, + test_first_indicator: testFiles > 0 && prodFiles > 0, + large_commit: totalLines > CONFIG.LARGE_COMMIT_THRESHOLD, + sprawling_commit: filesChanged > CONFIG.SPRAWLING_COMMIT_THRESHOLD, + outlier: false, + source_branch: branch, + change_ratio: totalDeletions > 0 ? (totalAdditions / totalDeletions).toFixed(2) : 'inf' + }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error(` Error analyzing commit ${sha}: ${msg}`); + return null; + } +} + +/** + * Fetch the stat summary and full diff for a single commit SHA. + * Returns a combined string truncated to AI_DIFF_MAX_CHARS. + * @param {string} sha + * @returns {string} + */ +function getCommitDiff(sha) { + const stat = runGitCommand(`git show --stat --format="" ${sha}`); + const diff = runGitCommand(`git show --format="" ${sha}`); + const combined = `--- File Summary ---\n${stat}\n\n--- Diff ---\n${diff}`; + return combined.substring(0, CONFIG.AI_DIFF_MAX_CHARS); +} + +module.exports = { runGitCommand, parseGitLog, isTestFile, analyzeCommit, getCommitDiff }; diff --git a/lib/metrics.js b/lib/metrics.js new file mode 100644 index 0000000..1b1b3c6 --- /dev/null +++ b/lib/metrics.js @@ -0,0 +1,98 @@ +// @ts-nocheck +'use strict'; + +const { CONFIG } = require('./config'); + +/** @type {RegExp} */ +const CONVENTIONAL_COMMIT_RE = /^(feat|fix|refactor|test|chore|docs|perf|ci|build|revert)(\(.+\))?:/i; + +/** + * Score a single commit message for quality. + * Quality = conventional commit format OR word count >= MESSAGE_QUALITY_MIN_WORDS. + * @param {string} message + * @returns {boolean} + */ +function scoreMessageQuality(message) { + if (!message) return false; + if (CONVENTIONAL_COMMIT_RE.test(message)) return true; + return message.split(/\s+/).filter(Boolean).length >= CONFIG.MESSAGE_QUALITY_MIN_WORDS; +} + +/** + * Classify a repo into a DORA team archetype based on summary metrics. + * Evaluated in priority order: harmonious-high-achiever → legacy-bottleneck → foundational-challenges → mixed-signals + * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_first_pct: string, message_quality_pct: string }} summary + * @returns {string} + */ +function classifyDoraArchetype(summary) { + const large = parseFloat(summary.large_commits_pct); + const sprawling = parseFloat(summary.sprawling_commits_pct); + const testFirst = parseFloat(summary.test_first_pct); + const msgQuality = parseFloat(summary.message_quality_pct); + + if (large < 20 && sprawling < 10 && testFirst > 50 && msgQuality > 60) return 'harmonious-high-achiever'; + if (sprawling > 25 && large > 30) return 'legacy-bottleneck'; + if (large > 40 || (testFirst < 30 && large > 20)) return 'foundational-challenges'; + return 'mixed-signals'; +} + +/** + * Generate insights based on metrics + * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_first_pct: string, avg_lines_changed: string }} summary + * @param {Array} metrics + * @returns {{ insights: string[], warnings: string[], recommendations: string[] }} + */ +function generateInsights(summary, metrics) { + const insights = []; + const warnings = []; + const recommendations = []; + + const largePct = parseFloat(summary.large_commits_pct); + const sprawlingPct = parseFloat(summary.sprawling_commits_pct); + const testFirstPct = parseFloat(summary.test_first_pct); + const avgLines = parseFloat(summary.avg_lines_changed); + + if (largePct > 40) { + warnings.push(`🚨 Very high large commit rate (${largePct}%) - Strong AI drift indicators`); + recommendations.push('Consider breaking AI-generated code into smaller, focused commits'); + } else if (largePct > 20) { + warnings.push(`⚠️ High large commit rate (${largePct}%) - Monitor AI tool usage patterns`); + } else { + insights.push(`✅ Healthy large commit rate (${largePct}%)`); + } + + if (sprawlingPct > 25) { + warnings.push(`🚨 Very high sprawling commit rate (${sprawlingPct}%) - Possible shotgun surgery`); + recommendations.push('Review if AI suggestions are causing scattered changes across unrelated files'); + } else if (sprawlingPct > 10) { + warnings.push(`⚠️ High sprawling commit rate (${sprawlingPct}%) - Watch for scope creep`); + } else { + insights.push(`✅ Good sprawling commit control (${sprawlingPct}%)`); + } + + if (testFirstPct > 50) { + insights.push(`✅ Strong test-first discipline (${testFirstPct}%)`); + } else if (testFirstPct < 30) { + warnings.push(`⚠️ Low test-first discipline (${testFirstPct}%) - AI tools may be bypassing TDD`); + recommendations.push('Ensure test coverage when accepting AI-generated code'); + } + + if (avgLines > 1000) { + warnings.push(`🚨 Very high average lines per commit (${avgLines}) - Extreme batch coding`); + recommendations.push('Implement strict commit size limits when using AI tools'); + } else if (avgLines > 500) { + warnings.push(`⚠️ High average lines per commit (${avgLines}) - Monitor AI batch acceptance`); + } + + const possibleAICommits = metrics.filter(m => + m.large_commit && m.total_additions > m.total_deletions * 3 + ).length; + + if (possibleAICommits > metrics.length * 0.3) { + warnings.push(`🤖 High proportion of addition-heavy large commits (${possibleAICommits}/${metrics.length}) - Possible AI batch acceptance`); + } + + return { insights, warnings, recommendations }; +} + +module.exports = { scoreMessageQuality, classifyDoraArchetype, generateInsights }; diff --git a/lib/statistics.js b/lib/statistics.js new file mode 100644 index 0000000..a7a9d67 --- /dev/null +++ b/lib/statistics.js @@ -0,0 +1,89 @@ +// @ts-nocheck +'use strict'; + +/** + * Compute commit velocity and trend from an array of ISO 8601 date strings. + * Input order does not matter — dates are sorted internally. + * @param {string[]} dates + * @returns {{ commits_per_day: number, trend: string }} + */ +function computeVelocity(dates) { + if (dates.length < 2) return { commits_per_day: dates.length, trend: 'stable' }; + + const ms = dates.map(d => new Date(d).getTime()).sort((a, b) => a - b); + const spanDays = (ms[ms.length - 1] - ms[0]) / 86400000 || 1; + const commits_per_day = dates.length / spanDays; + + // Split commits at time midpoint; compare first-half vs second-half rate + const midMs = (ms[0] + ms[ms.length - 1]) / 2; + const firstHalf = ms.filter(t => t <= midMs); + const secondHalf = ms.filter(t => t > midMs); + const halfSpan = spanDays / 2 || 1; + const firstRate = firstHalf.length / halfSpan; + const secondRate = secondHalf.length / halfSpan; + + let trend = 'stable'; + if (secondRate > firstRate * 1.25) trend = 'accelerating'; + else if (secondRate < firstRate * 0.75) trend = 'decelerating'; + + return { commits_per_day, trend }; +} + +/** + * Compute statistical distribution of a numeric array. + * @param {number[]} sizes + * @param {number[]} timestamps - epoch ms values, same length as sizes, time-ordered oldest first + * @returns {{ p50: number, p90: number, p95: number, mean: number, stddev: number, trend: string, isOutlier: (v: number) => boolean }} + */ +function computeStatistics(sizes, timestamps) { + if (sizes.length === 0) { + return { p50: 0, p90: 0, p95: 0, mean: 0, stddev: 0, trend: 'stable', isOutlier: () => false }; + } + + // Percentile (linear interpolation) + const sorted = [...sizes].sort((a, b) => a - b); + /** @param {number} p - 0..1 */ + function quantile(p) { + if (sorted.length === 1) return sorted[0]; + const idx = p * (sorted.length - 1); + const lo = Math.floor(idx); + const hi = Math.ceil(idx); + return sorted[lo] + (sorted[hi] - sorted[lo]) * (idx - lo); + } + + const mean = sizes.reduce((s, v) => s + v, 0) / sizes.length; + const variance = sizes.reduce((s, v) => s + (v - mean) ** 2, 0) / sizes.length; + const stddev = Math.sqrt(variance); + + // Trend: linear regression slope of size over time index + const n = sizes.length; + let trend = 'stable'; + if (n >= 2) { + // Normalize timestamps to [0..1] to avoid floating-point magnitude issues + const t0 = timestamps[0]; + const tRange = (timestamps[n - 1] - t0) || 1; + const xs = timestamps.map(t => (t - t0) / tRange); + const xMean = xs.reduce((s, v) => s + v, 0) / n; + const yMean = mean; + const num = xs.reduce((s, x, i) => s + (x - xMean) * (sizes[i] - yMean), 0); + const den = xs.reduce((s, x) => s + (x - xMean) ** 2, 0); + const slope = den === 0 ? 0 : num / den; + // Threshold: slope relative to mean — ignore noise below 5% of mean per unit + const threshold = yMean * 0.05; + if (slope > threshold) trend = 'growing'; + else if (slope < -threshold) trend = 'shrinking'; + } + + const cutoff = mean + 2 * stddev; + return { + p50: quantile(0.5), + p90: quantile(0.9), + p95: quantile(0.95), + mean, + stddev, + trend, + isOutlier: (v) => stddev > 0 && v > cutoff + }; +} + +module.exports = { computeVelocity, computeStatistics }; diff --git a/local-code-metrics.js b/local-code-metrics.js index 7caa79a..9c8d0bd 100644 --- a/local-code-metrics.js +++ b/local-code-metrics.js @@ -3,237 +3,33 @@ /** * AI Code Drift Local Analysis Script - * + * * Analyzes local Git repository for AI code drift patterns by examining * feature branches before they're merged and squashed. - * + * * Based on research by Ken Judy - https://github.com/stride-nyc/code-quality-metrics * Licensed under CC BY 4.0 - * - * Usage: node local-ai-drift-analysis.js [options] + * + * Usage: node local-code-metrics.js [options] */ -const { execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); -/** - * @typedef {{ sha: string, full_sha: string, date: string, author: string, message: string, source_branch?: string }} CommitInfo - * @typedef {{ total_additions: number, total_deletions: number, files_changed: number, binary_files: number, test_files_count: number, prod_files_count: number, test_first_indicator: boolean, large_commit: boolean, sprawling_commit: boolean, source_branch: string, change_ratio: string }} CommitStats - * @typedef {CommitInfo & CommitStats & { commit_type: string }} CommitMetric - */ - -// Configuration - Adjust these for your project -const CONFIG = { - ANALYSIS_DAYS: 30, - MAX_COMMITS: 50, - LARGE_COMMIT_THRESHOLD: 100, - SPRAWLING_COMMIT_THRESHOLD: 5, - - // Test file patterns - customize for your language/framework - TEST_FILE_PATTERNS: [ - /\.(test|spec)\./i, // file.test.js, file.spec.ts - /Tests?\.cs$/i, // FileTests.cs, FileTest.cs (C#) - /Test\.java$/i, // FileTest.java (Java) - /_test\.py$/i, // file_test.py (Python) - /test_.*\.py$/i, // test_file.py (Python) - /_test\.go$/i, // file_test.go (Go) - /__tests__/i, // __tests__ directory - /\/tests?\//i // /test/ or /tests/ directories - ] -}; - -/** - * Execute Git command with error handling - * @param {string} command - * @returns {string} - */ -function runGitCommand(command) { - try { - const result = execSync(command, { encoding: 'utf8' }).trim(); - return result; - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(`Error running Git command: ${command}`); - console.error(`Error: ${msg}`); - return ''; - } -} - -/** - * Parse Git log output into structured commit data - * @param {string} logOutput - * @returns {CommitInfo[]} - */ -function parseGitLog(logOutput) { - if (!logOutput) return []; - - /** @type {CommitInfo[]} */ - const commits = []; - const lines = logOutput.split('\n').filter(/** @param {string} line */ line => line.trim()); - - for (const line of lines) { - const parts = line.split('|'); - if (parts.length < 4) continue; - - const [sha, date, author, ...messageParts] = parts; - if (sha && sha.length === 40) { - commits.push({ - sha: sha.substring(0, 8), - full_sha: sha, - date, - author, - message: messageParts.join('|') - }); - } - } - return commits; -} +// Load .env file if present — allows ANTHROPIC_API_KEY to be set without exporting to the shell +require('dotenv').config({ quiet: true }); -/** - * Check if a filename matches test file patterns - * @param {string} filename - * @returns {boolean} - */ -function isTestFile(filename) { - return CONFIG.TEST_FILE_PATTERNS.some(pattern => pattern.test(filename)); -} - -/** - * Analyze a single commit for AI drift indicators - * @param {string} sha - * @param {string} branch - * @returns {CommitStats|null} - */ -function analyzeCommit(sha, branch) { - try { - // Get detailed commit statistics - const statsOutput = runGitCommand(`git show --numstat --format="" ${sha}`); - if (!statsOutput) { - console.warn(` Warning: No stats found for commit ${sha}`); - return null; - } - - const statsLines = statsOutput.split('\n').filter(line => line.trim()); - - let totalAdditions = 0; - let totalDeletions = 0; - let filesChanged = 0; - let testFiles = 0; - let prodFiles = 0; - let binaryFiles = 0; - - for (const line of statsLines) { - const [additions, deletions, filename] = line.split('\t'); - if (!filename) continue; - - filesChanged++; - - // Handle binary files (marked with '-' in git numstat) - if (additions === '-' && deletions === '-') { - binaryFiles++; - continue; - } - - const addNum = parseInt(additions) || 0; - const delNum = parseInt(deletions) || 0; - - totalAdditions += addNum; - totalDeletions += delNum; - - if (isTestFile(filename)) { - testFiles++; - } else { - prodFiles++; - } - } - - const totalLines = totalAdditions + totalDeletions; - - return { - total_additions: totalAdditions, - total_deletions: totalDeletions, - files_changed: filesChanged, - binary_files: binaryFiles, - test_files_count: testFiles, - prod_files_count: prodFiles, - test_first_indicator: testFiles > 0 && prodFiles > 0, - large_commit: totalLines > CONFIG.LARGE_COMMIT_THRESHOLD, - sprawling_commit: filesChanged > CONFIG.SPRAWLING_COMMIT_THRESHOLD, - source_branch: branch, - change_ratio: totalDeletions > 0 ? (totalAdditions / totalDeletions).toFixed(2) : 'inf' - }; - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error(` Error analyzing commit ${sha}: ${msg}`); - return null; - } -} +const { CONFIG } = require('./lib/config'); +const { runGitCommand, parseGitLog, isTestFile, analyzeCommit, getCommitDiff } = require('./lib/git'); +const { computeStatistics, computeVelocity } = require('./lib/statistics'); +const { scoreMessageQuality, classifyDoraArchetype, generateInsights } = require('./lib/metrics'); +const { CLAUDE_SYSTEM_PROMPT, getAnthropicClient, selectClaudeCommits, analyzeWithClaude } = require('./lib/claude'); /** - * Generate insights based on metrics - * @param {{ large_commits_pct: string, sprawling_commits_pct: string, test_first_pct: string, avg_lines_changed: string }} summary - * @param {CommitMetric[]} metrics - * @returns {{ insights: string[], warnings: string[], recommendations: string[] }} + * @typedef {{ sha: string, full_sha: string, date: string, author: string, message: string, source_branch?: string }} CommitInfo + * @typedef {{ total_additions: number, total_deletions: number, files_changed: number, binary_files: number, test_files_count: number, prod_files_count: number, test_first_indicator: boolean, large_commit: boolean, sprawling_commit: boolean, outlier: boolean, source_branch: string, change_ratio: string, ai_confidence?: number, risk_score?: number, patterns?: string[], architectural_concerns?: string[], claude_summary?: string }} CommitStats + * @typedef {CommitInfo & CommitStats & { commit_type: string }} CommitMetric */ -function generateInsights(summary, metrics) { - const insights = []; - const warnings = []; - const recommendations = []; - - // Analyze patterns - const largePct = parseFloat(summary.large_commits_pct); - const sprawlingPct = parseFloat(summary.sprawling_commits_pct); - const testFirstPct = parseFloat(summary.test_first_pct); - const avgLines = parseFloat(summary.avg_lines_changed); - - // Large commit analysis - if (largePct > 40) { - warnings.push(`🚨 Very high large commit rate (${largePct}%) - Strong AI drift indicators`); - recommendations.push('Consider breaking AI-generated code into smaller, focused commits'); - } else if (largePct > 20) { - warnings.push(`⚠️ High large commit rate (${largePct}%) - Monitor AI tool usage patterns`); - } else { - insights.push(`✅ Healthy large commit rate (${largePct}%)`); - } - - // Sprawling commit analysis - if (sprawlingPct > 25) { - warnings.push(`🚨 Very high sprawling commit rate (${sprawlingPct}%) - Possible shotgun surgery`); - recommendations.push('Review if AI suggestions are causing scattered changes across unrelated files'); - } else if (sprawlingPct > 10) { - warnings.push(`⚠️ High sprawling commit rate (${sprawlingPct}%) - Watch for scope creep`); - } else { - insights.push(`✅ Good sprawling commit control (${sprawlingPct}%)`); - } - - // Test discipline analysis - if (testFirstPct > 50) { - insights.push(`✅ Strong test-first discipline (${testFirstPct}%)`); - } else if (testFirstPct < 30) { - warnings.push(`⚠️ Low test-first discipline (${testFirstPct}%) - AI tools may be bypassing TDD`); - recommendations.push('Ensure test coverage when accepting AI-generated code'); - } - - // Average lines analysis - if (avgLines > 1000) { - warnings.push(`🚨 Very high average lines per commit (${avgLines}) - Extreme batch coding`); - recommendations.push('Implement strict commit size limits when using AI tools'); - } else if (avgLines > 500) { - warnings.push(`⚠️ High average lines per commit (${avgLines}) - Monitor AI batch acceptance`); - } - - // AI pattern detection - const possibleAICommits = metrics.filter(m => - m.large_commit && m.total_additions > m.total_deletions * 3 - ).length; - - if (possibleAICommits > metrics.length * 0.3) { - warnings.push(`🤖 High proportion of addition-heavy large commits (${possibleAICommits}/${metrics.length}) - Possible AI batch acceptance`); - } - - return { insights, warnings, recommendations }; -} /** * Main analysis function @@ -241,92 +37,92 @@ function generateInsights(summary, metrics) { async function collectLocalMetrics() { console.log('=== AI Code Drift Local Analysis ==='); console.log(''); - + // Verify we're in a Git repository const repoRoot = runGitCommand('git rev-parse --show-toplevel'); if (!repoRoot) { console.error('❌ Not in a Git repository or Git not available'); process.exit(1); } - + const remoteUrl = runGitCommand('git remote get-url origin') || 'No remote configured'; - + console.log(`📁 Repository: ${remoteUrl}`); console.log(`📍 Local path: ${repoRoot}`); console.log(`📅 Analysis period: Last ${CONFIG.ANALYSIS_DAYS} days`); console.log(''); - + // Get all local branches except main/master const branchesOutput = runGitCommand('git branch'); if (!branchesOutput) { console.error('❌ Unable to list Git branches'); process.exit(1); } - + const allBranches = branchesOutput.split('\n') .map(line => line.replace(/^\*?\s*/, '').trim()) .filter(branch => branch && !['main', 'master'].includes(branch.toLowerCase())); - + if (allBranches.length === 0) { console.log('⚠️ No feature branches found. Analysis works best with feature branch workflows.'); console.log('Consider preserving feature branches after merging for better AI drift detection.'); return; } - + console.log(`🌿 Found ${allBranches.length} feature branches:`); allBranches.forEach(branch => console.log(` • ${branch}`)); console.log(''); - + // Calculate date range const since = new Date(); since.setDate(since.getDate() - CONFIG.ANALYSIS_DAYS); const sinceStr = since.toISOString().split('T')[0]; - + console.log(`🔍 Looking for commits since: ${sinceStr}`); console.log(''); - + // Collect commits from all feature branches /** @type {CommitInfo[]} */ const allCommits = []; /** @type {Record} */ const branchCommitCounts = {}; - + for (const branch of allBranches) { process.stdout.write(`📊 Analyzing branch: ${branch}... `); - + try { const logOutput = runGitCommand( `git log --since="${sinceStr}" --pretty=format:"%H|%ai|%an|%s" ${branch}` ); - + const branchCommits = parseGitLog(logOutput); branchCommitCounts[branch] = branchCommits.length; - + // Add branch info to each commit branchCommits.forEach(commit => { commit.source_branch = branch; allCommits.push(commit); }); - + console.log(`${branchCommits.length} commits`); - + } catch (error) { const msg = error instanceof Error ? error.message : String(error); console.log(`❌ Error: ${msg}`); branchCommitCounts[branch] = 0; } } - + // Remove duplicate commits (same SHA) const uniqueCommits = allCommits.filter((commit, index, self) => index === self.findIndex(c => c.sha === commit.sha) ); - + console.log(''); console.log(`📈 Found ${allCommits.length} total commits, ${uniqueCommits.length} unique`); console.log('📊 Commits per branch:', branchCommitCounts); console.log(''); - + if (uniqueCommits.length === 0) { console.log('⚠️ No commits found in feature branches in the last 30 days.'); console.log('This could mean:'); @@ -335,35 +131,95 @@ async function collectLocalMetrics() { console.log(' • No development activity in the analysis period'); return; } - + // Analyze commits in detail const commitsToAnalyze = uniqueCommits.slice(0, CONFIG.MAX_COMMITS); console.log(`🔬 Analyzing ${commitsToAnalyze.length} commits in detail...`); console.log(''); - + + /** @type {CommitMetric[]} */ const metrics = []; const progressInterval = Math.max(1, Math.floor(commitsToAnalyze.length / 10)); - + for (let i = 0; i < commitsToAnalyze.length; i++) { const commit = commitsToAnalyze[i]; - + if (i % progressInterval === 0 || i === commitsToAnalyze.length - 1) { const progress = Math.round((i + 1) / commitsToAnalyze.length * 100); process.stdout.write(`\r⏳ Processing commits... ${progress}%`); } - + const analysis = analyzeCommit(commit.full_sha, commit.source_branch ?? ''); if (analysis) { - metrics.push({ + metrics.push(/** @type {CommitMetric} */ ({ ...commit, ...analysis, commit_type: 'feature_branch' - }); + })); } } - + console.log('\n'); - + + // Statistical distributions + const lineSizes = metrics.map(m => m.total_additions + m.total_deletions); + const fileCounts = metrics.map(m => m.files_changed); + const timestamps = metrics.map(m => new Date(m.date).getTime()); + const lineStats = computeStatistics(lineSizes, timestamps); + const fileStats = computeStatistics(fileCounts, timestamps); + + // Mark outlier commits in-place + metrics.forEach(m => { + m.outlier = lineStats.isOutlier(m.total_additions + m.total_deletions); + }); + + // Velocity + const dates = metrics.map(m => m.date); + const velocity = computeVelocity(dates); + + // Additions ratio distribution + const ratios = metrics.map(m => m.total_additions / (m.total_deletions || 1)); + const ratioStats = computeStatistics(ratios, timestamps); + + // Message quality + const qualityCount = metrics.filter(m => scoreMessageQuality(m.message)).length; + const message_quality_pct = metrics.length > 0 + ? ((qualityCount / metrics.length) * 100).toFixed(2) + : '0.00'; + + // Claude API analysis (optional — runs only when ANTHROPIC_API_KEY is set) + const anthropicClient = await getAnthropicClient(); + /** @type {any[]} */ + let claudeResults = []; + if (anthropicClient) { + const claudeTargets = selectClaudeCommits(metrics); + if (claudeTargets.length > 0) { + console.log(`🤖 Running Claude analysis on ${claudeTargets.length} high-risk commits...`); + claudeResults = await analyzeWithClaude(anthropicClient, claudeTargets); + for (const result of claudeResults) { + const metric = metrics.find(m => m.sha === result.sha); + if (metric && !result.error) { + Object.assign(metric, { + ai_confidence: result.ai_confidence, + risk_score: result.risk_score, + patterns: result.patterns, + architectural_concerns: result.architectural_concerns, + claude_summary: result.summary + }); + } + } + } else { + console.log('ℹ️ No commits met Claude analysis threshold'); + } + } else { + console.log('ℹ️ Claude analysis skipped (no ANTHROPIC_API_KEY set)'); + } + + // Pre-compute pct fields once — reused in both summary object and classifyDoraArchetype call + const large_commits_pct = metrics.length > 0 ? ((metrics.filter(m => m.large_commit).length / metrics.length) * 100).toFixed(2) : '0.00'; + const sprawling_commits_pct = metrics.length > 0 ? ((metrics.filter(m => m.sprawling_commit).length / metrics.length) * 100).toFixed(2) : '0.00'; + const test_first_pct = metrics.length > 0 ? ((metrics.filter(m => m.test_first_indicator).length / metrics.length) * 100).toFixed(2) : '0.00'; + // Generate summary statistics const summary = { analysis_date: new Date().toISOString(), @@ -372,26 +228,52 @@ async function collectLocalMetrics() { filtered_from: uniqueCommits.length, branches_analyzed: allBranches, branch_commit_counts: branchCommitCounts, - large_commits_pct: metrics.length > 0 ? ((metrics.filter(m => m.large_commit).length / metrics.length) * 100).toFixed(2) : "0.00", - sprawling_commits_pct: metrics.length > 0 ? ((metrics.filter(m => m.sprawling_commit).length / metrics.length) * 100).toFixed(2) : "0.00", - test_first_pct: metrics.length > 0 ? ((metrics.filter(m => m.test_first_indicator).length / metrics.length) * 100).toFixed(2) : "0.00", + large_commits_pct, + sprawling_commits_pct, + test_first_pct, avg_files_changed: metrics.length > 0 ? (metrics.reduce((sum, m) => sum + m.files_changed, 0) / metrics.length).toFixed(2) : "0.00", avg_lines_changed: metrics.length > 0 ? (metrics.reduce((sum, m) => sum + m.total_additions + m.total_deletions, 0) / metrics.length).toFixed(2) : "0.00", + p50_lines_changed: lineStats.p50, + p90_lines_changed: lineStats.p90, + p95_lines_changed: lineStats.p95, + stddev_lines_changed: lineStats.stddev, + p50_files_changed: fileStats.p50, + p90_files_changed: fileStats.p90, + commit_size_trend: lineStats.trend, + velocity_commits_per_day: velocity.commits_per_day, + velocity_trend: velocity.trend, + additions_ratio_median: ratioStats.p50, + additions_ratio_p90: ratioStats.p90, + message_quality_pct, + dora_archetype: classifyDoraArchetype({ large_commits_pct, sprawling_commits_pct, test_first_pct, message_quality_pct }), config: CONFIG, note: "Local feature branches analysis - shows actual development patterns before merge squashing" }; - + // Generate insights const { insights, warnings, recommendations } = generateInsights(summary, metrics); - + // Save results const outputDir = process.cwd(); const metricsFile = path.join(outputDir, 'local_commit_metrics.json'); const summaryFile = path.join(outputDir, 'local_metrics_summary.json'); - + fs.writeFileSync(metricsFile, JSON.stringify(metrics, null, 2)); fs.writeFileSync(summaryFile, JSON.stringify(summary, null, 2)); - + + if (claudeResults.length > 0) { + const claudeOutput = { + analyzed_at: new Date().toISOString(), + model: 'claude-sonnet-4-6', + commits_analyzed: claudeResults.filter(r => !r.error).length, + results: claudeResults + }; + fs.writeFileSync( + path.join(outputDir, 'local_claude_analysis.json'), + JSON.stringify(claudeOutput, null, 2) + ); + } + // Display results console.log('=== 📊 ANALYSIS RESULTS ==='); console.log(''); @@ -402,49 +284,60 @@ async function collectLocalMetrics() { console.log(`📂 Average files changed: ${summary.avg_files_changed}`); console.log(`📝 Average lines changed: ${summary.avg_lines_changed}`); console.log(''); - - // Display insights + if (insights.length > 0) { console.log('=== ✅ POSITIVE FINDINGS ==='); insights.forEach(insight => console.log(insight)); console.log(''); } - + if (warnings.length > 0) { console.log('=== ⚠️ CONCERNS DETECTED ==='); warnings.forEach(warning => console.log(warning)); console.log(''); } - + if (recommendations.length > 0) { console.log('=== 💡 RECOMMENDATIONS ==='); recommendations.forEach(rec => console.log(`• ${rec}`)); console.log(''); } - + + const claudeAnnotated = metrics.filter(m => m.ai_confidence !== undefined); + if (claudeAnnotated.length > 0) { + console.log('=== 🤖 CLAUDE AI ANALYSIS ==='); + claudeAnnotated.forEach(m => { + console.log(`${m.sha}: confidence=${m.ai_confidence}% risk=${m.risk_score}%`); + if (m.patterns && m.patterns.length) console.log(` Patterns: ${m.patterns.join(', ')}`); + if (m.architectural_concerns && m.architectural_concerns.length) console.log(` Architecture: ${m.architectural_concerns.join(', ')}`); + if (m.claude_summary) console.log(` ${m.claude_summary}`); + }); + console.log(''); + } + // Show sample commits if (metrics.length > 0) { console.log('=== 📋 SAMPLE COMMITS ==='); const sampleSize = Math.min(10, metrics.length); const samples = metrics.slice(0, sampleSize); - + samples.forEach(commit => { const lines = commit.total_additions + commit.total_deletions; const flags = []; if (commit.large_commit) flags.push('LARGE'); if (commit.sprawling_commit) flags.push('SPRAWLING'); if (commit.test_first_indicator) flags.push('TEST+PROD'); - + const flagStr = flags.length > 0 ? ` [${flags.join(', ')}]` : ''; console.log(`${commit.sha}: ${commit.message.substring(0, 60)}... (${lines} lines, ${commit.files_changed} files)${flagStr} [${commit.source_branch}]`); }); - + if (metrics.length > sampleSize) { console.log(`... and ${metrics.length - sampleSize} more commits`); } console.log(''); } - + // Output file information console.log('=== 💾 OUTPUT FILES ==='); console.log(`📄 Detailed metrics: ${metricsFile}`); @@ -459,12 +352,24 @@ async function collectLocalMetrics() { console.log('📚 Learn more: https://github.com/yourrepo/your-article'); } -// Script execution +module.exports = { + collectLocalMetrics, + CONFIG, + // git + runGitCommand, parseGitLog, isTestFile, analyzeCommit, getCommitDiff, + // statistics + computeStatistics, computeVelocity, + // metrics + scoreMessageQuality, classifyDoraArchetype, generateInsights, + // claude + CLAUDE_SYSTEM_PROMPT, getAnthropicClient, selectClaudeCommits, analyzeWithClaude +}; + +// Script execution — placed after all definitions and module.exports so all +// required lib modules are fully initialized before collectLocalMetrics() runs. if (require.main === module) { collectLocalMetrics().catch(error => { console.error('❌ Analysis failed:', error.message); process.exit(1); }); } - -module.exports = { collectLocalMetrics, parseGitLog, isTestFile, analyzeCommit, generateInsights, CONFIG }; \ No newline at end of file diff --git a/measuring-ai-code-drift-using-github-metrics.md b/measuring-ai-code-drift-using-github-metrics.md index 0e07d5a..03f0b4d 100644 --- a/measuring-ai-code-drift-using-github-metrics.md +++ b/measuring-ai-code-drift-using-github-metrics.md @@ -1,91 +1,162 @@ # Measuring AI Code Drift: Working with GitHub's Available Metrics to Track LLM Impact on Existing Codebases -As teams rapidly adopt AI coding tools, research suggests that perceived productivity gains from code generation may be offset by slowdowns elsewhere in the software delivery lifecycle, resulting in little to no net productivity improvement. Proposed causes are increased batch sizes, reduced refactoring and increased code cloning and their impact on deployment, support and maintenance. +The productivity case for AI coding tools seems straightforward: developers write code faster, complete more tasks, and merge more pull requests. But a growing body of research, most recently the DORA 2025 AI Capabilities Model report [1], is documenting a troubling paradox. The same teams reporting individual productivity gains are simultaneously experiencing slower delivery, more bugs, and longer code reviews. The tools are generating more code faster than organizations can safely absorb it. -So, we need to measure the impact of adoption on code quality, batch size, and delivery not just the rate of adoption itself. To get a quick start on this, teams can use GitHub's API to create early warning systems for problematic development patterns. +This article explains why standard measurement approaches miss the most important signals, what the research actually shows, and how to instrument your development workflow to detect AI code drift before it compounds. -## Five Metrics for Tracking AI Code Drift +## The Signal Destruction Problem -Here are the metrics I'm using, with findings an analysis from 50 commits across four feature branches in my current project: -### 1. **Large Commit Percentage** (Target: <20%) +Most software teams measure code quality at the wrong point in the process. Post-merge analysis (scanning your main branch, reviewing GitHub's aggregate statistics) sees a sanitized view of development that conceals how the code was actually written. -GitHub provides `additions + deletions` from commit stats, from which we can infer that commits with >100 lines of changes are worth investigating. Large commits often indicate developers accepting substantial AI-generated code blocks without proper decomposition, though they can also result from refactoring, clean up, or data updates. This metric serves as an early warning signal for wholesale AI code acceptance patterns. +The culprit is `git merge --squash`. This standard workflow collapses an entire feature branch (potentially dozens of commits representing days of iterative development) into a single merge commit on main. The granular signals that reveal AI-assisted development patterns (large individual commits, test discipline on a commit-by-commit basis, the ratio of additions to deletions) are destroyed at merge time. -46% of my commits contained large line count changes. +The practical consequence: local analysis of feature branches before merging consistently reveals 10x higher AI drift rates than analysis of the main branch after merging [2]. By the time the problem is visible in your standard metrics, it's embedded across your codebase. -### 2. **Sprawling Commit Percentage** (Target: <10%) +The fix is instrumentation at the right moment: capturing commit-level metrics from feature branches before they're squashed and deleted. -GitHub provides the `changed_files` count from commit details, allowing us to identify commits touching >5 files simultaneously. While not definitive, this pattern often correlates with AI-suggested "fixes" that ripple through unrelated components, though legitimate architectural changes can also cause this pattern. Sprawling commits may indicate "shotgun" problem-solving approaches that AI tools sometimes encourage. +## What DORA 2025 Research Found -20% of my commits were sprawling across multiple files. +The DORA (DevOps Research and Assessment) program, now part of Google Cloud, has tracked software delivery performance across thousands of organizations since 2014. Their 2025 AI Capabilities Model report [1] analyzed nearly 5,000 technology professionals and produced findings that challenge the prevailing narrative about AI coding tools. -### 3. **Test-First Discipline Rate** (Target: Trending upward) +### The Productivity Paradox -GitHub provides file paths and change types, from which we can infer the percentage of commits modifying both test and production files. This serves as a rough proxy for TDD practices, though it's imperfect since it can't distinguish between test-first and test-after approaches. This metric is particularly valuable because test discipline often declines when developers rely heavily on AI tools for rapid code generation. +Teams with high AI adoption reported measurable individual productivity gains: +- 98% more pull requests merged per developer +- 21% more tasks completed +- Reported improvements in documentation quality, code quality, and review speed -58% of commits showed test-first discipline, indicating good testing practices despite AI assistance. +But when DORA looked at team-level delivery metrics (the ones that actually reflect whether software is reaching users reliably), the picture reversed: -### 4. **Average Files Changed Per Commit** (Target: <5) +- **154% increase in pull request size**: AI-generated code arrives in larger batches +- **91% increase in code review time**: reviewers struggle with the volume and size of AI-generated changes +- **9% increase in bug rates**: more code, reviewed faster, with higher confidence in AI output, means more defects escape +- **7.2% reduction in delivery stability**: change failure rates increased initially with high AI adoption +- **1.5% decrease in overall delivery throughput**: despite individual productivity gains, teams delivered less -GitHub provides the count of modified files per commit, which helps us understand development granularity and change scope. High counts may indicate "shotgun" problem-solving often encouraged by AI tools, where developers make scattered changes across multiple components rather than focused, atomic modifications. However, legitimate cross-cutting changes can also produce this pattern. +More code is being produced and merged more quickly, reviewed more slowly, and breaking more often. -6.42 files changed per commit on average, above the target threshold. +### The AI Amplifier Effect -### 5. **Average Lines Changed Per Commit** (Target: <100) +DORA's central finding is that AI tools don't change a team's fundamental trajectory; they accelerate it. Teams with strong foundational practices (automated testing, CI/CD, version control discipline, working in small batches) found that AI amplified their existing strengths. Teams with weak foundations used AI tools to generate technical debt faster. -GitHub provides total additions and deletions, giving us insight into development work granularity. Massive changes often correlate with accepting large AI generations wholesale, though they can also indicate data migrations, major refactoring efforts, or significant feature implementations. This metric helps identify when AI assistance is leading to batch-style development rather than incremental progress. +DORA identifies seven organizational capabilities that amplify AI's positive outcomes [3]. Two of these are directly observable in commit history and form the foundation of the measurement approach described here: -9,053 lines changed per commit on average, indicating heavy AI-assisted batch coding. +- **Strong version control practices** (Capability 4): frequent commits, mature rollback capability, disciplined branching +- **Working in small batches** (Capability 5): a long-standing DORA principle that becomes even more critical in AI-assisted environments - **Note**. The standard github merge/squash workflow can delete feature branches unless "Automatically delete head branches" is turned off. This removes the history of individual commits that we want to analyze. We can set this analysis up as a git action, running against remote repos that retain feature branches -- or -- as a local node.js script. +The remaining five capabilities (organizational AI policy, data ecosystem quality, internal knowledge systems, user-centric focus, and platform quality) require organizational and infrastructure data not available in git history. + +### Team Archetypes + +DORA describes seven team archetypes based on the intersection of performance, stability, and AI adoption. Two are most relevant to code drift measurement: + +**Harmonious high-achievers**: Strong foundational practices + AI tools = compounding gains. In commit history: small batches, consistent test discipline, stable and measured velocity, high commit message specificity. + +**Foundational challenges**: Weak practices + AI tools = compounding debt. In commit history: large commits, low test-first discipline, erratic or accelerating velocity, vague commit messages. + +Identifying which archetype describes your team helps calibrate which signals to prioritize first. + +## Why This Matters: GitClear's Independent Evidence + +GitClear, a code intelligence platform specializing in AI drift detection, provides independent evidence that corroborates DORA's findings [4]. Their 2025 research on code churn patterns documented a threshold crossed for the first time: copy-paste operations now exceed code moves in repositories with high AI adoption. This is significant because copy-paste at scale is a leading indicator of the kind of technical debt that compounds invisibly: code that appears to work but creates hidden coupling and increases maintenance cost over time. + +GitClear's analysis also shows that churn rates (code written and then rewritten within two weeks) have increased alongside AI adoption, suggesting that AI-generated code requires more downstream rework than human-written code, offsetting the speed gains from generation. + +These findings are consistent with what DORA describes as the rework paradox: individual developers write code faster, but the downstream effects (larger reviews, more bugs, more rework) push net team productivity below the pre-AI baseline for teams without strong foundations. + +## What We Can Measure (and What We Can't) + +Git commit history is a rich data source for two of DORA's seven capabilities. It is silent on the other five. + +**What git reveals** (DORA Capabilities 4 and 5): +- Commit size distribution and trends +- Sprawl (files changed per commit) +- Test discipline (co-occurrence of test and production changes) +- Commit velocity and velocity trends +- Additions-to-deletions ratios (the batch-acceptance signature) +- Commit message specificity + +**What git cannot reveal**: +- The four core DORA delivery metrics (deployment frequency, lead time, change failure rate, MTTR): these require CI/CD and incident data +- Copy-paste and code cloning detection: requires AST-level diff analysis (GitClear's approach) +- Code review quality: reviewer count and comment depth require GitHub API data +- Architectural boundary violations: requires dependency graph analysis; the Claude API integration described below partially addresses this +- DORA capabilities 1, 2, 3, 6, 7: require organizational policy, data infrastructure, and product telemetry + +A detailed breakdown of measurable signals, gaps, and the tools that address each gap is available in the companion [Metrics Specification](metrics-specification.md). ## How to Measure These Patterns ### Option 1: GitHub Actions for Pre-Merge Analysis (Recommended) -The most scalable approach uses a dual GitHub Actions: one workflow for ongoing monitoring and another for real-time PR feedback. +The most scalable approach uses two GitHub Actions: one workflow for ongoing monitoring and another for real-time PR feedback. -**Weekly Analysis Workflow** runs every Sunday and analyzes feature branches from the last 30 days before they're merged and squashed. This workflow enumerates all branches except main/master, processes up to 50 commits, and generates detailed metrics including file-by-file analysis to distinguish between test and production code changes. - -The workflow automatically creates GitHub issues with concerning patterns and uploads detailed metrics as artifacts for historical tracking. +**Weekly Analysis Workflow** runs every Sunday and analyzes feature branches from the last 30 days before they're merged and squashed. The workflow enumerates all branches except main/master, processes up to 50 commits, and generates detailed metrics including file-by-file analysis to distinguish test from production code changes. It automatically creates GitHub issues with concerning patterns and uploads metrics artifacts for historical tracking. **Real-Time PR Analysis** triggers on every pull request and provides immediate feedback on size and scope. This prevents problematic patterns from being merged while they're still visible and actionable. + ### Option 2: Local Analysis Script -For teams wanting immediate analysis of their existing local development patterns, a Node.js script can process your local repository directly. This approach is particularly valuable for discovering the gap between your actual development patterns and what's visible remotely. +For teams wanting immediate analysis of existing local development patterns, a Node.js script can process the repository directly. This approach is particularly valuable for discovering the gap between actual development patterns and what's visible remotely after squash-merging. + +The script enumerates all local feature branches, analyzes commits from the last 30 days, and generates detailed metrics including multi-language test file detection. + +### Option 3: Claude API Diff-Level Analysis (Emerging) + +Heuristics catch the shape of a problem; AI analysis can explain what's actually wrong. Sending high-risk commit diffs to a Claude API endpoint adds semantic pattern detection that rule-based metrics cannot replicate: + +- **AI-generated code signature detection**: generic variable names (`data`, `result`, `item`), boilerplate CRUD without error handling, identically structured adjacent functions, absent domain language in identifiers +- **Architectural boundary violation detection**: code that crosses service or module boundaries in ways that violate established patterns in the codebase +- **Per-commit risk scoring**: a 0-100 confidence score with natural language explanation of specific concerns + +The practical implementation pre-filters commits where `large_commit = true AND additions > deletions x 3` to keep API costs low (typically 3-5 commits per analysis run). An `ANTHROPIC_API_KEY` environment variable gates the feature; if absent, the analysis skips gracefully and the rest of the metrics run unchanged. -The script enumerates all local feature branches, analyzes commits from the last 30 days, and generates detailed metrics including corrected test file detection patterns for various project types. +This approach works best as a second pass: heuristics flag the candidates, Claude explains what's actually problematic about them. ## Available Commercial Solutions -### **GitClear** +### GitClear -The most specialized solution for AI code drift detection. Goes beyond basic git stats to classify code operations including moved, copy/pasted, and duplicated blocks. Their 2025 research shows copy/paste frequency now exceeds code moves for the first time—a clear AI drift indicator. Offers free starter tier. +The most specialized solution for AI code drift detection. Goes beyond commit statistics to classify code operations including moved, copy/pasted, and duplicated blocks. This is the AST-level analysis that git heuristics cannot replicate. Their 2025 research on copy-paste exceeding code moves is the most direct quantitative evidence of structural AI drift available. Offers a free starter tier. -### **DX (Developer Experience Platform)** +### DX (Developer Experience Platform) -Focuses on broader productivity impacts. Tracks code review velocity and deployment frequency to detect when AI tools create downstream bottlenecks. Strong DORA metrics integration. Best for engineering leaders wanting full lifecycle visibility. +Focuses on broader productivity impacts. Tracks code review velocity and deployment frequency to detect when AI tools create downstream bottlenecks. Strong DORA metrics integration, including the delivery metrics (change failure rate, deployment frequency) that git analysis cannot surface. Best for engineering leaders who need full lifecycle visibility. -### **LinearB** +### LinearB -Provides engineering intelligence with indirect AI drift detection. Monitors pull request sizes, cycle times, and code review bottlenecks. Good for teams wanting comprehensive metrics that correlate AI adoption with delivery performance. +Provides engineering intelligence with indirect AI drift detection. Monitors pull request sizes, cycle times, and code review bottlenecks. Good for teams that want comprehensive metrics correlating AI adoption with delivery performance, without needing to instrument their own analysis. ## Recommendations for Teams -**Start with awareness:** Understand that your current measurement approach may be missing critical signals if you're only analyzing the main branch. +**Classify your team archetype first.** DORA's research shows the right intervention depends on where you are. A team in "foundational challenges" needs to strengthen testing and batch discipline before scaling AI usage. A "harmonious high-achiever" team can use drift metrics to fine-tune an already-healthy practice. The same metric reading means different things in different contexts. + +**Start with awareness of the signal destruction problem.** Understand that if you are only analyzing the main branch after merge, you are seeing a curated view that systematically hides the patterns that matter most. + +**Implement dual tracking.** Measure both pre-merge (real development patterns, via feature branch analysis) and post-merge (workflow efficiency, via delivery metrics). The gap between what pre-merge analysis shows and what post-merge shows is itself a signal: a large gap means your merge process is obscuring problematic patterns. + +**Use distributions, not averages.** A p90 commit size tells you more than a mean. An average of 65 lines that hides a p90 of 500 lines describes a fundamentally different team than one where both numbers are low. Averages normalize outliers; distributions expose them. + +**Focus on trends, not point-in-time readings.** A jump from 10% to 30% large commits over 60 days deserves investigation even if 30% is below your threshold. Velocity combined with direction is the leading indicator; an absolute number is the lagging one. + +**Calibrate thresholds to DORA capabilities.** The thresholds in this toolkit (large commit < 20%, sprawling commit < 10%, test discipline > 50%) correspond to the boundaries DORA found separating teams that benefit from AI tools from teams that are harmed by them. They are not arbitrary. + +--- + +## References -**Implement dual tracking:** Measure both pre-merge (real development patterns) and post-merge (workflow efficiency) metrics. +[1] DORA. *State of AI-Assisted Software Development 2025*. Google Cloud, 2025. Available: https://dora.dev/research/2025/dora-report/ -**Calibrate thresholds:** My 46% large commits would be concerning for most teams, but context matters. Establish baselines for your specific development patterns. +[2] Judy, K. *AI Code Drift Local Analysis Script*. GitHub, 2025. Available: https://github.com/stride-nyc/code-quality-metrics -**Focus on trends:** Look for degradation over time rather than absolute numbers. A jump from 10% to 30% large commits deserves investigation. +[3] DORA. *Introducing DORA's Inaugural AI Capabilities Model*. Google Cloud Blog, 2025. Available: https://cloud.google.com/blog/products/ai-machine-learning/introducing-doras-inaugural-ai-capabilities-model -## Conclusion +[4] GitClear. *AI Copilot Code Quality Research 2025*. GitClear, 2025. Available: https://www.gitclear.com/coding_on_copilot_data_shows_ais_downward_pressure_on_code_quality -AI coding tools offer genuine productivity benefits, but they require careful monitoring to prevent drift toward unmaintainable code patterns. The key insight from my analysis is that **we can't manage what we can't measure accurately**. +[5] DORA. *From Adoption to Impact: Putting the DORA AI Capabilities Model to Work*. Google Cloud Blog, 2025. Available: https://cloud.google.com/blog/products/ai-machine-learning/from-adoption-to-impact-putting-the-dora-ai-capabilities-model-to-work -Teams serious about sustainable AI adoption need measurement strategies that capture actual development behavior, not just the sanitized view that emerges after workflow processing. Whether through enhanced GitHub Actions, commercial tools, or local analysis scripts, the goal is maintaining visibility into the development patterns that determine long-term codebase health. +[6] DORA. *DORA's Software Delivery Performance Metrics*. dora.dev, 2024. Available: https://dora.dev/guides/dora-metrics/ -The choice isn't between using AI tools or not—it's between using them thoughtfully with proper measurement, or blindly and discovering the consequences later. +[7] SonarSource. *The Inevitable Rise of Poor Code Quality in AI-Accelerated Codebases*. Sonar Blog, 2025. Available: https://www.sonarsource.com/blog/the-inevitable-rise-of-poor-code-quality-in-ai-accelerated-codebases/ -By implementing these measures, I've created a baseline and can monitor how my code quality and batch size improve as I get more disciplined about applying a structured Plan-Do-Check-Act approach with AI tools. Future articles will compare these interaction styles using the same metrics framework. +[8] IT Revolution. *AI's Mirror Effect: How the 2025 DORA Report Reveals Your Organization's True Capabilities*. IT Revolution, 2025. Available: https://itrevolution.com/articles/ais-mirror-effect-how-the-2025-dora-report-reveals-your-organizations-true-capabilities/ diff --git a/metrics-specification.md b/metrics-specification.md new file mode 100644 index 0000000..4b0d5fd --- /dev/null +++ b/metrics-specification.md @@ -0,0 +1,637 @@ +# Code Drift Metrics Specification + +Technical reference for the AI Code Drift analysis toolkit. Covers what is measured, how each metric is computed, what the thresholds mean, what cannot be measured with this approach, and how the implementation is configured. + +For the research background and practitioner recommendations behind this specification, see [Measuring AI Code Drift](measuring-ai-code-drift-using-github-metrics.md). + +**Runtime requirement**: Node.js ≥ 18 (required for the optional `@anthropic-ai/sdk` Claude API integration). + +--- + +## DORA Capability Coverage Map + +This toolkit directly addresses two of DORA's seven AI-amplifying capabilities, which DORA research identifies as the most directly measurable predictors of AI tool outcomes. + +### Capabilities Covered by This Toolkit + +| DORA Capability | Coverage | Metrics | +|-----------------|----------|---------| +| #4 Strong Version Control Practices | Full | Message quality score, commit velocity, branch discipline | +| #5 Working in Small Batches | Full | Large commit %, sprawling commit %, lines/files distributions, velocity trend | + +### Capabilities Not Addressable via Git History + +| DORA Capability | Why Not Measurable | Gap | +|-----------------|-------------------|-----| +| #1 Clear and Communicated AI Stance | Organizational policy | Out of scope | +| #2 Healthy Data Ecosystems | Data infrastructure quality | Out of scope | +| #3 AI-Accessible Internal Data | Internal knowledge systems | Out of scope | +| #6 User-Centric Focus | Product and UX decisions | Out of scope | +| #7 Quality Internal Platforms | CI/CD and tooling quality | Partially visible in workflow file complexity; not commit patterns | + +### DORA Delivery Metrics Not Measurable from Git + +| DORA Metric | Data Required | Gap | +|-------------|--------------|-----| +| Deployment Frequency | CI/CD pipeline data | Use DX or LinearB for full lifecycle visibility | +| Lead Time for Changes | Commit → production timestamps | Partial proxy: branch lifetime (creation → merge) | +| Change Failure Rate | Incident / rollback data | Not addressable without incident tracking integration | +| Mean Time to Recovery (MTTR) | Incident duration data | Not addressable without incident tracking integration | + +--- + +## Metrics Reference + +### Metric 1: Large Commit Percentage + +**What it measures**: The proportion of commits that exceed a line-change threshold, used as a proxy for wholesale AI code acceptance. + +**Formula**: +``` +large_commit_pct = (commits where additions + deletions > LARGE_COMMIT_THRESHOLD) / total_commits × 100 +``` + +**Per-commit flag**: `large_commit: boolean` + +**Data source**: `git show --numstat {sha}` (additions and deletions per file, summed across all files) + +**CONFIG key**: `LARGE_COMMIT_THRESHOLD` (default: 100 lines) + +**Thresholds**: +| Range | Signal | +|-------|--------| +| < 20% | Healthy: consistent with DORA "small batches" capability | +| 20–40% | Warning: elevated AI batch-acceptance risk | +| > 40% | Critical: strong AI drift indicators | + +**False positives**: Legitimate large commits include data migrations, bulk refactoring, large file additions (assets, generated code), and one-time cleanup. Context from `large_commit AND additions > deletions × 3` narrows to the AI-specific pattern. + +--- + +### Metric 2: Sprawling Commit Percentage + +**What it measures**: The proportion of commits that touch more files than the threshold, used as a proxy for "shotgun" problem-solving where AI-suggested fixes ripple through unrelated components. + +**Formula**: +``` +sprawling_commit_pct = (commits where files_changed > SPRAWLING_COMMIT_THRESHOLD) / total_commits × 100 +``` + +**Per-commit flag**: `sprawling_commit: boolean` + +**Data source**: `git show --numstat {sha}` (count of file entries) + +**CONFIG key**: `SPRAWLING_COMMIT_THRESHOLD` (default: 5 files) + +**Thresholds**: +| Range | Signal | +|-------|--------| +| < 10% | Healthy | +| 10–25% | Warning: watch for scope creep | +| > 25% | Critical: possible shotgun surgery | + +**DORA connection**: DORA's research documents a 154% increase in pull request size with high AI adoption. Sprawling commits are the commit-level precursor to oversized PRs. + +--- + +### Metric 3: Test-First Discipline Rate + +**What it measures**: The proportion of commits that modify both test files and production files in the same commit, used as a proxy for test discipline under AI-assisted development. + +**Formula**: +``` +test_first_pct = (commits where test_files_count > 0 AND prod_files_count > 0) / total_commits × 100 +``` + +**Per-commit flag**: `test_first_indicator: boolean` + +**Data source**: `git show --numstat {sha}` (file paths matched against `TEST_FILE_PATTERNS`) + +**CONFIG key**: `TEST_FILE_PATTERNS` (array of 8 regex patterns; covers JS/TS, Python, Go, Java, C#) + +**Thresholds**: +| Range | Signal | +|-------|--------| +| > 50% | Healthy: strong test discipline | +| 30–50% | Warning: monitor AI tool usage patterns | +| < 30% | Critical: AI tools may be bypassing TDD practices | + +**Limitations**: This metric cannot distinguish test-first (TDD) from test-after. It measures co-occurrence, not ordering. A commit that adds production code and test code written afterward scores the same as one where tests were written first. + +**DORA connection**: DORA's research identifies automated testing as the single strongest predictor of whether AI tools help or hurt a team. Teams without it when they adopt AI see the fastest debt accumulation. + +--- + +### Metric 4: Lines Changed Per Commit (Distribution) + +**What it measures**: The statistical distribution of commit sizes by line count. Distributions reveal patterns that averages conceal: a p90 of 500 lines with a p50 of 30 lines describes a "mostly disciplined with occasional explosions" pattern that an average of 65 lines hides entirely. + +**Fields**: +``` +p50_lines_changed : median commit size (lines) +p90_lines_changed : 90th percentile commit size +p95_lines_changed : 95th percentile commit size +stddev_lines_changed : standard deviation +avg_lines_changed : mean (kept for backwards compatibility) +commit_size_trend : "growing" | "stable" | "shrinking" +``` + +**Formula** (commit size trend): +``` +Fit linear regression: commit_size ~ commit_index (time-ordered) +slope > 0: "growing" +slope < 0: "shrinking" +|slope| < threshold: "stable" +``` + +**Implementation**: `simple-statistics` library: `quantile()`, `mean()`, `standardDeviation()`, `linearRegression()` + +**Risk signal**: `commit_size_trend: "growing"` combined with `velocity_trend: "accelerating"` is the joint indicator DORA describes as "volume without discipline." + +**Thresholds** (p90 guidance): +| Range | Signal | +|-------|--------| +| p90 < 200 lines | Healthy | +| p90 200–500 lines | Monitor | +| p90 > 500 lines | Investigate: high review burden | + +--- + +### Metric 5: Files Changed Per Commit (Distribution) + +**What it measures**: The statistical distribution of commit scope by file count. Complements Metric 2 by showing the shape of the distribution, not just the percentage above threshold. + +**Fields**: +``` +p50_files_changed : median files per commit +p90_files_changed : 90th percentile files per commit +avg_files_changed : mean (kept for backwards compatibility) +``` + +**Implementation**: `simple-statistics` library: `quantile()`, `mean()` + +**Thresholds** (p90 guidance): +| Range | Signal | +|-------|--------| +| p90 < 8 files | Healthy | +| p90 8–15 files | Monitor | +| p90 > 15 files | Investigate: architectural scatter | + +--- + +### Metric 6: Commit Velocity Trend + +**What it measures**: How quickly commits are being produced, and whether that rate is accelerating or decelerating over the analysis window. Velocity alone is neutral; velocity combined with commit size trend is the meaningful signal. + +**Formulas**: +``` +velocity_commits_per_day = total_commits / (last_commit_date - first_commit_date) in days + +Velocity trend: + Split commits at time midpoint into first_half and second_half + first_half_rate = first_half_count / half_window_days + second_half_rate = second_half_count / half_window_days + + Accelerating: second_half_rate > first_half_rate × 1.25 + Decelerating: second_half_rate < first_half_rate × 0.75 + Stable: otherwise +``` + +**Fields**: +``` +velocity_commits_per_day : float +velocity_trend : "accelerating" | "stable" | "decelerating" +``` + +**Data source**: `date` field from `git log --pretty=format:"%ai"`. Already collected in the existing analysis loop; no new git calls required. + +**Risk signal**: `velocity_trend: "accelerating"` combined with `commit_size_trend: "growing"`. DORA research identifies this combination as the leading indicator of team archetype drift toward "foundational challenges." + +**Note**: A single-day analysis window (all commits on one day) yields `velocity_commits_per_day` but `velocity_trend: "stable"` by convention. + +--- + +### Metric 7: Additions-to-Deletions Ratio Distribution + +**What it measures**: The median and 90th percentile of the per-commit ratio of lines added to lines deleted. High ratios indicate that new code is being added without commensurate refactoring or removal of replaced code. This is the systematic batch-acceptance pattern DORA associates with architectural debt accumulation. + +**Formula**: +``` +per-commit ratio = total_additions / max(total_deletions, 1) +additions_ratio_median = quantile(all_ratios, 0.5) +additions_ratio_p90 = quantile(all_ratios, 0.9) +``` + +**Fields**: +``` +additions_ratio_median : float (median ratio across all commits) +additions_ratio_p90 : float (90th percentile ratio) +``` + +**Data source**: `total_additions` and `total_deletions` already collected per commit; no new git calls required + +**Thresholds**: +| Range (median) | Signal | +|----------------|--------| +| < 2.0 | Healthy: balanced additions and deletions | +| 2.0–3.0 | Monitor: additions outpacing deletions | +| > 3.0 | Warning: systematic batch-acceptance pattern | + +**Relationship to existing heuristic**: The existing `generateInsights()` function counts commits where `large_commit AND additions > deletions × 3` as "possible AI commits." This metric expresses the same pattern at the aggregate level with a distribution, so outlier commits don't distort the reading. + +--- + +### Metric 8: Commit Message Quality Score + +**What it measures**: The proportion of commit messages that meet a minimum quality bar: following conventional commit format, or containing enough words to be considered specific. Message quality declines with AI over-reliance as developers accept AI-suggested vague descriptions ("update stuff", "fix issue", "wip"). + +**Formula**: +``` +For each commit message: + conventional = /^(feat|fix|refactor|test|chore|docs|perf|ci|build|revert)(\(.+\))?:/i.test(message) + specific = message.split(' ').filter(Boolean).length >= MESSAGE_QUALITY_MIN_WORDS + quality = conventional OR specific + +message_quality_pct = (quality commits / total commits) × 100 +``` + +**Fields**: +``` +message_quality_pct : float (percentage of quality commits) +``` + +**CONFIG key**: `MESSAGE_QUALITY_MIN_WORDS` (default: 10) + +**Thresholds**: +| Range | Signal | +|-------|--------| +| > 60% | Healthy: good commit message discipline | +| 40–60% | Monitor | +| < 40% | Warning: messaging discipline issues | + +**Design decision: why not NLP**: Conventional commit classification requires a 3-line regex. Word count requires one line. Adding a 200KB+ NLP library (`compromise`, `wink-nlp`) for these two signals is unjustified. The regex approach is zero-dependency, faster, more maintainable, and easier to test. + +**Limitations**: This metric cannot assess semantic quality. A message that says "feat: add user authentication for all supported OAuth providers" scores the same as "feat: a." The word-count threshold partially compensates, but it cannot detect technically-compliant messages that are still vague. + +--- + +## Derived Metrics + +### Per-Commit Outlier Flag + +**What it measures**: Whether an individual commit is a statistical outlier relative to the rest of the analysis window. + +**Formula**: +``` +mean_lines = mean(all commit sizes) +stddev_lines = standardDeviation(all commit sizes) +outlier = (total_additions + total_deletions) > (mean_lines + 2 × stddev_lines) +``` + +**Per-commit field**: `outlier: boolean` + +**Use**: Displayed in the sample commits table in console output. Useful for manual investigation: outlier commits are the ones most likely to warrant direct review. + +--- + +### DORA Archetype Classification + +**What it measures**: Which of four DORA team archetypes best describes the commit patterns in the analysis window. This is a heuristic classification based on the composite of all eight metrics, intended to contextualize threshold readings rather than replace them. + +**Classification logic** (evaluated in order): + +``` +harmonious-high-achiever: + large_commits_pct < 20 + AND sprawling_commits_pct < 10 + AND test_first_pct > 50 + AND message_quality_pct > 60 + +legacy-bottleneck: + sprawling_commits_pct > 25 + AND large_commits_pct > 30 + +foundational-challenges: + large_commits_pct > 40 + OR (test_first_pct < 30 AND large_commits_pct > 20) + +mixed-signals: + (all other combinations) +``` + +**Field**: `dora_archetype: "harmonious-high-achiever" | "foundational-challenges" | "legacy-bottleneck" | "mixed-signals"` + +**Interpretation**: + +| Archetype | What It Suggests | +|-----------|-----------------| +| `harmonious-high-achiever` | Strong foundation; AI tools likely amplifying positive outcomes | +| `foundational-challenges` | Weak testing/batch discipline; AI tools likely accelerating debt | +| `legacy-bottleneck` | Architectural scatter; AI making cross-cutting changes worse | +| `mixed-signals` | Inconsistent patterns; investigate specific outliers | + +**Limitation**: This classification is based on a 30-day window of at most 50 commits. It is a directional signal, not a definitive assessment. Teams near archetype boundaries should look at individual metric thresholds, not just the archetype label. + +--- + +## Claude API Integration (Optional) + +When `ANTHROPIC_API_KEY` is set in the environment, the toolkit performs a supplementary AI-powered analysis of the highest-risk commits. This feature is completely optional. All eight metrics above run with zero external dependencies when the key is absent. + +### Pre-Filter Logic + +To limit API costs, only a subset of commits are sent for analysis: + +``` +Candidates = commits where: + large_commit = true + AND total_additions > total_deletions × AI_RISK_ADDITIONS_RATIO + +Sort candidates by (total_additions + total_deletions) descending +Take top AI_ANALYSIS_MAX_COMMITS +``` + +**CONFIG keys**: +- `AI_ANALYSIS_MAX_COMMITS` (default: 5) +- `AI_RISK_ADDITIONS_RATIO` (default: 3; also used in the `generateInsights()` heuristic) + +### Diff Extraction + +For each selected commit: +```bash +git show --stat {sha} # file summary +git diff {sha}^ {sha} -- # full diff +``` + +Combined output truncated at `AI_DIFF_MAX_CHARS` characters (default: 4000). Truncation drops from the end of the diff, preserving file headers and early hunks. + +### Structured Output (claude-sonnet-4-6) + +The diff is sent with a system prompt describing AI code patterns to detect. The model responds with structured JSON: + +```json +{ + "ai_confidence": 0-100, + "risk_score": 0-100, + "patterns": ["string", ...], + "architectural_concerns": ["string", ...], + "summary": "string" +} +``` + +**Pattern categories detected**: +- Generic variable names (`data`, `result`, `item`, `temp`) +- Boilerplate CRUD without error handling +- Identically structured adjacent functions (copy-paste with variable substitution) +- Absent domain language in identifiers +- Imports that don't match the rest of the file's dependency patterns + +**Architectural concerns detected** (Claude infers these from diff context): +- Code crossing service/module boundaries in ways inconsistent with established patterns +- New dependencies on modules that aren't imported elsewhere in the changed files +- Structural patterns inconsistent with the existing file's approach + +### Output File + +Results are written to `local_claude_analysis.json`: + +```json +{ + "analyzed_at": "ISO 8601 timestamp", + "model": "claude-sonnet-4-6", + "commits_analyzed": 5, + "results": [ + { + "sha": "abc12345", + "ai_confidence": 78, + "risk_score": 82, + "patterns": ["generic variable names", "boilerplate CRUD without error handling"], + "architectural_concerns": ["crosses auth/billing service boundary"], + "summary": "High probability AI-generated boilerplate. Three functions have identical structure with variable substitution. No domain-specific error handling." + } + ] +} +``` + +Claude findings are also annotated onto the matching commit entries in `local_commit_metrics.json`. + +### Graceful Degradation + +If `ANTHROPIC_API_KEY` is absent: +- A single log line: `Claude analysis skipped (no ANTHROPIC_API_KEY set)` +- All other metrics run unchanged +- `local_claude_analysis.json` is not written +- No error or exit code change + +### Cost Estimate + +At the default `AI_ANALYSIS_MAX_COMMITS: 5` with 4000-char diffs, a typical run costs approximately $0.02–0.05 USD using claude-sonnet-4-6. Actual cost depends on diff sizes. + +--- + +## Persistent Measurement Gaps + +These signals are not addressable by this toolkit. Each gap is noted with the best alternative approach: + +1. **Copy-paste and code cloning detection**: Requires AST-level diff analysis to detect when code is duplicated with minor modifications. GitClear is the specialized commercial solution. This toolkit's additions-ratio metric is a proxy for the outcome (more code added than removed) but cannot detect the structural pattern directly. + +2. **DORA delivery metrics** (deployment frequency, lead time, change failure rate, MTTR): Require integration with CI/CD pipelines and incident tracking systems. DX and LinearB provide these for organizations that want full lifecycle visibility alongside git-level analysis. + +3. **Code review quality**: Reviewer count, comment depth, and review turnaround time are available via GitHub API. The GitHub workflow variant of this toolkit (`pr-metrics.yml`) surfaces PR-level signals, but the local script has no access to review data. + +4. **Architectural boundary violations without Claude**: Detecting whether code crosses architectural boundaries (service layers, domain boundaries, module dependencies) without semantic analysis requires a dependency graph of the codebase. Without Claude API enabled, this toolkit can detect structural patterns (sprawl, large commits) but not semantic architectural violations. + +5. **AI tool usage specifics**: Which AI tools are being used, how frequently suggestions are accepted, and which patterns come from which models require IDE telemetry. This is not available in git history. + +6. **DORA capabilities 1, 2, 3, 6, 7**: Organizational AI stance, data ecosystem quality, internal knowledge accessibility, user-centric focus, and platform quality all require organizational survey data or infrastructure telemetry. DORA measures these through their survey instrument. + +7. **Developer well-being and burnout**: DORA research shows that AI adoption affects developer well-being, which in turn affects all other metrics. This requires survey data. + +--- + +## Configuration Reference + +All thresholds are set in the `CONFIG` object at the top of `local-code-metrics.js`. The GitHub workflows have equivalent values hard-coded in their shell/jq logic. Update both locations when adjusting thresholds. + +```javascript +const CONFIG = { + // Analysis window + ANALYSIS_DAYS: 30, // days of history to analyze + MAX_COMMITS: 50, // maximum commits to analyze (most recent first) + + // Commit size thresholds + LARGE_COMMIT_THRESHOLD: 100, // lines changed threshold for large_commit flag + SPRAWLING_COMMIT_THRESHOLD: 5, // files changed threshold for sprawling_commit flag + + // Message quality + MESSAGE_QUALITY_MIN_WORDS: 10, // minimum word count for a "specific" message + // (applies when message doesn't match conventional format) + + // Claude API integration (optional) + AI_ANALYSIS_MAX_COMMITS: 5, // maximum commits sent to Claude per run + AI_DIFF_MAX_CHARS: 4000, // character limit for diffs sent to Claude + AI_RISK_ADDITIONS_RATIO: 3, // additions/deletions multiplier for Claude pre-filter + // also used in generateInsights() heuristic + + // Test file detection (customize for your language/framework) + TEST_FILE_PATTERNS: [ + /\.(test|spec)\./i, // file.test.js, file.spec.ts + /Tests?\.cs$/i, // FileTests.cs, FileTest.cs (C#) + /Test\.java$/i, // FileTest.java (Java) + /_test\.py$/i, // file_test.py (Python) + /test_.*\.py$/i, // test_file.py (Python) + /_test\.go$/i, // file_test.go (Go) + /__tests__/i, // __tests__ directory + /\/tests?\//i // /test/ or /tests/ directories + ] +}; +``` + +--- + +## Output Format Reference + +### `local_commit_metrics.json` + +Array of `CommitMetric` objects, one per analyzed commit: + +```typescript +{ + // Identity (from git log) + sha: string, // 8-character short SHA + full_sha: string, // full 40-character SHA + date: string, // ISO 8601 timestamp + author: string, // author name + message: string, // commit subject line + source_branch: string, // branch this commit was found on + + // File statistics (from git show --numstat) + total_additions: number, + total_deletions: number, + files_changed: number, + binary_files: number, + test_files_count: number, + prod_files_count: number, + + // Derived flags + test_first_indicator: boolean, + large_commit: boolean, + sprawling_commit: boolean, + change_ratio: string, // "X.XX" or "inf" + outlier: boolean, // true if > mean + 2σ for this analysis window + commit_type: "feature_branch", + + // Message quality (new) + message_quality: boolean, // true if message meets quality threshold + + // Claude API annotation (present only when ANTHROPIC_API_KEY is set and commit was analyzed) + ai_confidence?: number, // 0-100 + risk_score?: number, // 0-100 + patterns?: string[], + architectural_concerns?: string[], + claude_summary?: string +} +``` + +### `local_metrics_summary.json` + +Single summary object for the analysis run: + +```typescript +{ + // Run metadata + analysis_date: string, // ISO 8601 + analysis_period_days: number, + total_commits: number, + filtered_from: number, // unique commits before MAX_COMMITS cap + branches_analyzed: string[], + branch_commit_counts: Record, + + // Original 5 metrics (preserved for backwards compatibility) + large_commits_pct: string, // "XX.XX" + sprawling_commits_pct: string, + test_first_pct: string, + avg_files_changed: string, + avg_lines_changed: string, + + // Statistical distributions (new) + p50_lines_changed: number, + p90_lines_changed: number, + p95_lines_changed: number, + stddev_lines_changed: number, + p50_files_changed: number, + p90_files_changed: number, + commit_size_trend: "growing" | "stable" | "shrinking", + + // Velocity metrics (new) + velocity_commits_per_day: number, + velocity_trend: "accelerating" | "stable" | "decelerating", + + // Additions ratio distribution (new) + additions_ratio_median: number, + additions_ratio_p90: number, + + // Message quality (new) + message_quality_pct: string, // "XX.XX" + + // DORA archetype (new) + dora_archetype: "harmonious-high-achiever" | "foundational-challenges" | "legacy-bottleneck" | "mixed-signals", + + // Configuration snapshot + config: CONFIG, + note: string +} +``` + +### `local_claude_analysis.json` + +Written only when `ANTHROPIC_API_KEY` is set: + +```typescript +{ + analyzed_at: string, // ISO 8601 + model: string, // "claude-sonnet-4-6" + commits_analyzed: number, + results: Array<{ + sha: string, + ai_confidence: number, // 0-100 + risk_score: number, // 0-100 + patterns: string[], + architectural_concerns: string[], + summary: string + }> +} +``` + +--- + +## Implementation Libraries + +### `simple-statistics` (production dependency) + +Used for: `quantile()` (p50/p90/p95), `mean()`, `standardDeviation()`, `linearRegression()` (trend slope), `median()`. + +**Why chosen**: Zero dependencies, 47KB, works in Node and browser, comprehensive coverage of the statistical operations needed. Replaces four hand-rolled average calculations with a single well-tested library. + +**Why not a larger ML library**: This toolkit needs descriptive statistics and linear trend detection, not machine learning, clustering, or inference. `simple-statistics` covers exactly the needed surface without the overhead of `ml.js`, `tensorflow.js`, or equivalent. + +### `@anthropic-ai/sdk` (production dependency, optional at runtime) + +Used for: Claude API calls in the diff-level analysis feature (Metric 3 supplement / Claude integration section above). + +**Why chosen**: Official Anthropic SDK, actively maintained, full TypeScript types, supports structured JSON output mode, prompt caching available. + +**Runtime dependency, not hard requirement**: The SDK is imported conditionally. If `ANTHROPIC_API_KEY` is absent at runtime, the import path is never reached and no network calls are made. Users on Node 16 who don't set the API key are unaffected by the Node 18+ requirement. + +### What Was Explicitly Rejected + +| Library | Reason Rejected | +|---------|----------------| +| `compromise` (NLP) | 200KB+ for what 3 lines of regex accomplish; message classification does not need ML | +| `wink-nlp` | Same rationale as compromise; heavier and more complex | +| `simple-git` | Shell exec approach in `runGitCommand()` is already abstracted, tested, and working; no benefit from wrapping it further | +| `nodegit` | Native compilation dependencies; declining maintenance; not worth the complexity for shell-replaceable operations | +| `isomorphic-git` | No browser requirement; pure-JS advantage doesn't apply to a Node CLI tool | +| `plato` | Deprecated: last updated 9 years ago, no ES6+ support | +| `escomplex` | Poor TypeScript support; superseded by typhonjs-escomplex, but file complexity analysis is outside the scope of this toolkit's commit-level focus | +| `@octokit/rest` | Already used in GitHub workflows; not needed in the local script which uses git CLI directly | diff --git a/package-lock.json b/package-lock.json index 657c6b8..538bb0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,10 +7,14 @@ "": { "name": "code-quality-metrics", "version": "1.0.0", + "dependencies": { + "@anthropic-ai/sdk": "^0.80.0", + "dotenv": "^17.3.1" + }, "devDependencies": { "@eslint/js": "^9.0.0", - "eslint": "^9.0.0", - "eslint-plugin-jest": "^28.0.0", + "eslint": "^9.39.4", + "eslint-plugin-jest": "^29.15.1", "jest": "^29.7.0", "typescript": "^5.9.3" }, @@ -18,6 +22,26 @@ "node": ">=14" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.80.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.80.0.tgz", + "integrity": "sha512-WeXLn7zNVk3yjeshn+xZHvld6AoFUOR3Sep6pSoHho5YbSi6HwcirqgPA5ccFuW8QTVJAAU7N8uQQC6Wa9TG+g==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", @@ -479,6 +503,15 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.2.tgz", + "integrity": "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", @@ -641,6 +674,26 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@eslint/js": { "version": "9.39.4", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", @@ -678,6 +731,43 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@eslint/plugin-kit/node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/@eslint/plugin-kit/node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/@eslint/plugin-kit/node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -747,96 +837,6 @@ "node": ">=8" } }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/@istanbuljs/schema": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", @@ -1526,19 +1526,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -1636,11 +1623,14 @@ } }, "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "dev": true, - "license": "Python-2.0" + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } }, "node_modules/babel-jest": { "version": "29.7.0", @@ -1776,9 +1766,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.10.10", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.10.tgz", - "integrity": "sha512-sUoJ3IMxx4AyRqO4MLeHlnGDkyXRoUG0/AI9fjK+vS72ekpV0yWVY7O0BVjmBcRtkNcsAO2QDZ4tdKKGoI6YaQ==", + "version": "2.10.11", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.11.tgz", + "integrity": "sha512-DAKrHphkJyiGuau/cFieRYhcTFeK/lBuD++C7cZ6KZHbMhBrisoi+EvhQ5RZrIfV5qwsW8kgQ07JIC+MDJRAhg==", "dev": true, "license": "Apache-2.0", "bin": { @@ -2128,6 +2118,18 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/dotenv": { + "version": "17.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.3.1.tgz", + "integrity": "sha512-IO8C/dzEb6O3F9/twg6ZLXz164a2fhTnEWb95H23Dm4OuN+92NmEAlTrupP9VW6Jm3sO26tQlqyvyi4CsnY9GA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.325", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.325.tgz", @@ -2249,21 +2251,22 @@ } }, "node_modules/eslint-plugin-jest": { - "version": "28.14.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-28.14.0.tgz", - "integrity": "sha512-P9s/qXSMTpRTerE2FQ0qJet2gKbcGyFTPAJipoKxmWqR6uuFqIqk8FuEfg5yBieOezVrEfAMZrEwJ6yEp+1MFQ==", + "version": "29.15.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-29.15.1.tgz", + "integrity": "sha512-6BjyErCQauz3zfJvzLw/kAez2lf4LEpbHLvWBfEcG4EI0ZiRSwjoH2uZulMouU8kRkBH+S0rhqn11IhTvxKgKw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" + "@typescript-eslint/utils": "^8.0.0" }, "engines": { - "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + "node": "^20.12.0 || ^22.0.0 || >=24.0.0" }, "peerDependencies": { - "@typescript-eslint/eslint-plugin": "^6.0.0 || ^7.0.0 || ^8.0.0", - "eslint": "^7.0.0 || ^8.0.0 || ^9.0.0", - "jest": "*" + "@typescript-eslint/eslint-plugin": "^8.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "jest": "*", + "typescript": ">=4.8.4 <7.0.0" }, "peerDependenciesMeta": { "@typescript-eslint/eslint-plugin": { @@ -2271,6 +2274,9 @@ }, "jest": { "optional": true + }, + "typescript": { + "optional": true } } }, @@ -2292,6 +2298,19 @@ } }, "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/eslint-visitor-keys": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", @@ -2304,57 +2323,187 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "node_modules/eslint/node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", "dev": true, - "license": "BSD-2-Clause", + "license": "MIT", "dependencies": { - "acorn": "^8.15.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">=10" }, "funding": { - "url": "https://opencollective.com/eslint" + "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "node_modules/eslint/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" }, "engines": { - "node": ">=4" + "node": ">=10.13.0" } }, - "node_modules/esquery": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", - "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "node_modules/eslint/node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", "dev": true, - "license": "BSD-3-Clause", + "license": "MIT", "dependencies": { - "estraverse": "^5.1.0" + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" }, "engines": { - "node": ">=0.10" + "node": ">= 0.8.0" } }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "node_modules/eslint/node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, - "license": "BSD-2-Clause", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint/node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/eslint/node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint/node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/eslint/node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree/node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", "dependencies": { "estraverse": "^5.2.0" }, @@ -2490,20 +2639,17 @@ } }, "node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, "license": "MIT", "dependencies": { - "locate-path": "^6.0.0", + "locate-path": "^5.0.0", "path-exists": "^4.0.0" }, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=8" } }, "node_modules/flat-cache": { @@ -2624,19 +2770,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -2724,6 +2857,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/import-fresh/node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/import-local": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", @@ -3531,13 +3674,14 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", "dev": true, "license": "MIT", "dependencies": { - "argparse": "^2.0.1" + "argparse": "^1.0.7", + "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" @@ -3570,6 +3714,19 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", @@ -3627,20 +3784,6 @@ "node": ">=6" } }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -3649,19 +3792,16 @@ "license": "MIT" }, "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, "license": "MIT", "dependencies": { - "p-locate": "^5.0.0" + "p-locate": "^4.1.0" }, "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=8" } }, "node_modules/lodash.merge": { @@ -3828,24 +3968,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3863,16 +3985,29 @@ } }, "node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, "license": "MIT", "dependencies": { - "p-limit": "^3.0.2" + "p-limit": "^2.2.0" }, "engines": { - "node": ">=10" + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -4000,72 +4135,6 @@ "node": ">=8" } }, - "node_modules/pkg-dir/node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pkg-dir/node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/pretty-format": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", @@ -4186,7 +4255,7 @@ "node": ">=8" } }, - "node_modules/resolve-cwd/node_modules/resolve-from": { + "node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", @@ -4196,16 +4265,6 @@ "node": ">=8" } }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/resolve.exports": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", @@ -4511,6 +4570,12 @@ "node": ">=8.0" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", @@ -4524,19 +4589,6 @@ "typescript": ">=4.8.4" } }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", diff --git a/package.json b/package.json index efdbc98..16d8208 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,13 @@ }, "devDependencies": { "@eslint/js": "^9.0.0", - "eslint": "^9.0.0", - "eslint-plugin-jest": "^28.0.0", + "eslint": "^9.39.4", + "eslint-plugin-jest": "^29.15.1", "jest": "^29.7.0", "typescript": "^5.9.3" + }, + "dependencies": { + "@anthropic-ai/sdk": "^0.80.0", + "dotenv": "^17.3.1" } } diff --git a/tsconfig.json b/tsconfig.json index 1f491d7..36a49ce 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,6 @@ "noImplicitAny": true, "strictNullChecks": false }, - "include": ["local-code-metrics.js"] + "include": ["local-code-metrics.js"], + "exclude": ["lib/", "node_modules"] }