feat: add two-layer duplicate code detection to PR check (#42)

.beads/interactions.jsonl

@@ -22,3 +22,13 @@
 {"id":"int-67abf2c2","kind":"field_change","created_at":"2026-04-07T15:19:31.62249Z","actor":"Ken Judy","issue_id":"code-quality-metrics-38c","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
 {"id":"int-1177d69a","kind":"field_change","created_at":"2026-04-07T15:23:05.721325Z","actor":"Ken Judy","issue_id":"code-quality-metrics-moo","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
 {"id":"int-ac0d33c7","kind":"field_change","created_at":"2026-04-07T15:23:06.286044Z","actor":"Ken Judy","issue_id":"code-quality-metrics-cek","extra":{"field":"status","new_value":"closed","old_value":"in_progress"}}
+{"id":"int-938d7ec1","kind":"field_change","created_at":"2026-06-09T15:15:13.208542Z","actor":"Ken Judy","issue_id":"code-quality-metrics-een","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-0076bf03","kind":"field_change","created_at":"2026-06-09T15:30:02.111356Z","actor":"Ken Judy","issue_id":"code-quality-metrics-20o","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-38edcf94","kind":"field_change","created_at":"2026-06-12T14:25:49.983553Z","actor":"Ken Judy","issue_id":"code-quality-metrics-uhs","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-ba832b82","kind":"field_change","created_at":"2026-06-12T14:29:40.015721Z","actor":"Ken Judy","issue_id":"code-quality-metrics-790","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
+{"id":"int-b34a5661","kind":"field_change","created_at":"2026-06-12T15:27:45.761818Z","actor":"Ken Judy","issue_id":"code-quality-metrics-5o3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-1aadd2c0","kind":"field_change","created_at":"2026-06-12T15:34:45.826228Z","actor":"Ken Judy","issue_id":"code-quality-metrics-729","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-9340d4cf","kind":"field_change","created_at":"2026-06-12T15:48:14.57581Z","actor":"Ken Judy","issue_id":"code-quality-metrics-3pa","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-b4942b1f","kind":"field_change","created_at":"2026-06-12T15:52:30.533678Z","actor":"Ken Judy","issue_id":"code-quality-metrics-a28","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-4ff6dea3","kind":"field_change","created_at":"2026-06-12T16:00:15.448713Z","actor":"Ken Judy","issue_id":"code-quality-metrics-qsg","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
+{"id":"int-efeb5ce0","kind":"field_change","created_at":"2026-06-12T16:00:17.634271Z","actor":"Ken Judy","issue_id":"code-quality-metrics-aat","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}

.github/workflows/pr-metrics.yml

@@ -24,6 +24,8 @@ jobs:
             // lib/ modules are available after checkout — require() resolves from $GITHUB_WORKSPACE
             const { CONFIG } = require('./lib/config');
             const { scoreMessageQuality, classifyDoraArchetype, isTestFile } = require('./lib/metrics');
+            const { runDuplicateCheck, resolveModuleNeighbors } = require('./lib/duplicate');
+            const { getAnthropicClient, analyzeDuplicatesWithClaude } = require('./lib/claude');
 
             // ---------------------------------------------------------------
             // Fetch PR data
@@ -114,6 +116,16 @@ jobs:
             const prodChanges = prodAdditions + prodDeletions;
             const prodFilesCount = prodFiles.length;
 
+            const staticDuplicates = runDuplicateCheck(prodFiles.map(f => f.filename));
+            const anthropicClient = await getAnthropicClient();
+            const semanticFindings = anthropicClient
+              ? await analyzeDuplicatesWithClaude(
+                  anthropicClient,
+                  resolveModuleNeighbors(prodFiles.map(f => f.filename)),
+                  staticDuplicates
+                )
+              : [];
+
             const testAdditions = testFiles.reduce((sum, f) => sum + f.additions, 0);
             const testDeletions = testFiles.reduce((sum, f) => sum + f.deletions, 0);
             const testChanges = testAdditions + testDeletions;
@@ -304,6 +316,35 @@ jobs:
               `| AI Risk Signal | Net additions ratio (median) | ${medianAdditionsRatio.toFixed(2)} | <0.50 |`
             ].join('\n') : '';
 
+            const layerIndicator = anthropicClient
+              ? '_Layer 1 (static) and Layer 2 (semantic) ran._'
+              : '_Layer 1 (static) ran. Set ANTHROPIC_API_KEY to enable semantic analysis._';
+
+            const showDuplicationSection = staticDuplicates.length > 0 || semanticFindings.length > 0 || anthropicClient !== null;
+
+            const staticBlock = staticDuplicates.length > 0 ? [
+              `Found ${staticDuplicates.length} duplicate block(s) across the changed files:`,
+              ...staticDuplicates.slice(0, 10).map(d =>
+                `- \`${d.firstFile.name}:${d.firstFile.start}-${d.firstFile.end}\` duplicates \`${d.secondFile.name}:${d.secondFile.start}-${d.secondFile.end}\` (${d.lines} lines, ${d.tokens} tokens)`
+              ),
+              ...(staticDuplicates.length > 10 ? [`and ${staticDuplicates.length - 10} more...`] : [])
+            ].join('\n') : 'No static duplicates found.';
+
+            const semanticBlock = semanticFindings.length > 0 ? [
+              '### Semantic Similarity',
+              ...semanticFindings.map(f => `- \`${f.file1}\` / \`${f.file2}\`: ${f.similarity} (confidence: ${f.confidence})`)
+            ].join('\n') : '';
+
+            const duplicationSection = showDuplicationSection ? [
+              '## Duplicate Code',
+              '',
+              staticBlock,
+              '',
+              semanticBlock,
+              '',
+              layerIndicator
+            ].filter(s => s !== '').join('\n') : '';
+
             const commentParts = [
               '## PR Analysis',
               '',
@@ -350,6 +391,8 @@ jobs:
               '',
               doraSection,
               '',
+              duplicationSection,
+              '',
               '_Automated by Code Metrics Workflow_'
             ].filter(s => s !== '').join('\n');
 

CLAUDE.md

@@ -53,7 +53,7 @@ Three public components sharing pure-computation logic via `lib/`:
 
 2. **`.github/workflows/code-metrics.yml`**: Weekly GitHub Actions workflow. Uses the GitHub API to analyze feature branches from the past 30 days. Requires `lib/config.js`, `lib/statistics.js`, and `lib/metrics.js` via `require()`. Outputs a JSON artifact and creates a GitHub issue with the summary.
 
-3. **`.github/workflows/pr-metrics.yml`**: Per-PR GitHub Actions workflow. Requires `lib/config.js` and `lib/metrics.js` via `require()`. Posts a detailed comment on each PR with commit-by-commit analysis, test adequacy, and development pattern detection.
+3. **`.github/workflows/pr-metrics.yml`**: Per-PR GitHub Actions workflow. Requires `lib/config.js`, `lib/metrics.js`, `lib/duplicate.js`, and `lib/claude.js` via `require()`. Posts a detailed comment on each PR with commit-by-commit analysis, test adequacy, development pattern detection, and two-layer duplicate code detection (Layer 1 jscpd always-on; Layer 2 semantic via Claude when `ANTHROPIC_API_KEY` is set).
 
 ## Key Metrics and Thresholds
 
@@ -101,9 +101,34 @@ Thresholds are configured in the `CONFIG` object in `lib/config.js`, which is th
 | `AI_ANALYSIS_MAX_COMMITS` | 5 | Max commits sent to Claude per run |
 | `AI_DIFF_MAX_CHARS` | 4000 | Diff truncation limit for Claude API calls |
 | `AI_RISK_ADDITIONS_RATIO` | 3 | Additions/deletions multiplier for Claude pre-filter |
+| `DUPLICATE_MIN_LINES` | 5 | Minimum lines for jscpd to flag a duplicate block |
+| `DUPLICATE_MIN_TOKENS` | 50 | Minimum tokens for jscpd to flag a duplicate block |
+| `DUPLICATE_IGNORE_PATTERNS` | `[]` | Glob patterns for jscpd to ignore (e.g. generated files) |
+| `DUPLICATE_SCAN_PATHS` | `[]` | Paths jscpd scans; empty means all provided file paths |
 
 Test file detection uses patterns for JS, Python, Go, Java, and C#. Extend `TEST_FILE_PATTERNS` in `lib/config.js` — the change propagates automatically to all three components.
 
+### Duplicate Detection Tuning
+
+The defaults (`DUPLICATE_MIN_LINES: 5`, `DUPLICATE_MIN_TOKENS: 50`) are calibrated for JavaScript. Other languages typically need higher thresholds to suppress boilerplate false positives:
+
+```js
+// Java — longer method signatures and boilerplate
+DUPLICATE_MIN_LINES: 10,
+DUPLICATE_MIN_TOKENS: 100,
+DUPLICATE_IGNORE_PATTERNS: ['**/*Test.java', '**/generated/**'],
+
+// Python — decorators and docstrings inflate token counts
+DUPLICATE_MIN_LINES: 8,
+DUPLICATE_MIN_TOKENS: 80,
+DUPLICATE_IGNORE_PATTERNS: ['**/migrations/**', '**/__pycache__/**'],
+
+// Go — interface implementations repeat predictably
+DUPLICATE_MIN_LINES: 8,
+DUPLICATE_MIN_TOKENS: 75,
+DUPLICATE_IGNORE_PATTERNS: ['**/*_test.go', '**/vendor/**'],
+```
+
 ## Workflow Permissions
 
 The GitHub workflows require:

__tests__/claudeAnalysis.test.js

@@ -20,13 +20,15 @@ jest.mock('child_process');
 jest.mock('fs');
 
 const { execSync } = require('child_process');
+const fs = require('fs');
 const {
   getAnthropicClient,
   selectClaudeCommits,
   getCommitDiff,
   analyzeWithClaude,
   CONFIG,
 } = require('../local-code-metrics');
+const { analyzeDuplicatesWithClaude } = require('../lib/claude');
 
 beforeEach(() => {
   jest.clearAllMocks();
@@ -213,3 +215,58 @@ describe('selectClaudeCommits', () => {
     expect(result[0].sha).toBe('large');
   });
 });
+
+// ---------------------------------------------------------------------------
+// analyzeDuplicatesWithClaude
+// ---------------------------------------------------------------------------
+
+describe('analyzeDuplicatesWithClaude', () => {
+  const FIXTURE_FINDING = {
+    file1: 'src/lib/git.js',
+    file2: 'src/lib/metrics.js',
+    similarity: 'both parse line-delimited data with the same loop structure',
+    concern: 'copy-paste risk in parsing logic',
+    confidence: 'high'
+  };
+
+  function makeClient(responsePayload) {
+    const mockCreate = jest.fn().mockResolvedValue({
+      content: [{ type: 'text', text: JSON.stringify(responsePayload || [FIXTURE_FINDING]) }]
+    });
+    return { messages: { create: mockCreate } };
+  }
+
+  test('returns empty array when client is null', async () => {
+    const result = await analyzeDuplicatesWithClaude(null, ['src/lib/git.js'], []);
+    expect(result).toEqual([]);
+  });
+
+  test('calls messages.create with file contents and staticFindings count', async () => {
+    const client = makeClient();
+    fs.readFileSync.mockReturnValue('function parse(line) {}');
+    await analyzeDuplicatesWithClaude(client, ['src/lib/git.js'], [{ lines: 10 }]);
+    expect(client.messages.create).toHaveBeenCalled();
+    const call = client.messages.create.mock.calls[0][0];
+    expect(call.messages[0].content).toContain('src/lib/git.js');
+    expect(call.messages[0].content).toContain('1 static finding');
+  });
+
+  test('parses structured response into file1/file2/similarity/concern/confidence', async () => {
+    const client = makeClient();
+    fs.readFileSync.mockReturnValue('function parse(line) {}');
+    const result = await analyzeDuplicatesWithClaude(client, ['src/lib/git.js'], []);
+    expect(result).toHaveLength(1);
+    expect(result[0].file1).toBe('src/lib/git.js');
+    expect(result[0].confidence).toBe('high');
+  });
+
+  test('returns empty array when API response is malformed', async () => {
+    const client = makeClient('not-json-array');
+    client.messages.create.mockResolvedValue({
+      content: [{ type: 'text', text: 'not valid json at all {{{' }]
+    });
+    fs.readFileSync.mockReturnValue('function parse(line) {}');
+    const result = await analyzeDuplicatesWithClaude(client, ['src/lib/git.js'], []);
+    expect(result).toEqual([]);
+  });
+});

__tests__/config.test.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const { CONFIG } = require('../lib/config');
+
+describe('CONFIG duplicate detection defaults', () => {
+  test('CONFIG.DUPLICATE_MIN_LINES defaults to 5', () => {
+    expect(CONFIG.DUPLICATE_MIN_LINES).toBe(5);
+  });
+
+  test('CONFIG.DUPLICATE_MIN_TOKENS defaults to 50', () => {
+    expect(CONFIG.DUPLICATE_MIN_TOKENS).toBe(50);
+  });
+
+  test('CONFIG.DUPLICATE_IGNORE_PATTERNS is an array', () => {
+    expect(Array.isArray(CONFIG.DUPLICATE_IGNORE_PATTERNS)).toBe(true);
+  });
+
+  test('CONFIG.DUPLICATE_SCAN_PATHS is an array', () => {
+    expect(Array.isArray(CONFIG.DUPLICATE_SCAN_PATHS)).toBe(true);
+  });
+});

__tests__/duplicate.test.js

@@ -0,0 +1,100 @@
+'use strict';
+
+jest.mock('child_process');
+jest.mock('fs');
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const { runDuplicateCheck, resolveModuleNeighbors } = require('../lib/duplicate');
+
+const FIXTURE_DUPLICATE = {
+  firstFile:  { name: 'src/lib/git.js',     start: 10, end: 25 },
+  secondFile: { name: 'src/lib/metrics.js', start: 5,  end: 20 },
+  lines:  15,
+  tokens: 120
+};
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  execSync.mockReturnValue('');
+  fs.existsSync.mockReturnValue(true);
+  fs.readFileSync.mockReturnValue(JSON.stringify({ duplicates: [FIXTURE_DUPLICATE] }));
+});
+
+describe('runDuplicateCheck', () => {
+  test('returns empty array when filePaths is empty without calling jscpd', () => {
+    const result = runDuplicateCheck([]);
+    expect(result).toEqual([]);
+    expect(execSync).not.toHaveBeenCalled();
+  });
+
+  test('returns empty array when jscpd output file does not exist', () => {
+    fs.existsSync.mockReturnValue(false);
+    const result = runDuplicateCheck(['src/lib/git.js']);
+    expect(result).toEqual([]);
+  });
+
+  test('parses jscpd JSON and returns firstFile/secondFile/lines/tokens for each duplicate', () => {
+    const result = runDuplicateCheck(['src/lib/git.js', 'src/lib/metrics.js']);
+    expect(result).toHaveLength(1);
+    expect(result[0].firstFile).toEqual(FIXTURE_DUPLICATE.firstFile);
+    expect(result[0].secondFile).toEqual(FIXTURE_DUPLICATE.secondFile);
+    expect(result[0].lines).toBe(15);
+    expect(result[0].tokens).toBe(120);
+  });
+
+  test('returns empty array when jscpd exits non-zero', () => {
+    execSync.mockImplementation(() => { throw new Error('exit code 1'); });
+    const result = runDuplicateCheck(['src/lib/git.js']);
+    expect(result).toEqual([]);
+  });
+});
+
+describe('resolveModuleNeighbors', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    fs.existsSync.mockReturnValue(true);
+  });
+
+  test('returns only the input files when they have no local imports', () => {
+    fs.readFileSync.mockReturnValue('const x = 1;');
+    const input = ['/project/src/lib/git.js'];
+    const result = resolveModuleNeighbors(input);
+    expect(result).toContain('/project/src/lib/git.js');
+    expect(result).toHaveLength(1);
+  });
+
+  test('returns changed files plus resolved local imports', () => {
+    fs.readFileSync.mockReturnValue("const { CONFIG } = require('./config');");
+    const input = ['/project/src/lib/git.js'];
+    const result = resolveModuleNeighbors(input);
+    expect(result).toContain('/project/src/lib/git.js');
+    expect(result).toContain('/project/src/lib/config.js');
+    expect(result).toHaveLength(2);
+  });
+
+  test('skips import resolution for non-JS files and includes them as-is', () => {
+    const input = ['/project/.github/workflows/pr-metrics.yml'];
+    const result = resolveModuleNeighbors(input);
+    expect(result).toContain('/project/.github/workflows/pr-metrics.yml');
+    expect(fs.readFileSync).not.toHaveBeenCalled();
+  });
+
+  test('skips files that do not exist on disk', () => {
+    fs.existsSync.mockReturnValue(false);
+    const input = ['/project/src/deleted.js'];
+    const result = resolveModuleNeighbors(input);
+    expect(result).toEqual([]);
+  });
+
+  test('ignores external and node_modules imports', () => {
+    fs.readFileSync.mockReturnValue(
+      "const fs = require('fs');\nconst x = require('lodash');\nconst y = require('./local');"
+    );
+    const input = ['/project/src/lib/git.js'];
+    const result = resolveModuleNeighbors(input);
+    expect(result).toContain('/project/src/lib/local.js');
+    expect(result.some(p => p.includes('lodash'))).toBe(false);
+    expect(result.some(p => p.includes('node_modules'))).toBe(false);
+  });
+});

demo/temp-reset.sh

@@ -4,4 +4,8 @@ DEMO_SHA="267fa1d19840b5a7215d6aac59dacb42df223bea"
 git stash
 git branch -f "$DEMO_BRANCH" "$DEMO_SHA"
 git switch "$DEMO_BRANCH"
-echo "y" | "/Users/kenjudy/ObsidianVaults/PDCA Process/claude-skill/install-skill.sh" personal
+if [[ -d "${PDCA_SKILL_DIR}" ]]; then
+  echo "y" | "${PDCA_SKILL_DIR}/install-skill.sh" personal
+else
+  echo "Warning: PDCA_SKILL_DIR not found (${PDCA_SKILL_DIR}), skipping skill install."
+fi