Crawlith · saurabhsharma2u · Feb 25, 2026 · Feb 25, 2026
diff --git a/plugins/core/src/graph/cluster.ts b/plugins/core/src/graph/cluster.ts
@@ -1,5 +1,6 @@
 import { Graph, GraphNode, ClusterInfo } from './graph.js';
 import { SimHash } from './simhash.js';
+import { load } from 'cheerio';
 
 /**
  * Detects content clusters using 64-bit SimHash and Hamming Distance.
@@ -154,14 +155,68 @@ function selectPrimaryUrl(urls: string[], graph: Graph): string {
  * Calculates cannibalization risk based on title and H1 similarity within the cluster.
  */
 function calculateClusterRisk(nodes: GraphNode[]): 'low' | 'medium' | 'high' {
-    // Logic: Check if there's significant overlap in Titles or H1s among cluster members.
-    // This is a heuristic as requested.
-    // Simplified heuristic: risk is based on cluster density and size
-    // Large clusters of highly similar content are high risk.
-
-    // Fallback to a safe categorization
-    if (nodes.length > 5) return 'high';
-    if (nodes.length > 2) return 'medium';
+    if (nodes.length <= 1) return 'low';
+
+    // Count title and H1 occurrences
+    const titleCounts = new Map<string, number>();
+    const h1Counts = new Map<string, number>();
+    let processedCount = 0;
+
+    for (const node of nodes) {
+        if (!node.html) continue;
+
+        try {
+            const $ = load(node.html);
+            const title = $('title').text().trim().toLowerCase();
+            const h1 = $('h1').first().text().trim().toLowerCase();
+
+            if (title) {
+                titleCounts.set(title, (titleCounts.get(title) || 0) + 1);
+            }
+            if (h1) {
+                h1Counts.set(h1, (h1Counts.get(h1) || 0) + 1);
+            }
+            processedCount++;
+        } catch {
+            // Ignore parsing errors
+        }
+    }
+
+    // If we couldn't parse enough content (e.g., no HTML stored), fallback to size-based heuristic
+    if (processedCount < nodes.length * 0.5) {
+        if (nodes.length > 5) return 'high';
+        if (nodes.length > 2) return 'medium';
+        return 'low';
+    }
+
+    // Calculate duplicate ratios
+    let duplicateTitleCount = 0;
+    let duplicateH1Count = 0;
+
+    for (const count of titleCounts.values()) {
+        if (count > 1) duplicateTitleCount += count;
+    }
+    for (const count of h1Counts.values()) {
+        if (count > 1) duplicateH1Count += count;
+    }
+
+    const titleDupeRatio = duplicateTitleCount / nodes.length;
+    const h1DupeRatio = duplicateH1Count / nodes.length;
+
+    // Heuristic 1: High Risk
+    // Significant overlap in Titles OR H1s (e.g., > 30% of cluster members are duplicates)
+    if (titleDupeRatio > 0.3 || h1DupeRatio > 0.3) {
+        return 'high';
+    }
+
+    // Heuristic 2: Medium Risk
+    // Any overlap, or very large clusters (potential template issues or thin content)
+    if (titleDupeRatio > 0 || h1DupeRatio > 0 || nodes.length > 10) {
+        return 'medium';
+    }
+
+    // Heuristic 3: Low Risk
+    // Unique content and manageable cluster size
     return 'low';
 }
 

diff --git a/plugins/core/tests/clustering_risk.test.ts b/plugins/core/tests/clustering_risk.test.ts
@@ -0,0 +1,118 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { Graph } from '../src/graph/graph.js';
+import { detectContentClusters } from '../src/graph/cluster.js';
+
+describe('Cluster Risk Heuristic', () => {
+    let graph: Graph;
+
+    beforeEach(() => {
+        graph = new Graph();
+    });
+
+    it('should assign HIGH risk to clusters with identical titles', () => {
+        const html = '<html><head><title>Duplicate Title</title></head><body>Content</body></html>';
+        const h = 0b101010n.toString();
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+        graph.addNode('https://example.com/p3', 0, 200);
+
+        graph.updateNodeData('https://example.com/p1', { simhash: h, html });
+        graph.updateNodeData('https://example.com/p2', { simhash: h, html });
+        graph.updateNodeData('https://example.com/p3', { simhash: h, html });
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(1);
+        expect(clusters[0].risk).toBe('high');
+    });
+
+    it('should assign HIGH risk to clusters with identical H1s', () => {
+        const h = 0b101010n.toString();
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+        graph.addNode('https://example.com/p3', 0, 200);
+
+        // Different titles, same H1
+        graph.updateNodeData('https://example.com/p1', {
+            simhash: h,
+            html: '<html><head><title>Page 1</title></head><body><h1>Duplicate Header</h1></body></html>'
+        });
+        graph.updateNodeData('https://example.com/p2', {
+            simhash: h,
+            html: '<html><head><title>Page 2</title></head><body><h1>Duplicate Header</h1></body></html>'
+        });
+        graph.updateNodeData('https://example.com/p3', {
+            simhash: h,
+            html: '<html><head><title>Page 3</title></head><body><h1>Duplicate Header</h1></body></html>'
+        });
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(1);
+        expect(clusters[0].risk).toBe('high');
+    });
+
+    it('should assign LOW risk to small clusters with unique titles and H1s', () => {
+        const h = 0b101010n.toString();
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+        graph.addNode('https://example.com/p3', 0, 200);
+
+        graph.updateNodeData('https://example.com/p1', {
+            simhash: h,
+            html: '<html><head><title>Page 1</title></head><body><h1>Header 1</h1></body></html>'
+        });
+        graph.updateNodeData('https://example.com/p2', {
+            simhash: h,
+            html: '<html><head><title>Page 2</title></head><body><h1>Header 2</h1></body></html>'
+        });
+        graph.updateNodeData('https://example.com/p3', {
+            simhash: h,
+            html: '<html><head><title>Page 3</title></head><body><h1>Header 3</h1></body></html>'
+        });
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(1);
+        expect(clusters[0].risk).toBe('low');
+    });
+
+    it('should assign MEDIUM risk to large clusters even with unique titles', () => {
+        const h = 0b101010n.toString();
+
+        // 12 nodes, all unique titles
+        for (let i = 0; i < 12; i++) {
+            const url = `https://example.com/p${i}`;
+            graph.addNode(url, 0, 200);
+            graph.updateNodeData(url, {
+                simhash: h,
+                html: `<html><head><title>Page ${i}</title></head><body><h1>Header ${i}</h1></body></html>`
+            });
+        }
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(1);
+        expect(clusters[0].risk).toBe('medium');
+    });
+
+    it('should handle missing HTML gracefully', () => {
+         const h = 0b101010n.toString();
+
+        graph.addNode('https://example.com/p1', 0, 200);
+        graph.addNode('https://example.com/p2', 0, 200);
+
+        // No HTML provided
+        graph.updateNodeData('https://example.com/p1', { simhash: h });
+        graph.updateNodeData('https://example.com/p2', { simhash: h });
+
+        const clusters = detectContentClusters(graph, 2, 2);
+
+        expect(clusters.length).toBe(1);
+        // Fallback to size based? 2 nodes -> low risk
+        expect(clusters[0].risk).toBe('low');
+    });
+});