From f3c8625b2b3e566eb9a19ff2da0971f9473d68e8 Mon Sep 17 00:00:00 2001 From: saurabhsharma2u <41580629+saurabhsharma2u@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:28:40 +0000 Subject: [PATCH] feat(core): improve cluster risk heuristic to use content similarity This commit replaces the simplified cluster risk heuristic with a content-based approach. It now parses the HTML of cluster nodes to extract Titles and H1s, calculating duplication ratios to determine risk levels more accurately. - Added `cheerio` import to `plugins/core/src/graph/cluster.ts`. - Implemented HTML parsing and duplication counting in `calculateClusterRisk`. - Defined risk levels based on duplication ratios (> 30% -> High, > 0% -> Medium). - Retained fallback to size-based heuristic if HTML is missing. - Added comprehensive tests in `plugins/core/tests/clustering_risk.test.ts`. --- plugins/core/src/graph/cluster.ts | 71 +++++++++++-- plugins/core/tests/clustering_risk.test.ts | 118 +++++++++++++++++++++ 2 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 plugins/core/tests/clustering_risk.test.ts diff --git a/plugins/core/src/graph/cluster.ts b/plugins/core/src/graph/cluster.ts index 89296dc..4880976 100644 --- a/plugins/core/src/graph/cluster.ts +++ b/plugins/core/src/graph/cluster.ts @@ -1,5 +1,6 @@ import { Graph, GraphNode, ClusterInfo } from './graph.js'; import { SimHash } from './simhash.js'; +import { load } from 'cheerio'; /** * Detects content clusters using 64-bit SimHash and Hamming Distance. @@ -154,14 +155,68 @@ function selectPrimaryUrl(urls: string[], graph: Graph): string { * Calculates cannibalization risk based on title and H1 similarity within the cluster. */ function calculateClusterRisk(nodes: GraphNode[]): 'low' | 'medium' | 'high' { - // Logic: Check if there's significant overlap in Titles or H1s among cluster members. - // This is a heuristic as requested. - // Simplified heuristic: risk is based on cluster density and size - // Large clusters of highly similar content are high risk. - - // Fallback to a safe categorization - if (nodes.length > 5) return 'high'; - if (nodes.length > 2) return 'medium'; + if (nodes.length <= 1) return 'low'; + + // Count title and H1 occurrences + const titleCounts = new Map(); + const h1Counts = new Map(); + let processedCount = 0; + + for (const node of nodes) { + if (!node.html) continue; + + try { + const $ = load(node.html); + const title = $('title').text().trim().toLowerCase(); + const h1 = $('h1').first().text().trim().toLowerCase(); + + if (title) { + titleCounts.set(title, (titleCounts.get(title) || 0) + 1); + } + if (h1) { + h1Counts.set(h1, (h1Counts.get(h1) || 0) + 1); + } + processedCount++; + } catch { + // Ignore parsing errors + } + } + + // If we couldn't parse enough content (e.g., no HTML stored), fallback to size-based heuristic + if (processedCount < nodes.length * 0.5) { + if (nodes.length > 5) return 'high'; + if (nodes.length > 2) return 'medium'; + return 'low'; + } + + // Calculate duplicate ratios + let duplicateTitleCount = 0; + let duplicateH1Count = 0; + + for (const count of titleCounts.values()) { + if (count > 1) duplicateTitleCount += count; + } + for (const count of h1Counts.values()) { + if (count > 1) duplicateH1Count += count; + } + + const titleDupeRatio = duplicateTitleCount / nodes.length; + const h1DupeRatio = duplicateH1Count / nodes.length; + + // Heuristic 1: High Risk + // Significant overlap in Titles OR H1s (e.g., > 30% of cluster members are duplicates) + if (titleDupeRatio > 0.3 || h1DupeRatio > 0.3) { + return 'high'; + } + + // Heuristic 2: Medium Risk + // Any overlap, or very large clusters (potential template issues or thin content) + if (titleDupeRatio > 0 || h1DupeRatio > 0 || nodes.length > 10) { + return 'medium'; + } + + // Heuristic 3: Low Risk + // Unique content and manageable cluster size return 'low'; } diff --git a/plugins/core/tests/clustering_risk.test.ts b/plugins/core/tests/clustering_risk.test.ts new file mode 100644 index 0000000..21262b1 --- /dev/null +++ b/plugins/core/tests/clustering_risk.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { Graph } from '../src/graph/graph.js'; +import { detectContentClusters } from '../src/graph/cluster.js'; + +describe('Cluster Risk Heuristic', () => { + let graph: Graph; + + beforeEach(() => { + graph = new Graph(); + }); + + it('should assign HIGH risk to clusters with identical titles', () => { + const html = 'Duplicate TitleContent'; + const h = 0b101010n.toString(); + + graph.addNode('https://example.com/p1', 0, 200); + graph.addNode('https://example.com/p2', 0, 200); + graph.addNode('https://example.com/p3', 0, 200); + + graph.updateNodeData('https://example.com/p1', { simhash: h, html }); + graph.updateNodeData('https://example.com/p2', { simhash: h, html }); + graph.updateNodeData('https://example.com/p3', { simhash: h, html }); + + const clusters = detectContentClusters(graph, 2, 2); + + expect(clusters.length).toBe(1); + expect(clusters[0].risk).toBe('high'); + }); + + it('should assign HIGH risk to clusters with identical H1s', () => { + const h = 0b101010n.toString(); + + graph.addNode('https://example.com/p1', 0, 200); + graph.addNode('https://example.com/p2', 0, 200); + graph.addNode('https://example.com/p3', 0, 200); + + // Different titles, same H1 + graph.updateNodeData('https://example.com/p1', { + simhash: h, + html: 'Page 1

Duplicate Header

' + }); + graph.updateNodeData('https://example.com/p2', { + simhash: h, + html: 'Page 2

Duplicate Header

' + }); + graph.updateNodeData('https://example.com/p3', { + simhash: h, + html: 'Page 3

Duplicate Header

' + }); + + const clusters = detectContentClusters(graph, 2, 2); + + expect(clusters.length).toBe(1); + expect(clusters[0].risk).toBe('high'); + }); + + it('should assign LOW risk to small clusters with unique titles and H1s', () => { + const h = 0b101010n.toString(); + + graph.addNode('https://example.com/p1', 0, 200); + graph.addNode('https://example.com/p2', 0, 200); + graph.addNode('https://example.com/p3', 0, 200); + + graph.updateNodeData('https://example.com/p1', { + simhash: h, + html: 'Page 1

Header 1

' + }); + graph.updateNodeData('https://example.com/p2', { + simhash: h, + html: 'Page 2

Header 2

' + }); + graph.updateNodeData('https://example.com/p3', { + simhash: h, + html: 'Page 3

Header 3

' + }); + + const clusters = detectContentClusters(graph, 2, 2); + + expect(clusters.length).toBe(1); + expect(clusters[0].risk).toBe('low'); + }); + + it('should assign MEDIUM risk to large clusters even with unique titles', () => { + const h = 0b101010n.toString(); + + // 12 nodes, all unique titles + for (let i = 0; i < 12; i++) { + const url = `https://example.com/p${i}`; + graph.addNode(url, 0, 200); + graph.updateNodeData(url, { + simhash: h, + html: `Page ${i}

Header ${i}

` + }); + } + + const clusters = detectContentClusters(graph, 2, 2); + + expect(clusters.length).toBe(1); + expect(clusters[0].risk).toBe('medium'); + }); + + it('should handle missing HTML gracefully', () => { + const h = 0b101010n.toString(); + + graph.addNode('https://example.com/p1', 0, 200); + graph.addNode('https://example.com/p2', 0, 200); + + // No HTML provided + graph.updateNodeData('https://example.com/p1', { simhash: h }); + graph.updateNodeData('https://example.com/p2', { simhash: h }); + + const clusters = detectContentClusters(graph, 2, 2); + + expect(clusters.length).toBe(1); + // Fallback to size based? 2 nodes -> low risk + expect(clusters[0].risk).toBe('low'); + }); +});