diff --git a/plugins/core/src/graph/cluster.ts b/plugins/core/src/graph/cluster.ts index 229c9d8..c7ead9e 100644 --- a/plugins/core/src/graph/cluster.ts +++ b/plugins/core/src/graph/cluster.ts @@ -19,39 +19,27 @@ export function detectContentClusters( // Banding Optimization (4 bands of 16 bits) // Note: For threshold > 3, this is a heuristic and may miss some pairs, // but it dramatically reduces the search space as requested. - const buckets: Map>[] = Array.from({ length: SimHash.BANDS }, () => new Map()); - - for (const node of nodes) { - const hash = BigInt(node.simhash!); - const bandValues = SimHash.getBands(hash); - - bandValues.forEach((bandValue, b) => { - if (!buckets[b].has(bandValue)) { - buckets[b].set(bandValue, new Set()); - } - buckets[b].get(bandValue)!.add(node.url); - }); - } + const buckets = SimHash.groupIntoBands(nodes, node => BigInt(node.simhash!)); const checkedPairs = new Set(); for (let b = 0; b < SimHash.BANDS; b++) { for (const bucket of buckets[b].values()) { - if (bucket.size < 2) continue; - const bucketNodes = Array.from(bucket); + if (bucket.length < 2) continue; + const bucketNodes = bucket; for (let i = 0; i < bucketNodes.length; i++) { for (let j = i + 1; j < bucketNodes.length; j++) { - const u1 = bucketNodes[i]; - const u2 = bucketNodes[j]; + const n1 = bucketNodes[i]; + const n2 = bucketNodes[j]; + const u1 = n1.url; + const u2 = n2.url; + if (u1 === u2) continue; const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`; if (checkedPairs.has(pairKey)) continue; checkedPairs.add(pairKey); - const n1 = graph.nodes.get(u1)!; - const n2 = graph.nodes.get(u2)!; - const dist = SimHash.hammingDistance(BigInt(n1.simhash!), BigInt(n2.simhash!)); if (dist <= threshold) { if (!adjacency.has(u1)) adjacency.set(u1, new Set()); diff --git a/plugins/core/src/graph/duplicate.ts b/plugins/core/src/graph/duplicate.ts index 29abb9a..47e39ab 100644 --- a/plugins/core/src/graph/duplicate.ts +++ b/plugins/core/src/graph/duplicate.ts @@ -133,22 +133,7 @@ function findNearDuplicates(candidates: GraphNode[], threshold: number, startId: const nearClusters: DuplicateCluster[] = []; let clusterCounter = startId; - const bandsMaps: Map[] = Array.from({ length: SimHash.BANDS }, () => new Map()); - - for (const node of candidates) { - if (!node.simhash) continue; - const simhash = BigInt(node.simhash); - const bands = SimHash.getBands(simhash); - - for (let i = 0; i < SimHash.BANDS; i++) { - let arr = bandsMaps[i].get(bands[i]); - if (!arr) { - arr = []; - bandsMaps[i].set(bands[i], arr); - } - arr.push(node); - } - } + const bandsMaps = SimHash.groupIntoBands(candidates, node => node.simhash ? BigInt(node.simhash) : undefined); // Use Union-Find to track connected components of near-duplicates const uf = new UnionFind(); diff --git a/plugins/core/src/graph/simhash.ts b/plugins/core/src/graph/simhash.ts index 2de9a39..c8a5508 100644 --- a/plugins/core/src/graph/simhash.ts +++ b/plugins/core/src/graph/simhash.ts @@ -73,4 +73,28 @@ export class SimHash { } return distance; } + + /** + * Groups items into SimHash bands/buckets. + */ + static groupIntoBands(items: Iterable, hashExtractor: (item: T) => bigint | undefined): Map[] { + const buckets: Map[] = Array.from({ length: SimHash.BANDS }, () => new Map()); + + for (const item of items) { + const hash = hashExtractor(item); + if (hash === undefined) continue; + + const bandValues = SimHash.getBands(hash); + + bandValues.forEach((bandValue, b) => { + let bucket = buckets[b].get(bandValue); + if (!bucket) { + bucket = []; + buckets[b].set(bandValue, bucket); + } + bucket.push(item); + }); + } + return buckets; + } }