Refactor SimHash banding logic into shared utility

plugins/core/src/graph/cluster.ts

@@ -19,39 +19,27 @@ export function detectContentClusters(
     // Banding Optimization (4 bands of 16 bits)
     // Note: For threshold > 3, this is a heuristic and may miss some pairs,
     // but it dramatically reduces the search space as requested.
-    const buckets: Map<number, Set<string>>[] = Array.from({ length: SimHash.BANDS }, () => new Map());
-
-    for (const node of nodes) {
-        const hash = BigInt(node.simhash!);
-        const bandValues = SimHash.getBands(hash);
-
-        bandValues.forEach((bandValue, b) => {
-            if (!buckets[b].has(bandValue)) {
-                buckets[b].set(bandValue, new Set());
-            }
-            buckets[b].get(bandValue)!.add(node.url);
-        });
-    }
+    const buckets = SimHash.groupIntoBands(nodes, node => BigInt(node.simhash!));
 
     const checkedPairs = new Set<string>();
 
     for (let b = 0; b < SimHash.BANDS; b++) {
         for (const bucket of buckets[b].values()) {
-            if (bucket.size < 2) continue;
-            const bucketNodes = Array.from(bucket);
+            if (bucket.length < 2) continue;
+            const bucketNodes = bucket;
             for (let i = 0; i < bucketNodes.length; i++) {
                 for (let j = i + 1; j < bucketNodes.length; j++) {
-                    const u1 = bucketNodes[i];
-                    const u2 = bucketNodes[j];
+                    const n1 = bucketNodes[i];
+                    const n2 = bucketNodes[j];
+                    const u1 = n1.url;
+                    const u2 = n2.url;
+
                     if (u1 === u2) continue;
 
                     const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`;
                     if (checkedPairs.has(pairKey)) continue;
                     checkedPairs.add(pairKey);
 
-                    const n1 = graph.nodes.get(u1)!;
-                    const n2 = graph.nodes.get(u2)!;
-
                     const dist = SimHash.hammingDistance(BigInt(n1.simhash!), BigInt(n2.simhash!));
                     if (dist <= threshold) {
                         if (!adjacency.has(u1)) adjacency.set(u1, new Set());

plugins/core/src/graph/duplicate.ts

@@ -133,22 +133,7 @@ function findNearDuplicates(candidates: GraphNode[], threshold: number, startId:
     const nearClusters: DuplicateCluster[] = [];
     let clusterCounter = startId;
 
-    const bandsMaps: Map<number, GraphNode[]>[] = Array.from({ length: SimHash.BANDS }, () => new Map());
-
-    for (const node of candidates) {
-        if (!node.simhash) continue;
-        const simhash = BigInt(node.simhash);
-        const bands = SimHash.getBands(simhash);
-
-        for (let i = 0; i < SimHash.BANDS; i++) {
-            let arr = bandsMaps[i].get(bands[i]);
-            if (!arr) {
-                arr = [];
-                bandsMaps[i].set(bands[i], arr);
-            }
-            arr.push(node);
-        }
-    }
+    const bandsMaps = SimHash.groupIntoBands(candidates, node => node.simhash ? BigInt(node.simhash) : undefined);
 
     // Use Union-Find to track connected components of near-duplicates
     const uf = new UnionFind<string>();

plugins/core/src/graph/simhash.ts

@@ -73,4 +73,28 @@ export class SimHash {
         }
         return distance;
     }
+
+    /**
+     * Groups items into SimHash bands/buckets.
+     */
+    static groupIntoBands<T>(items: Iterable<T>, hashExtractor: (item: T) => bigint | undefined): Map<number, T[]>[] {
+        const buckets: Map<number, T[]>[] = Array.from({ length: SimHash.BANDS }, () => new Map());
+
+        for (const item of items) {
+            const hash = hashExtractor(item);
+            if (hash === undefined) continue;
+
+            const bandValues = SimHash.getBands(hash);
+
+            bandValues.forEach((bandValue, b) => {
+                let bucket = buckets[b].get(bandValue);
+                if (!bucket) {
+                    bucket = [];
+                    buckets[b].set(bandValue, bucket);
+                }
+                bucket.push(item);
+            });
+        }
+        return buckets;
+    }
 }