Refactor duplicated SimHash banding logic

plugins/core/src/graph/cluster.ts

@@ -18,24 +18,23 @@ export function detectContentClusters(
     // Banding Optimization (4 bands of 16 bits)
     // Note: For threshold > 3, this is a heuristic and may miss some pairs,
     // but it dramatically reduces the search space as requested.
-    const bands = 4;
-    const bandWidth = 16;
-    const buckets: Map<number, Set<string>>[] = Array.from({ length: bands }, () => new Map());
+    const buckets: Map<number, Set<string>>[] = Array.from({ length: SimHash.BANDS }, () => new Map());
 
     for (const node of nodes) {
         const hash = BigInt(node.simhash!);
-        for (let b = 0; b < bands; b++) {
-            const bandValue = Number((hash >> BigInt(b * bandWidth)) & 0xFFFFn);
+        const bandValues = SimHash.getBands(hash);
+
+        bandValues.forEach((bandValue, b) => {
             if (!buckets[b].has(bandValue)) {
                 buckets[b].set(bandValue, new Set());
             }
             buckets[b].get(bandValue)!.add(node.url);
-        }
+        });
     }
 
     const checkedPairs = new Set<string>();
 
-    for (let b = 0; b < bands; b++) {
+    for (let b = 0; b < SimHash.BANDS; b++) {
         for (const bucket of buckets[b].values()) {
             if (bucket.size < 2) continue;
             const bucketNodes = Array.from(bucket);

plugins/core/src/graph/duplicate.ts

@@ -133,24 +133,14 @@ function findNearDuplicates(candidates: GraphNode[], threshold: number, startId:
     const nearClusters: DuplicateCluster[] = [];
     let clusterCounter = startId;
 
-    const bandsMaps = [
-        new Map<number, GraphNode[]>(),
-        new Map<number, GraphNode[]>(),
-        new Map<number, GraphNode[]>(),
-        new Map<number, GraphNode[]>()
-    ];
+    const bandsMaps: Map<number, GraphNode[]>[] = Array.from({ length: SimHash.BANDS }, () => new Map());
 
     for (const node of candidates) {
         if (!node.simhash) continue;
         const simhash = BigInt(node.simhash);
+        const bands = SimHash.getBands(simhash);
 
-        const b0 = Number(simhash & 0xFFFFn);
-        const b1 = Number((simhash >> 16n) & 0xFFFFn);
-        const b2 = Number((simhash >> 32n) & 0xFFFFn);
-        const b3 = Number((simhash >> 48n) & 0xFFFFn);
-
-        const bands = [b0, b1, b2, b3];
-        for (let i = 0; i < 4; i++) {
+        for (let i = 0; i < SimHash.BANDS; i++) {
             let arr = bandsMaps[i].get(bands[i]);
             if (!arr) {
                 arr = [];
@@ -165,7 +155,7 @@ function findNearDuplicates(candidates: GraphNode[], threshold: number, startId:
     const involvedNodes = new Set<GraphNode>();
     const checkedPairs = new Set<string>();
 
-    for (let i = 0; i < 4; i++) {
+    for (let i = 0; i < SimHash.BANDS; i++) {
         for (const [_bandVal, bucketNodes] of bandsMaps[i].entries()) {
             if (bucketNodes.length < 2) continue;
 

plugins/core/src/graph/simhash.ts

@@ -2,6 +2,8 @@ export class SimHash {
     private static FNV_PRIME = 1099511628211n;
     private static FNV_OFFSET_BASIS = 14695981039346656037n;
     private static MAX_UINT64 = 0xffffffffffffffffn;
+    public static readonly BANDS = 4;
+    public static readonly BAND_WIDTH = 16;
 
     /**
      * Generates a 64-bit FNV-1a hash for a given string token.
@@ -45,6 +47,19 @@ export class SimHash {
         return simhash;
     }
 
+    /**
+     * Splits a 64-bit SimHash into 4 bands of 16 bits.
+     */
+    static getBands(simhash: bigint): number[] {
+        const bands: number[] = [];
+        for (let i = 0; i < SimHash.BANDS; i++) {
+            // Extract 16-bit chunks
+            const chunk = Number((simhash >> BigInt(i * SimHash.BAND_WIDTH)) & 0xFFFFn);
+            bands.push(chunk);
+        }
+        return bands;
+    }
+
     /**
      * Computes the Hamming distance between two 64-bit hashes.
      */