diff --git a/plugins/core/src/graph/cluster.ts b/plugins/core/src/graph/cluster.ts index 89296dc..55cb633 100644 --- a/plugins/core/src/graph/cluster.ts +++ b/plugins/core/src/graph/cluster.ts @@ -18,24 +18,23 @@ export function detectContentClusters( // Banding Optimization (4 bands of 16 bits) // Note: For threshold > 3, this is a heuristic and may miss some pairs, // but it dramatically reduces the search space as requested. - const bands = 4; - const bandWidth = 16; - const buckets: Map>[] = Array.from({ length: bands }, () => new Map()); + const buckets: Map>[] = Array.from({ length: SimHash.BANDS }, () => new Map()); for (const node of nodes) { const hash = BigInt(node.simhash!); - for (let b = 0; b < bands; b++) { - const bandValue = Number((hash >> BigInt(b * bandWidth)) & 0xFFFFn); + const bandValues = SimHash.getBands(hash); + + bandValues.forEach((bandValue, b) => { if (!buckets[b].has(bandValue)) { buckets[b].set(bandValue, new Set()); } buckets[b].get(bandValue)!.add(node.url); - } + }); } const checkedPairs = new Set(); - for (let b = 0; b < bands; b++) { + for (let b = 0; b < SimHash.BANDS; b++) { for (const bucket of buckets[b].values()) { if (bucket.size < 2) continue; const bucketNodes = Array.from(bucket); diff --git a/plugins/core/src/graph/duplicate.ts b/plugins/core/src/graph/duplicate.ts index 49103dc..29abb9a 100644 --- a/plugins/core/src/graph/duplicate.ts +++ b/plugins/core/src/graph/duplicate.ts @@ -133,24 +133,14 @@ function findNearDuplicates(candidates: GraphNode[], threshold: number, startId: const nearClusters: DuplicateCluster[] = []; let clusterCounter = startId; - const bandsMaps = [ - new Map(), - new Map(), - new Map(), - new Map() - ]; + const bandsMaps: Map[] = Array.from({ length: SimHash.BANDS }, () => new Map()); for (const node of candidates) { if (!node.simhash) continue; const simhash = BigInt(node.simhash); + const bands = SimHash.getBands(simhash); - const b0 = Number(simhash & 0xFFFFn); - const b1 = Number((simhash >> 16n) & 0xFFFFn); - const b2 = Number((simhash >> 32n) & 0xFFFFn); - const b3 = Number((simhash >> 48n) & 0xFFFFn); - - const bands = [b0, b1, b2, b3]; - for (let i = 0; i < 4; i++) { + for (let i = 0; i < SimHash.BANDS; i++) { let arr = bandsMaps[i].get(bands[i]); if (!arr) { arr = []; @@ -165,7 +155,7 @@ function findNearDuplicates(candidates: GraphNode[], threshold: number, startId: const involvedNodes = new Set(); const checkedPairs = new Set(); - for (let i = 0; i < 4; i++) { + for (let i = 0; i < SimHash.BANDS; i++) { for (const [_bandVal, bucketNodes] of bandsMaps[i].entries()) { if (bucketNodes.length < 2) continue; diff --git a/plugins/core/src/graph/simhash.ts b/plugins/core/src/graph/simhash.ts index 31d01f5..2de9a39 100644 --- a/plugins/core/src/graph/simhash.ts +++ b/plugins/core/src/graph/simhash.ts @@ -2,6 +2,8 @@ export class SimHash { private static FNV_PRIME = 1099511628211n; private static FNV_OFFSET_BASIS = 14695981039346656037n; private static MAX_UINT64 = 0xffffffffffffffffn; + public static readonly BANDS = 4; + public static readonly BAND_WIDTH = 16; /** * Generates a 64-bit FNV-1a hash for a given string token. @@ -45,6 +47,19 @@ export class SimHash { return simhash; } + /** + * Splits a 64-bit SimHash into 4 bands of 16 bits. + */ + static getBands(simhash: bigint): number[] { + const bands: number[] = []; + for (let i = 0; i < SimHash.BANDS; i++) { + // Extract 16-bit chunks + const chunk = Number((simhash >> BigInt(i * SimHash.BAND_WIDTH)) & 0xFFFFn); + bands.push(chunk); + } + return bands; + } + /** * Computes the Hamming distance between two 64-bit hashes. */