From 90f629ea61c5b17a01e0a4feb5a5a5a24a51c272 Mon Sep 17 00:00:00 2001 From: saurabhsharma2u <41580629+saurabhsharma2u@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:24:46 +0000 Subject: [PATCH] Refactor Graph edge key storage to use JSON serialization - Introduced `Graph.getEdgeKey` and `Graph.parseEdgeKey` to encapsulate edge key generation logic. - Replaced `|` delimiter with `JSON.stringify` to ensure robustness against delimiter collisions in URLs. - Updated `Graph` methods `addEdge`, `getEdges`, and `fromJSON` to use the new key management methods. - Refactored `plugins/core/src/graph/duplicate.ts` to use the new `Graph` static methods for edge processing. - Updated `plugins/core/tests/duplicate.test.ts` to use `graph.addEdge` and `Graph.getEdgeKey`, removing fragile manual string concatenation. --- plugins/core/src/graph/duplicate.ts | 6 +++--- plugins/core/src/graph/graph.ts | 23 +++++++++++++++++++---- plugins/core/tests/duplicate.test.ts | 4 ++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/plugins/core/src/graph/duplicate.ts b/plugins/core/src/graph/duplicate.ts index 6c7fcfc..9301226 100644 --- a/plugins/core/src/graph/duplicate.ts +++ b/plugins/core/src/graph/duplicate.ts @@ -105,7 +105,7 @@ export function detectDuplicates(graph: Graph, options: DuplicateOptions = {}) { // Ensure n1 < n2 lexicographically to avoid duplicate pairs const [a, b] = n1.url < n2.url ? [n1, n2] : [n2, n1]; - const pairKey = `${a.url}|${b.url}`; + const pairKey = Graph.getEdgeKey(a.url, b.url); if (checkedPairs.has(pairKey)) continue; checkedPairs.add(pairKey); @@ -262,7 +262,7 @@ export function detectDuplicates(graph: Graph, options: DuplicateOptions = {}) { // Skip self-referential edges caused by repointing if (actualSource === actualTarget) continue; - const edgeKey = `${actualSource}|${actualTarget}`; + const edgeKey = Graph.getEdgeKey(actualSource, actualTarget); const existingWeight = updatedEdges.get(edgeKey) || 0; updatedEdges.set(edgeKey, Math.max(existingWeight, edge.weight)); // deduplicate } @@ -276,7 +276,7 @@ export function detectDuplicates(graph: Graph, options: DuplicateOptions = {}) { node.outLinks = 0; } for (const [edgeKey, _weight] of updatedEdges.entries()) { - const [src, tgt] = edgeKey.split('|'); + const { source: src, target: tgt } = Graph.parseEdgeKey(edgeKey); const sn = graph.nodes.get(src); const tn = graph.nodes.get(tgt); if (sn) sn.outLinks++; diff --git a/plugins/core/src/graph/graph.ts b/plugins/core/src/graph/graph.ts index 327e901..e8cdb16 100644 --- a/plugins/core/src/graph/graph.ts +++ b/plugins/core/src/graph/graph.ts @@ -60,7 +60,7 @@ export interface CrawlStats { export class Graph { nodes: Map = new Map(); - // Using string "source|target" to ensure uniqueness efficiently. Mapping to weight. + // Using JSON string of [source, target] to ensure uniqueness. Mapping to weight. edges: Map = new Map(); limitReached: boolean = false; sessionStats: CrawlStats = { @@ -73,6 +73,21 @@ export class Graph { duplicateClusters: { id: string; type: 'exact' | 'near' | 'template_heavy'; size: number; representative: string; severity: 'low' | 'medium' | 'high' }[] = []; contentClusters: ClusterInfo[] = []; + /** + * Generates a unique key for an edge. + */ + static getEdgeKey(source: string, target: string): string { + return JSON.stringify([source, target]); + } + + /** + * Parses an edge key back into source and target. + */ + static parseEdgeKey(key: string): { source: string; target: string } { + const [source, target] = JSON.parse(key); + return { source, target }; + } + /** * Adds a node to the graph if it doesn't exist. * If it exists, updates the status if the new status is non-zero (meaning we crawled it). @@ -113,7 +128,7 @@ export class Graph { const targetNode = this.nodes.get(target); if (sourceNode && targetNode) { - const edgeKey = `${source}|${target}`; + const edgeKey = Graph.getEdgeKey(source, target); if (!this.edges.has(edgeKey)) { this.edges.set(edgeKey, weight); sourceNode.outLinks++; @@ -134,7 +149,7 @@ export class Graph { getEdges(): GraphEdge[] { return Array.from(this.edges.entries()).map(([edge, weight]) => { - const [source, target] = edge.split('|'); + const { source, target } = Graph.parseEdgeKey(edge); return { source, target, weight }; }); } @@ -157,7 +172,7 @@ export class Graph { } if (json.edges) { for (const edge of json.edges) { - const key = `${edge.source}|${edge.target}`; + const key = Graph.getEdgeKey(edge.source, edge.target); graph.edges.set(key, edge.weight || 1.0); } } diff --git a/plugins/core/tests/duplicate.test.ts b/plugins/core/tests/duplicate.test.ts index 32e3026..cb83fb8 100644 --- a/plugins/core/tests/duplicate.test.ts +++ b/plugins/core/tests/duplicate.test.ts @@ -89,7 +89,7 @@ describe('Duplicate Detection', () => { graph.updateNodeData('https://example.com/b', { contentHash: 'h1' }); // Add edge pointing to B - graph.edges.set('https://example.com/source|https://example.com/b', 1); + graph.addEdge('https://example.com/source', 'https://example.com/b', 1); // Force A to be the representative by giving it higher inLinks manually, though it's determined dynamically graph.nodes.get('https://example.com/a')!.inLinks = 10; @@ -105,6 +105,6 @@ describe('Duplicate Detection', () => { expect(b.collapseInto).toBe('https://example.com/a'); // Check edge transfer - expect(graph.edges.has('https://example.com/source|https://example.com/a')).toBe(true); + expect(graph.edges.has(Graph.getEdgeKey('https://example.com/source', 'https://example.com/a'))).toBe(true); }); });