diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index f03aa38c4980..245162d24d52 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -85,6 +85,11 @@ Optimizations Bug Fixes --------------------- +* GITHUB#16251: Fix UpdateGraphsUtils#computeJoinSet to not request more coverage for a node than + the node's degree. Previously nodes with a degree of 1 were always forced into the join set, + which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs. + (Mayya Sharipova) + * GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward) * GITHUB#16378: Accumulate join Total/Avg scores in double precision in diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java index ce103952d507..4a551a38f3f2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java +++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java @@ -49,7 +49,7 @@ public static IntHashSet computeJoinSet(HnswGraph graph) throws IOException { for (int v = 0; v < size; v++) { graph.seek(0, v); int degree = graph.neighborCount(); - k = degree < 9 ? 2 : Math.ceilDiv(degree, 4); + k = coverage(degree); gExit += k; int gain = k + degree; heap.push(encode(gain, v)); @@ -67,7 +67,7 @@ public static IntHashSet computeJoinSet(HnswGraph graph) throws IOException { for (int u = graph.nextNeighbor(); u != NO_MORE_DOCS; u = graph.nextNeighbor()) { ns[i++] = u; } - k = degree < 9 ? 2 : Math.ceilDiv(degree, 4); + k = coverage(degree); if (stale[v]) { // if stale, recalculate gain int newGain = Math.max(0, k - counts[v]); for (int u : ns) { @@ -101,6 +101,22 @@ public static IntHashSet computeJoinSet(HnswGraph graph) throws IOException { return j; } + /** + * How many times a node needs to be covered by the nodes of the join set before we consider it + * covered. + * + *

A node's neighbours are the only nodes that can cover it, so a node can never be covered + * more than {@code degree} times. Requesting a bigger coverage than that would make the coverage + * unsatisfiable, and the node would be forced to join the join set itself. This happens on + * degenerate graphs (e.g. hub-and-spoke shapes produced by highly duplicated or low dimensional + * vectors) where most of the nodes have a degree of 1: without clamping, every leaf ends up in + * the join set, and the join set degenerates to the whole graph. + */ + private static int coverage(int degree) { + int k = degree < 9 ? 2 : Math.ceilDiv(degree, 4); + return Math.min(k, degree); + } + private static long encode(int value1, int value2) { return (((long) -value1) << 32) | (value2 & 0xFFFFFFFFL); }