From 21c198969764b19a188921202f725c97a10ff238 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 30 Jul 2026 09:31:17 -0400 Subject: [PATCH 1/2] Fix join set degeneration on low-degree nodes UpdateGraphsUtils#computeJoinSet asked for a coverage of 2 for every node with a degree below 9, including nodes with a degree of 1. Such a node can be covered at most once (only its neighbours can cover it), so the coverage was unsatisfiable and the node always ended up in the join set itself. On hub-and-spoke shaped graphs, which HNSW produces for 1-dimensional or heavily duplicated vectors, almost all nodes have a degree of 1, so the join set degenerated to the whole graph and merging lost all its savings. Clamp the requested coverage by the node's degree; graphs where all nodes have a degree of 2 or more are unaffected. Fixes #16251 --- lucene/CHANGES.txt | 5 +++++ .../lucene/util/hnsw/UpdateGraphsUtils.java | 20 +++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 44924919b4fb..3aa47eaf6a76 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -193,6 +193,11 @@ Optimizations Bug Fixes --------------------- +* GITHUB#16251: Fix UpdateGraphsUtils#computeJoinSet to not request more coverage for a node than + the node's degree. Previously nodes with a degree of 1 were always forced into the join set, + which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs. + (Mayya Sharipova) + * GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero when all values are identical. (Mike Sokolov) diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java index ce103952d507..4a551a38f3f2 100644 --- a/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java +++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java @@ -49,7 +49,7 @@ public static IntHashSet computeJoinSet(HnswGraph graph) throws IOException { for (int v = 0; v < size; v++) { graph.seek(0, v); int degree = graph.neighborCount(); - k = degree < 9 ? 2 : Math.ceilDiv(degree, 4); + k = coverage(degree); gExit += k; int gain = k + degree; heap.push(encode(gain, v)); @@ -67,7 +67,7 @@ public static IntHashSet computeJoinSet(HnswGraph graph) throws IOException { for (int u = graph.nextNeighbor(); u != NO_MORE_DOCS; u = graph.nextNeighbor()) { ns[i++] = u; } - k = degree < 9 ? 2 : Math.ceilDiv(degree, 4); + k = coverage(degree); if (stale[v]) { // if stale, recalculate gain int newGain = Math.max(0, k - counts[v]); for (int u : ns) { @@ -101,6 +101,22 @@ public static IntHashSet computeJoinSet(HnswGraph graph) throws IOException { return j; } + /** + * How many times a node needs to be covered by the nodes of the join set before we consider it + * covered. + * + *

A node's neighbours are the only nodes that can cover it, so a node can never be covered + * more than {@code degree} times. Requesting a bigger coverage than that would make the coverage + * unsatisfiable, and the node would be forced to join the join set itself. This happens on + * degenerate graphs (e.g. hub-and-spoke shapes produced by highly duplicated or low dimensional + * vectors) where most of the nodes have a degree of 1: without clamping, every leaf ends up in + * the join set, and the join set degenerates to the whole graph. + */ + private static int coverage(int degree) { + int k = degree < 9 ? 2 : Math.ceilDiv(degree, 4); + return Math.min(k, degree); + } + private static long encode(int value1, int value2) { return (((long) -value1) << 32) | (value2 & 0xFFFFFFFFL); } From 054057420ee2d6cdb9565eb29f8fa06755f312fd Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 30 Jul 2026 10:55:51 -0400 Subject: [PATCH 2/2] Move CHANGES entry to 10.6.0 for backport The fix is going to be backported to branch_10x, so it will ship in 10.6.0 rather than 11.0.0. --- lucene/CHANGES.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 3aa47eaf6a76..90d5a0393253 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -193,11 +193,6 @@ Optimizations Bug Fixes --------------------- -* GITHUB#16251: Fix UpdateGraphsUtils#computeJoinSet to not request more coverage for a node than - the node's degree. Previously nodes with a degree of 1 were always forced into the join set, - which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs. - (Mayya Sharipova) - * GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero when all values are identical. (Mike Sokolov) @@ -403,6 +398,11 @@ Optimizations Bug Fixes --------------------- +* GITHUB#16251: Fix UpdateGraphsUtils#computeJoinSet to not request more coverage for a node than + the node's degree. Previously nodes with a degree of 1 were always forced into the join set, + which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs. + (Mayya Sharipova) + * GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward) * GITHUB#16378: Accumulate join Total/Avg scores in double precision in