From 775907495f5a118513df88d85793dbea7ced8efb Mon Sep 17 00:00:00 2001 From: ChrisHegarty Date: Fri, 24 Jul 2026 17:50:03 +0100 Subject: [PATCH 1/5] Use svand_u8_x in SVE D1Q4 bulk scorer to avoid movprfx --- libs/native/libraries/build.gradle | 2 +- libs/simdvec/native/publish_vec_binaries.sh | 2 +- .../native/src/vec/c/aarch64/vec_bbq_2.cpp | 40 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/libs/native/libraries/build.gradle b/libs/native/libraries/build.gradle index be632194697e3..48fa735c8b4bf 100644 --- a/libs/native/libraries/build.gradle +++ b/libs/native/libraries/build.gradle @@ -38,7 +38,7 @@ configurations.consumable('nativeLibs') { } var zstdVersion = "1.5.7" -var vecVersion = "1.0.134" +var vecVersion = "1.0.135" var pqrsVersion = "0.2.0" repositories { diff --git a/libs/simdvec/native/publish_vec_binaries.sh b/libs/simdvec/native/publish_vec_binaries.sh index 03ab8a708c6c1..13a3be3e67ddf 100755 --- a/libs/simdvec/native/publish_vec_binaries.sh +++ b/libs/simdvec/native/publish_vec_binaries.sh @@ -17,7 +17,7 @@ set -euo pipefail -VERSION="1.0.134" +VERSION="1.0.135" LOCAL=false FORCE_UPLOAD=false diff --git a/libs/simdvec/native/src/vec/c/aarch64/vec_bbq_2.cpp b/libs/simdvec/native/src/vec/c/aarch64/vec_bbq_2.cpp index c6cf14613709f..6a8cc687b2dfb 100644 --- a/libs/simdvec/native/src/vec/c/aarch64/vec_bbq_2.cpp +++ b/libs/simdvec/native/src/vec/c/aarch64/vec_bbq_2.cpp @@ -21,7 +21,7 @@ static inline svuint64_t dot_bit_sv(const svbool_t pg, const svuint8_t a, const int8_t* b) { const svuint8_t q0 = svld1_u8(pg, (const uint8_t*)b); // reinterpret the u8 result as u64 so the count doesn't overflow - return svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, q0, a))); + return svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), q0, a))); } static inline int64_t dotd1q4_inner(const int8_t* a, const int8_t* query, const int32_t length) { @@ -202,25 +202,25 @@ static inline void dotd1q4_inner_bulk( const svuint8_t v2 = svld1_u8(pg, (const uint8_t*)(a2 + r)); const svuint8_t v3 = svld1_u8(pg, (const uint8_t*)(a3 + r)); - acc0_0 = svadd_u64_x(svptrue_b64(), acc0_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q0)))); - acc1_0 = svadd_u64_x(svptrue_b64(), acc1_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q1)))); - acc2_0 = svadd_u64_x(svptrue_b64(), acc2_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q2)))); - acc3_0 = svadd_u64_x(svptrue_b64(), acc3_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q3)))); - - acc0_1 = svadd_u64_x(svptrue_b64(), acc0_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q0)))); - acc1_1 = svadd_u64_x(svptrue_b64(), acc1_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q1)))); - acc2_1 = svadd_u64_x(svptrue_b64(), acc2_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q2)))); - acc3_1 = svadd_u64_x(svptrue_b64(), acc3_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q3)))); - - acc0_2 = svadd_u64_x(svptrue_b64(), acc0_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q0)))); - acc1_2 = svadd_u64_x(svptrue_b64(), acc1_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q1)))); - acc2_2 = svadd_u64_x(svptrue_b64(), acc2_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q2)))); - acc3_2 = svadd_u64_x(svptrue_b64(), acc3_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q3)))); - - acc0_3 = svadd_u64_x(svptrue_b64(), acc0_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q0)))); - acc1_3 = svadd_u64_x(svptrue_b64(), acc1_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q1)))); - acc2_3 = svadd_u64_x(svptrue_b64(), acc2_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q2)))); - acc3_3 = svadd_u64_x(svptrue_b64(), acc3_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q3)))); + acc0_0 = svadd_u64_x(svptrue_b64(), acc0_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q0)))); + acc1_0 = svadd_u64_x(svptrue_b64(), acc1_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q1)))); + acc2_0 = svadd_u64_x(svptrue_b64(), acc2_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q2)))); + acc3_0 = svadd_u64_x(svptrue_b64(), acc3_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q3)))); + + acc0_1 = svadd_u64_x(svptrue_b64(), acc0_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q0)))); + acc1_1 = svadd_u64_x(svptrue_b64(), acc1_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q1)))); + acc2_1 = svadd_u64_x(svptrue_b64(), acc2_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q2)))); + acc3_1 = svadd_u64_x(svptrue_b64(), acc3_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q3)))); + + acc0_2 = svadd_u64_x(svptrue_b64(), acc0_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q0)))); + acc1_2 = svadd_u64_x(svptrue_b64(), acc1_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q1)))); + acc2_2 = svadd_u64_x(svptrue_b64(), acc2_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q2)))); + acc3_2 = svadd_u64_x(svptrue_b64(), acc3_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q3)))); + + acc0_3 = svadd_u64_x(svptrue_b64(), acc0_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q0)))); + acc1_3 = svadd_u64_x(svptrue_b64(), acc1_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q1)))); + acc2_3 = svadd_u64_x(svptrue_b64(), acc2_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q2)))); + acc3_3 = svadd_u64_x(svptrue_b64(), acc3_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q3)))); r += svcntb(); } From 6c9abf180e622210d414916fb30072a65b2fef68 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:12:24 +0200 Subject: [PATCH 2/5] Update docs/changelog/155001.yaml --- docs/changelog/155001.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/155001.yaml diff --git a/docs/changelog/155001.yaml b/docs/changelog/155001.yaml new file mode 100644 index 0000000000000..b29d6292ba223 --- /dev/null +++ b/docs/changelog/155001.yaml @@ -0,0 +1,5 @@ +area: "Performance, Vector Search" +issues: [] +pr: 155001 +summary: Eliminate movprfx stalls in SVE BBQ D1Q4 bulk scorer (~14% throughput improvement) +type: enhancement From e2495ddf0ee2147c86bfe9d8ba06b922d8b9df37 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:23:32 +0100 Subject: [PATCH 3/5] Update build.gradle --- libs/native/libraries/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/native/libraries/build.gradle b/libs/native/libraries/build.gradle index 48fa735c8b4bf..40f3e940a0516 100644 --- a/libs/native/libraries/build.gradle +++ b/libs/native/libraries/build.gradle @@ -38,7 +38,7 @@ configurations.consumable('nativeLibs') { } var zstdVersion = "1.5.7" -var vecVersion = "1.0.135" +var vecVersion = "1.0.137" var pqrsVersion = "0.2.0" repositories { From 07216ae1594f1f04f43d5a1a71d972da812d4f22 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:23:50 +0100 Subject: [PATCH 4/5] Update publish_vec_binaries.sh --- libs/simdvec/native/publish_vec_binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/simdvec/native/publish_vec_binaries.sh b/libs/simdvec/native/publish_vec_binaries.sh index 13a3be3e67ddf..007e45e441801 100755 --- a/libs/simdvec/native/publish_vec_binaries.sh +++ b/libs/simdvec/native/publish_vec_binaries.sh @@ -17,7 +17,7 @@ set -euo pipefail -VERSION="1.0.135" +VERSION="1.0.137" LOCAL=false FORCE_UPLOAD=false From 3f290f4ee99d82aa4b6b29f1c1407e2b732ea2fb Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:45:45 +0100 Subject: [PATCH 5/5] Update 155001.yaml --- docs/changelog/155001.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/155001.yaml b/docs/changelog/155001.yaml index b29d6292ba223..a8920a7ee4fbb 100644 --- a/docs/changelog/155001.yaml +++ b/docs/changelog/155001.yaml @@ -1,4 +1,4 @@ -area: "Performance, Vector Search" +area: "Vector Search" issues: [] pr: 155001 summary: Eliminate movprfx stalls in SVE BBQ D1Q4 bulk scorer (~14% throughput improvement)