Skip to content

Eliminate movprfx stalls in SVE BBQ D1Q4 bulk scorer (~14% throughput improvement) #155001

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChrisHegarty wants to merge 7 commits into
base: main
Choose a base branch
from
+27 −22
Open

Eliminate movprfx stalls in SVE BBQ D1Q4 bulk scorer (~14% throughput improvement) #155001

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7759074
Use svand_u8_x in SVE D1Q4 bulk scorer to avoid movprfx
ChrisHegarty Jul 24, 2026
6c9abf1
Update docs/changelog/155001.yaml
ChrisHegarty Jul 24, 2026
d430a10
Merge branch 'main' into chegar/sve-d1q4-movprfx
ChrisHegarty Jul 24, 2026
e2495dd
Update build.gradle
ChrisHegarty Jul 24, 2026
07216ae
Update publish_vec_binaries.sh
ChrisHegarty Jul 24, 2026
3f290f4
Update 155001.yaml
ChrisHegarty Jul 24, 2026
617baf6
Merge branch 'main' into chegar/sve-d1q4-movprfx
ChrisHegarty Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/155001.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
area: "Vector Search"
issues: []
pr: 155001
summary: Eliminate movprfx stalls in SVE BBQ D1Q4 bulk scorer (~14% throughput improvement)
type: enhancement
2 changes: 1 addition & 1 deletion libs/native/libraries/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ configurations.consumable('nativeLibs') {
}

var zstdVersion = "1.5.7"
var vecVersion = "1.0.134"
var vecVersion = "1.0.137"
var pqrsVersion = "0.2.0"

repositories {
Expand Down
2 changes: 1 addition & 1 deletion libs/simdvec/native/publish_vec_binaries.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

set -euo pipefail

VERSION="1.0.134"
VERSION="1.0.137"

LOCAL=false
FORCE_UPLOAD=false
Expand Down
40 changes: 20 additions & 20 deletions libs/simdvec/native/src/vec/c/aarch64/vec_bbq_2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
static inline svuint64_t dot_bit_sv(const svbool_t pg, const svuint8_t a, const int8_t* b) {
const svuint8_t q0 = svld1_u8(pg, (const uint8_t*)b);
// reinterpret the u8 result as u64 so the count doesn't overflow
return svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, q0, a)));
return svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), q0, a)));
}

static inline int64_t dotd1q4_inner(const int8_t* a, const int8_t* query, const int32_t length) {
Expand Down Expand Up @@ -202,25 +202,25 @@ static inline void dotd1q4_inner_bulk(
const svuint8_t v2 = svld1_u8(pg, (const uint8_t*)(a2 + r));
const svuint8_t v3 = svld1_u8(pg, (const uint8_t*)(a3 + r));

acc0_0 = svadd_u64_x(svptrue_b64(), acc0_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q0))));
acc1_0 = svadd_u64_x(svptrue_b64(), acc1_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q1))));
acc2_0 = svadd_u64_x(svptrue_b64(), acc2_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q2))));
acc3_0 = svadd_u64_x(svptrue_b64(), acc3_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v0, q3))));

acc0_1 = svadd_u64_x(svptrue_b64(), acc0_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q0))));
acc1_1 = svadd_u64_x(svptrue_b64(), acc1_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q1))));
acc2_1 = svadd_u64_x(svptrue_b64(), acc2_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q2))));
acc3_1 = svadd_u64_x(svptrue_b64(), acc3_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v1, q3))));

acc0_2 = svadd_u64_x(svptrue_b64(), acc0_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q0))));
acc1_2 = svadd_u64_x(svptrue_b64(), acc1_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q1))));
acc2_2 = svadd_u64_x(svptrue_b64(), acc2_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q2))));
acc3_2 = svadd_u64_x(svptrue_b64(), acc3_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v2, q3))));

acc0_3 = svadd_u64_x(svptrue_b64(), acc0_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q0))));
acc1_3 = svadd_u64_x(svptrue_b64(), acc1_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q1))));
acc2_3 = svadd_u64_x(svptrue_b64(), acc2_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q2))));
acc3_3 = svadd_u64_x(svptrue_b64(), acc3_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_z(pg, v3, q3))));
acc0_0 = svadd_u64_x(svptrue_b64(), acc0_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q0))));
acc1_0 = svadd_u64_x(svptrue_b64(), acc1_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q1))));
acc2_0 = svadd_u64_x(svptrue_b64(), acc2_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q2))));
acc3_0 = svadd_u64_x(svptrue_b64(), acc3_0, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v0, q3))));

acc0_1 = svadd_u64_x(svptrue_b64(), acc0_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q0))));
acc1_1 = svadd_u64_x(svptrue_b64(), acc1_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q1))));
acc2_1 = svadd_u64_x(svptrue_b64(), acc2_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q2))));
acc3_1 = svadd_u64_x(svptrue_b64(), acc3_1, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v1, q3))));

acc0_2 = svadd_u64_x(svptrue_b64(), acc0_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q0))));
acc1_2 = svadd_u64_x(svptrue_b64(), acc1_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q1))));
acc2_2 = svadd_u64_x(svptrue_b64(), acc2_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q2))));
acc3_2 = svadd_u64_x(svptrue_b64(), acc3_2, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v2, q3))));

acc0_3 = svadd_u64_x(svptrue_b64(), acc0_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q0))));
acc1_3 = svadd_u64_x(svptrue_b64(), acc1_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q1))));
acc2_3 = svadd_u64_x(svptrue_b64(), acc2_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q2))));
acc3_3 = svadd_u64_x(svptrue_b64(), acc3_3, svcnt_u64_x(svptrue_b64(), svreinterpret_u64(svand_u8_x(svptrue_b8(), v3, q3))));

r += svcntb();
}
Expand Down
Loading