Skip to content

bench: add qdrant comparison to vector benchmark #145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kacy merged 4 commits into from
Feb 15, 2026
Merged

bench: add qdrant comparison to vector benchmark #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ae24abb
bench: add qdrant comparison to vector benchmark
kacy Feb 15, 2026
a1edb85
merge main: resolve conflicts with ember-grpc addition
kacy Feb 15, 2026
a780870
fix qdrant client for current SDK and display table
kacy Feb 15, 2026
a276751
merge main: keep fixed qdrant SDK usage
kacy Feb 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 11 additions & 8 deletions bench/bench-vector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,11 +250,14 @@ class QdrantClient_(VectorClient):

def __init__(self, host: str = "127.0.0.1", port: int = 6333):
from qdrant_client import QdrantClient as _QC
from qdrant_client.models import Distance, VectorParams, PointStruct
from qdrant_client.models import (
Distance, VectorParams, PointStruct, HnswConfigDiff,
)
self._qc = _QC(host=host, port=port)
self._PointStruct = PointStruct
self._Distance = Distance
self._VectorParams = VectorParams
self._HnswConfigDiff = HnswConfigDiff
self.collection_name = "bench_vectors"

def setup(self, dim: int, metric: str = "cosine"):
Expand All @@ -269,10 +272,10 @@ def setup(self, dim: int, metric: str = "cosine"):
size=dim,
distance=self._Distance.COSINE,
),
hnsw_config={
"m": 16,
"ef_construct": 64,
},
hnsw_config=self._HnswConfigDiff(
m=16,
ef_construct=64,
),
)

def insert_batch(self, ids: list, vectors: np.ndarray):
Expand All @@ -290,14 +293,14 @@ def insert_batch(self, ids: list, vectors: np.ndarray):
)

def query(self, vector: np.ndarray, k: int) -> list:
results = self._qc.search(
response = self._qc.query_points(
collection_name=self.collection_name,
query_vector=vector.tolist(),
query=vector.tolist(),
limit=k,
)
return [
r.payload.get("name", str(r.id)) if r.payload else str(r.id)
for r in results
for r in response.points
]

def teardown(self):
Expand Down
19 changes: 17 additions & 2 deletions bench/bench-vector.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,17 @@ fi
echo "hnsw: M=16, ef_construction=64, cosine similarity, k=$K"
echo ""

if [[ "$EMBER_ONLY" == "true" ]]; then
if [[ "$EMBER_ONLY" == "true" ]] && [[ "$QDRANT" == "true" ]]; then
fmt="%-24s %14s %14s"
printf "$fmt\n" "metric" "ember" "qdrant"
printf "$fmt\n" "------" "-----" "------"
printf "$fmt\n" "insert (vectors/sec)" "$E_INSERT" "$Q_INSERT"
printf "$fmt\n" "query (queries/sec)" "$E_QUERY" "$Q_QUERY"
printf "$fmt\n" "query p50 (ms)" "$E_P50" "$Q_P50"
printf "$fmt\n" "query p95 (ms)" "$E_P95" "$Q_P95"
printf "$fmt\n" "query p99 (ms)" "$E_P99" "$Q_P99"
printf "$fmt\n" "memory (MB)" "$EMBER_RSS" "$QDRANT_RSS"
elif [[ "$EMBER_ONLY" == "true" ]]; then
fmt="%-24s %14s"
printf "$fmt\n" "metric" "ember"
printf "$fmt\n" "------" "-----"
Expand Down Expand Up @@ -586,7 +596,12 @@ if [[ "$SIFT_MODE" == "true" ]]; then

E_RECALL=$(extract "$EMBER_JSON" "recall.recall_at_k")

if [[ "$EMBER_ONLY" == "true" ]]; then
if [[ "$EMBER_ONLY" == "true" ]] && [[ "$QDRANT" == "true" ]]; then
Q_RECALL=$(extract "$QDRANT_JSON" "recall.recall_at_k")
printf "%-24s %14s %14s\n" "metric" "ember" "qdrant"
printf "%-24s %14s %14s\n" "------" "-----" "------"
printf "%-24s %14s %14s\n" "recall@$K" "$E_RECALL" "$Q_RECALL"
elif [[ "$EMBER_ONLY" == "true" ]]; then
printf "%-24s %14s\n" "metric" "ember"
printf "%-24s %14s\n" "------" "-----"
printf "%-24s %14s\n" "recall@$K" "$E_RECALL"
Expand Down
Loading