From ae24abb35ab364c3030a6760ef394aef9015c556 Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sat, 14 Feb 2026 19:22:09 -0500 Subject: [PATCH 1/2] bench: add qdrant comparison to vector benchmark adds qdrant as an optional fourth comparison target in the vector similarity benchmark suite. opt-in via --qdrant flag to avoid requiring docker for the default run. - qdrant python client class using official SDK - docker container lifecycle (startup, health check, teardown) - results parsing, display table, and CSV output columns - pip dependency management for qdrant-client --- bench/bench-vector.py | 70 +++++++++++++++++++++++++- bench/bench-vector.sh | 113 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 173 insertions(+), 10 deletions(-) diff --git a/bench/bench-vector.py b/bench/bench-vector.py index 485d3f9e..9a5360e2 100755 --- a/bench/bench-vector.py +++ b/bench/bench-vector.py @@ -213,6 +213,71 @@ def name(self) -> str: return "pgvector" +class QdrantClient_(VectorClient): + """qdrant client via the official python SDK.""" + + def __init__(self, host: str = "127.0.0.1", port: int = 6333): + from qdrant_client import QdrantClient as _QC + from qdrant_client.models import Distance, VectorParams, PointStruct + self._qc = _QC(host=host, port=port) + self._PointStruct = PointStruct + self._Distance = Distance + self._VectorParams = VectorParams + self.collection_name = "bench_vectors" + + def setup(self, dim: int, metric: str = "cosine"): + # delete if exists + try: + self._qc.delete_collection(self.collection_name) + except Exception: + pass + self._qc.create_collection( + collection_name=self.collection_name, + vectors_config=self._VectorParams( + size=dim, + distance=self._Distance.COSINE, + ), + hnsw_config={ + "m": 16, + "ef_construct": 64, + }, + ) + + def insert_batch(self, ids: list, vectors: np.ndarray): + points = [ + self._PointStruct( + id=idx, + vector=vectors[idx].tolist(), + payload={"name": vid}, + ) + for idx, vid in enumerate(ids) + ] + self._qc.upsert( + collection_name=self.collection_name, + points=points, + ) + + def query(self, vector: np.ndarray, k: int) -> list: + results = self._qc.search( + collection_name=self.collection_name, + query_vector=vector.tolist(), + limit=k, + ) + return [ + r.payload.get("name", str(r.id)) if r.payload else str(r.id) + for r in results + ] + + def teardown(self): + try: + self._qc.delete_collection(self.collection_name) + except Exception: + pass + + def name(self) -> str: + return "qdrant" + + # --------------------------------------------------------------------------- # benchmark functions # --------------------------------------------------------------------------- @@ -308,7 +373,7 @@ def compute_recall(client: VectorClient, queries: np.ndarray, def main(): parser = argparse.ArgumentParser(description="vector similarity benchmark") - parser.add_argument("--system", required=True, choices=["ember", "chromadb", "pgvector"]) + parser.add_argument("--system", required=True, choices=["ember", "chromadb", "pgvector", "qdrant"]) parser.add_argument("--mode", default="random", choices=["random", "sift"]) parser.add_argument("--dim", type=int, default=128) parser.add_argument("--count", type=int, default=100000) @@ -318,6 +383,7 @@ def main(): parser.add_argument("--ember-port", type=int, default=6379) parser.add_argument("--chroma-port", type=int, default=8000) parser.add_argument("--pgvector-port", type=int, default=5432) + parser.add_argument("--qdrant-port", type=int, default=6333) parser.add_argument("--sift-dir", default="bench/vector_data") parser.add_argument("--output", help="JSON output file") args = parser.parse_args() @@ -329,6 +395,8 @@ def main(): client = ChromaClient(port=args.chroma_port) elif args.system == "pgvector": client = PgVectorClient(port=args.pgvector_port) + elif args.system == "qdrant": + client = QdrantClient_(port=args.qdrant_port) # load or generate vectors if args.mode == "sift": diff --git a/bench/bench-vector.sh b/bench/bench-vector.sh index 75752ef3..56d18c5c 100755 --- a/bench/bench-vector.sh +++ b/bench/bench-vector.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# vector similarity benchmark: ember vs chromadb vs pgvector. +# vector similarity benchmark: ember vs chromadb vs pgvector (+ optional qdrant). # # compares insert throughput, query throughput/latency, and memory usage # across three vector databases using identical HNSW parameters (M=16, @@ -10,12 +10,14 @@ # bash bench/bench-vector.sh # full comparison (100k vectors) # bash bench/bench-vector.sh --ember-only # ember only (no docker needed) # bash bench/bench-vector.sh --quick # quick run (1k vectors) +# bash bench/bench-vector.sh --qdrant # include qdrant comparison # bash bench/bench-vector.sh --sift # SIFT1M recall accuracy # # environment variables: # EMBER_PORT ember port (default: 6379) # CHROMA_PORT chromadb port (default: 8000) # PGVECTOR_PORT pgvector port (default: 5432) +# QDRANT_PORT qdrant port (default: 6333) # VECTOR_COUNT base vector count (default: 100000) # VECTOR_DIM vector dimensions (default: 128) # QUERY_COUNT query vector count (default: 1000) @@ -28,6 +30,7 @@ set -euo pipefail EMBER_PORT="${EMBER_PORT:-6379}" CHROMA_PORT="${CHROMA_PORT:-8000}" PGVECTOR_PORT="${PGVECTOR_PORT:-5432}" +QDRANT_PORT="${QDRANT_PORT:-6333}" VECTOR_COUNT="${VECTOR_COUNT:-100000}" VECTOR_DIM="${VECTOR_DIM:-128}" QUERY_COUNT="${QUERY_COUNT:-1000}" @@ -40,12 +43,14 @@ BENCH_SCRIPT="bench/bench-vector.py" SIFT_DIR="bench/vector_data" EMBER_ONLY=false +QDRANT=false QUICK_MODE=false SIFT_MODE=false for arg in "$@"; do case "$arg" in --ember-only) EMBER_ONLY=true ;; + --qdrant) QDRANT=true ;; --quick) QUICK_MODE=true ;; --sift) SIFT_MODE=true ;; *) echo "unknown flag: $arg"; exit 1 ;; @@ -62,6 +67,7 @@ fi EMBER_PID="" CHROMA_CONTAINER="" PGVECTOR_CONTAINER="" +QDRANT_CONTAINER="" cleanup() { echo "" @@ -69,6 +75,7 @@ cleanup() { [[ -n "$EMBER_PID" ]] && kill "$EMBER_PID" 2>/dev/null && wait "$EMBER_PID" 2>/dev/null || true [[ -n "$CHROMA_CONTAINER" ]] && docker rm -f "$CHROMA_CONTAINER" > /dev/null 2>&1 || true [[ -n "$PGVECTOR_CONTAINER" ]] && docker rm -f "$PGVECTOR_CONTAINER" > /dev/null 2>&1 || true + [[ -n "$QDRANT_CONTAINER" ]] && docker rm -f "$QDRANT_CONTAINER" > /dev/null 2>&1 || true rm -f /tmp/bench_vector_*.json } trap cleanup EXIT @@ -159,6 +166,19 @@ wait_for_pgvector() { sleep 1 } +wait_for_qdrant() { + local port=$1 + local retries=60 + while ! curl -sf "http://127.0.0.1:$port/healthz" > /dev/null 2>&1; do + retries=$((retries - 1)) + if [[ $retries -le 0 ]]; then + echo "error: qdrant did not start on port $port" >&2 + exit 1 + fi + sleep 0.5 + done +} + # --- dependency checks --- if ! command -v python3 &> /dev/null; then @@ -195,6 +215,15 @@ if [[ "$EMBER_ONLY" == "false" ]]; then fi fi +if [[ "$QDRANT" == "true" ]]; then + REQUIRED_DEPS="$REQUIRED_DEPS qdrant-client" + + if ! command -v docker &> /dev/null; then + echo "error: docker required for qdrant" >&2 + exit 1 + fi +fi + # check if all deps are available in current python; if not, use venv if ! python3 -c "import redis, numpy" 2>/dev/null; then ensure_venv @@ -202,6 +231,9 @@ if ! python3 -c "import redis, numpy" 2>/dev/null; then elif [[ "$EMBER_ONLY" == "false" ]] && ! python3 -c "import chromadb, psycopg2" 2>/dev/null; then ensure_venv pip install --quiet $REQUIRED_DEPS +elif [[ "$QDRANT" == "true" ]] && ! python3 -c "import qdrant_client" 2>/dev/null; then + ensure_venv + pip install --quiet $REQUIRED_DEPS fi # build ember if needed @@ -252,6 +284,7 @@ BENCH_ARGS=( --ember-port "$EMBER_PORT" --chroma-port "$CHROMA_PORT" --pgvector-port "$PGVECTOR_PORT" + --qdrant-port "$QDRANT_PORT" --sift-dir "$SIFT_DIR" ) @@ -335,6 +368,32 @@ if [[ "$EMBER_ONLY" == "false" ]]; then echo "" fi +# --- benchmark qdrant --- + +QDRANT_JSON="/tmp/bench_vector_qdrant.json" +QDRANT_RSS=0 + +if [[ "$QDRANT" == "true" ]]; then + echo "--- qdrant ---" + echo "" + echo " starting qdrant container on port $QDRANT_PORT..." + QDRANT_CONTAINER=$(docker run -d --rm \ + -p "$QDRANT_PORT:6333" \ + qdrant/qdrant \ + 2>/dev/null) + wait_for_qdrant "$QDRANT_PORT" + + python3 "$BENCH_SCRIPT" --system qdrant "${BENCH_ARGS[@]}" --output "$QDRANT_JSON" + + QDRANT_RSS=$(get_container_rss_mb "$QDRANT_CONTAINER") + echo " memory (RSS): ${QDRANT_RSS} MB" + + docker rm -f "$QDRANT_CONTAINER" > /dev/null 2>&1 || true + QDRANT_CONTAINER="" + sleep 0.5 + echo "" +fi + # --- parse results --- # extract values from JSON files using python (avoids jq dependency) @@ -392,6 +451,21 @@ if [[ "$EMBER_ONLY" == "false" ]] && [[ -f "$PGVECTOR_JSON" ]]; then P_P99=$(extract "$PGVECTOR_JSON" "query.p99_ms") fi +# qdrant results +Q_INSERT="—" +Q_QUERY="—" +Q_P50="—" +Q_P95="—" +Q_P99="—" + +if [[ "$QDRANT" == "true" ]] && [[ -f "$QDRANT_JSON" ]]; then + Q_INSERT=$(extract "$QDRANT_JSON" "insert.throughput") + Q_QUERY=$(extract "$QDRANT_JSON" "query.throughput") + Q_P50=$(extract "$QDRANT_JSON" "query.p50_ms") + Q_P95=$(extract "$QDRANT_JSON" "query.p95_ms") + Q_P99=$(extract "$QDRANT_JSON" "query.p99_ms") +fi + # --- display results --- DATE=$(date +%Y-%m-%d) @@ -421,6 +495,16 @@ if [[ "$EMBER_ONLY" == "true" ]]; then printf "$fmt\n" "query p95 (ms)" "$E_P95" printf "$fmt\n" "query p99 (ms)" "$E_P99" printf "$fmt\n" "memory (MB)" "$EMBER_RSS" +elif [[ "$QDRANT" == "true" ]]; then + fmt="%-24s %14s %14s %14s %14s" + printf "$fmt\n" "metric" "ember" "chromadb" "pgvector" "qdrant" + printf "$fmt\n" "------" "-----" "--------" "--------" "------" + printf "$fmt\n" "insert (vectors/sec)" "$E_INSERT" "$C_INSERT" "$P_INSERT" "$Q_INSERT" + printf "$fmt\n" "query (queries/sec)" "$E_QUERY" "$C_QUERY" "$P_QUERY" "$Q_QUERY" + printf "$fmt\n" "query p50 (ms)" "$E_P50" "$C_P50" "$P_P50" "$Q_P50" + printf "$fmt\n" "query p95 (ms)" "$E_P95" "$C_P95" "$P_P95" "$Q_P95" + printf "$fmt\n" "query p99 (ms)" "$E_P99" "$C_P99" "$P_P99" "$Q_P99" + printf "$fmt\n" "memory (MB)" "$EMBER_RSS" "$CHROMA_RSS" "$PGVECTOR_RSS" "$QDRANT_RSS" else fmt="%-24s %14s %14s %14s" printf "$fmt\n" "metric" "ember" "chromadb" "pgvector" @@ -445,6 +529,13 @@ if [[ "$SIFT_MODE" == "true" ]]; then printf "%-24s %14s\n" "metric" "ember" printf "%-24s %14s\n" "------" "-----" printf "%-24s %14s\n" "recall@$K" "$E_RECALL" + elif [[ "$QDRANT" == "true" ]]; then + C_RECALL=$(extract "$CHROMA_JSON" "recall.recall_at_k") + P_RECALL=$(extract "$PGVECTOR_JSON" "recall.recall_at_k") + Q_RECALL=$(extract "$QDRANT_JSON" "recall.recall_at_k") + printf "%-24s %14s %14s %14s %14s\n" "metric" "ember" "chromadb" "pgvector" "qdrant" + printf "%-24s %14s %14s %14s %14s\n" "------" "-----" "--------" "--------" "------" + printf "%-24s %14s %14s %14s %14s\n" "recall@$K" "$E_RECALL" "$C_RECALL" "$P_RECALL" "$Q_RECALL" else C_RECALL=$(extract "$CHROMA_JSON" "recall.recall_at_k") P_RECALL=$(extract "$PGVECTOR_JSON" "recall.recall_at_k") @@ -460,22 +551,26 @@ echo "" RESULT_FILE="$RESULTS_DIR/${TIMESTAMP}-vector.csv" { - echo "metric,ember,chromadb,pgvector" - echo "insert_throughput,$E_INSERT,$C_INSERT,$P_INSERT" - echo "query_throughput,$E_QUERY,$C_QUERY,$P_QUERY" - echo "query_p50_ms,$E_P50,$C_P50,$P_P50" - echo "query_p95_ms,$E_P95,$C_P95,$P_P95" - echo "query_p99_ms,$E_P99,$C_P99,$P_P99" - echo "memory_mb,$EMBER_RSS,$CHROMA_RSS,$PGVECTOR_RSS" + echo "metric,ember,chromadb,pgvector,qdrant" + echo "insert_throughput,$E_INSERT,$C_INSERT,$P_INSERT,$Q_INSERT" + echo "query_throughput,$E_QUERY,$C_QUERY,$P_QUERY,$Q_QUERY" + echo "query_p50_ms,$E_P50,$C_P50,$P_P50,$Q_P50" + echo "query_p95_ms,$E_P95,$C_P95,$P_P95,$Q_P95" + echo "query_p99_ms,$E_P99,$C_P99,$P_P99,$Q_P99" + echo "memory_mb,$EMBER_RSS,$CHROMA_RSS,$PGVECTOR_RSS,$QDRANT_RSS" if [[ "$SIFT_MODE" == "true" ]]; then E_RECALL=$(extract "$EMBER_JSON" "recall.recall_at_k") C_RECALL="—" P_RECALL="—" + Q_RECALL="—" if [[ "$EMBER_ONLY" == "false" ]]; then C_RECALL=$(extract "$CHROMA_JSON" "recall.recall_at_k") P_RECALL=$(extract "$PGVECTOR_JSON" "recall.recall_at_k") fi - echo "recall_at_$K,$E_RECALL,$C_RECALL,$P_RECALL" + if [[ "$QDRANT" == "true" ]]; then + Q_RECALL=$(extract "$QDRANT_JSON" "recall.recall_at_k") + fi + echo "recall_at_$K,$E_RECALL,$C_RECALL,$P_RECALL,$Q_RECALL" fi } > "$RESULT_FILE" From a780870b6e433ca8399188de40da90c1f7bd8b75 Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sun, 15 Feb 2026 00:27:13 -0500 Subject: [PATCH 2/2] fix qdrant client for current SDK and display table - use query_points instead of deprecated search method - use HnswConfigDiff model instead of raw dict - handle --ember-only --qdrant displaying both columns --- bench/bench-vector.py | 19 +++++++++++-------- bench/bench-vector.sh | 19 +++++++++++++++++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/bench/bench-vector.py b/bench/bench-vector.py index 8a7a1d8f..fbf13ede 100755 --- a/bench/bench-vector.py +++ b/bench/bench-vector.py @@ -250,11 +250,14 @@ class QdrantClient_(VectorClient): def __init__(self, host: str = "127.0.0.1", port: int = 6333): from qdrant_client import QdrantClient as _QC - from qdrant_client.models import Distance, VectorParams, PointStruct + from qdrant_client.models import ( + Distance, VectorParams, PointStruct, HnswConfigDiff, + ) self._qc = _QC(host=host, port=port) self._PointStruct = PointStruct self._Distance = Distance self._VectorParams = VectorParams + self._HnswConfigDiff = HnswConfigDiff self.collection_name = "bench_vectors" def setup(self, dim: int, metric: str = "cosine"): @@ -269,10 +272,10 @@ def setup(self, dim: int, metric: str = "cosine"): size=dim, distance=self._Distance.COSINE, ), - hnsw_config={ - "m": 16, - "ef_construct": 64, - }, + hnsw_config=self._HnswConfigDiff( + m=16, + ef_construct=64, + ), ) def insert_batch(self, ids: list, vectors: np.ndarray): @@ -290,14 +293,14 @@ def insert_batch(self, ids: list, vectors: np.ndarray): ) def query(self, vector: np.ndarray, k: int) -> list: - results = self._qc.search( + response = self._qc.query_points( collection_name=self.collection_name, - query_vector=vector.tolist(), + query=vector.tolist(), limit=k, ) return [ r.payload.get("name", str(r.id)) if r.payload else str(r.id) - for r in results + for r in response.points ] def teardown(self): diff --git a/bench/bench-vector.sh b/bench/bench-vector.sh index 517a35c7..b4b899ec 100755 --- a/bench/bench-vector.sh +++ b/bench/bench-vector.sh @@ -531,7 +531,17 @@ fi echo "hnsw: M=16, ef_construction=64, cosine similarity, k=$K" echo "" -if [[ "$EMBER_ONLY" == "true" ]]; then +if [[ "$EMBER_ONLY" == "true" ]] && [[ "$QDRANT" == "true" ]]; then + fmt="%-24s %14s %14s" + printf "$fmt\n" "metric" "ember" "qdrant" + printf "$fmt\n" "------" "-----" "------" + printf "$fmt\n" "insert (vectors/sec)" "$E_INSERT" "$Q_INSERT" + printf "$fmt\n" "query (queries/sec)" "$E_QUERY" "$Q_QUERY" + printf "$fmt\n" "query p50 (ms)" "$E_P50" "$Q_P50" + printf "$fmt\n" "query p95 (ms)" "$E_P95" "$Q_P95" + printf "$fmt\n" "query p99 (ms)" "$E_P99" "$Q_P99" + printf "$fmt\n" "memory (MB)" "$EMBER_RSS" "$QDRANT_RSS" +elif [[ "$EMBER_ONLY" == "true" ]]; then fmt="%-24s %14s" printf "$fmt\n" "metric" "ember" printf "$fmt\n" "------" "-----" @@ -586,7 +596,12 @@ if [[ "$SIFT_MODE" == "true" ]]; then E_RECALL=$(extract "$EMBER_JSON" "recall.recall_at_k") - if [[ "$EMBER_ONLY" == "true" ]]; then + if [[ "$EMBER_ONLY" == "true" ]] && [[ "$QDRANT" == "true" ]]; then + Q_RECALL=$(extract "$QDRANT_JSON" "recall.recall_at_k") + printf "%-24s %14s %14s\n" "metric" "ember" "qdrant" + printf "%-24s %14s %14s\n" "------" "-----" "------" + printf "%-24s %14s %14s\n" "recall@$K" "$E_RECALL" "$Q_RECALL" + elif [[ "$EMBER_ONLY" == "true" ]]; then printf "%-24s %14s\n" "metric" "ember" printf "%-24s %14s\n" "------" "-----" printf "%-24s %14s\n" "recall@$K" "$E_RECALL"