diff --git a/bench/bench-vector.py b/bench/bench-vector.py index 85dce0be..8a7a1d8f 100755 --- a/bench/bench-vector.py +++ b/bench/bench-vector.py @@ -245,6 +245,71 @@ def name(self) -> str: return "pgvector" +class QdrantClient_(VectorClient): + """qdrant client via the official python SDK.""" + + def __init__(self, host: str = "127.0.0.1", port: int = 6333): + from qdrant_client import QdrantClient as _QC + from qdrant_client.models import Distance, VectorParams, PointStruct + self._qc = _QC(host=host, port=port) + self._PointStruct = PointStruct + self._Distance = Distance + self._VectorParams = VectorParams + self.collection_name = "bench_vectors" + + def setup(self, dim: int, metric: str = "cosine"): + # delete if exists + try: + self._qc.delete_collection(self.collection_name) + except Exception: + pass + self._qc.create_collection( + collection_name=self.collection_name, + vectors_config=self._VectorParams( + size=dim, + distance=self._Distance.COSINE, + ), + hnsw_config={ + "m": 16, + "ef_construct": 64, + }, + ) + + def insert_batch(self, ids: list, vectors: np.ndarray): + points = [ + self._PointStruct( + id=idx, + vector=vectors[idx].tolist(), + payload={"name": vid}, + ) + for idx, vid in enumerate(ids) + ] + self._qc.upsert( + collection_name=self.collection_name, + points=points, + ) + + def query(self, vector: np.ndarray, k: int) -> list: + results = self._qc.search( + collection_name=self.collection_name, + query_vector=vector.tolist(), + limit=k, + ) + return [ + r.payload.get("name", str(r.id)) if r.payload else str(r.id) + for r in results + ] + + def teardown(self): + try: + self._qc.delete_collection(self.collection_name) + except Exception: + pass + + def name(self) -> str: + return "qdrant" + + # --------------------------------------------------------------------------- # benchmark functions # --------------------------------------------------------------------------- @@ -340,7 +405,7 @@ def compute_recall(client: VectorClient, queries: np.ndarray, def main(): parser = argparse.ArgumentParser(description="vector similarity benchmark") - parser.add_argument("--system", required=True, choices=["ember", "ember-grpc", "chromadb", "pgvector"]) + parser.add_argument("--system", required=True, choices=["ember", "ember-grpc", "chromadb", "pgvector", "qdrant"]) parser.add_argument("--mode", default="random", choices=["random", "sift"]) parser.add_argument("--dim", type=int, default=128) parser.add_argument("--count", type=int, default=100000) @@ -351,6 +416,7 @@ def main(): parser.add_argument("--ember-grpc-port", type=int, default=6380) parser.add_argument("--chroma-port", type=int, default=8000) parser.add_argument("--pgvector-port", type=int, default=5432) + parser.add_argument("--qdrant-port", type=int, default=6333) parser.add_argument("--sift-dir", default="bench/vector_data") parser.add_argument("--output", help="JSON output file") args = parser.parse_args() @@ -364,6 +430,8 @@ def main(): client = ChromaClient(port=args.chroma_port) elif args.system == "pgvector": client = PgVectorClient(port=args.pgvector_port) + elif args.system == "qdrant": + client = QdrantClient_(port=args.qdrant_port) # load or generate vectors if args.mode == "sift": diff --git a/bench/bench-vector.sh b/bench/bench-vector.sh index 6d7d1a8a..517a35c7 100755 --- a/bench/bench-vector.sh +++ b/bench/bench-vector.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# vector similarity benchmark: ember vs chromadb vs pgvector. +# vector similarity benchmark: ember vs chromadb vs pgvector (+ optional qdrant). # # compares insert throughput, query throughput/latency, and memory usage # across three vector databases using identical HNSW parameters (M=16, @@ -11,6 +11,7 @@ # bash bench/bench-vector.sh --ember-only # ember only (no docker needed) # bash bench/bench-vector.sh --ember-grpc # include ember gRPC benchmark # bash bench/bench-vector.sh --quick # quick run (1k vectors) +# bash bench/bench-vector.sh --qdrant # include qdrant comparison # bash bench/bench-vector.sh --sift # SIFT1M recall accuracy # # environment variables: @@ -18,6 +19,7 @@ # EMBER_GRPC_PORT ember gRPC port (default: 6380) # CHROMA_PORT chromadb port (default: 8000) # PGVECTOR_PORT pgvector port (default: 5432) +# QDRANT_PORT qdrant port (default: 6333) # VECTOR_COUNT base vector count (default: 100000) # VECTOR_DIM vector dimensions (default: 128) # QUERY_COUNT query vector count (default: 1000) @@ -31,6 +33,7 @@ EMBER_PORT="${EMBER_PORT:-6379}" EMBER_GRPC_PORT="${EMBER_GRPC_PORT:-6380}" CHROMA_PORT="${CHROMA_PORT:-8000}" PGVECTOR_PORT="${PGVECTOR_PORT:-5432}" +QDRANT_PORT="${QDRANT_PORT:-6333}" VECTOR_COUNT="${VECTOR_COUNT:-100000}" VECTOR_DIM="${VECTOR_DIM:-128}" QUERY_COUNT="${QUERY_COUNT:-1000}" @@ -43,6 +46,7 @@ BENCH_SCRIPT="bench/bench-vector.py" SIFT_DIR="bench/vector_data" EMBER_ONLY=false +QDRANT=false EMBER_GRPC=false QUICK_MODE=false SIFT_MODE=false @@ -50,6 +54,7 @@ SIFT_MODE=false for arg in "$@"; do case "$arg" in --ember-only) EMBER_ONLY=true ;; + --qdrant) QDRANT=true ;; --ember-grpc) EMBER_GRPC=true ;; --quick) QUICK_MODE=true ;; --sift) SIFT_MODE=true ;; @@ -67,6 +72,7 @@ fi EMBER_PID="" CHROMA_CONTAINER="" PGVECTOR_CONTAINER="" +QDRANT_CONTAINER="" cleanup() { echo "" @@ -74,6 +80,7 @@ cleanup() { [[ -n "$EMBER_PID" ]] && kill "$EMBER_PID" 2>/dev/null && wait "$EMBER_PID" 2>/dev/null || true [[ -n "$CHROMA_CONTAINER" ]] && docker rm -f "$CHROMA_CONTAINER" > /dev/null 2>&1 || true [[ -n "$PGVECTOR_CONTAINER" ]] && docker rm -f "$PGVECTOR_CONTAINER" > /dev/null 2>&1 || true + [[ -n "$QDRANT_CONTAINER" ]] && docker rm -f "$QDRANT_CONTAINER" > /dev/null 2>&1 || true rm -f /tmp/bench_vector_*.json } trap cleanup EXIT @@ -164,6 +171,19 @@ wait_for_pgvector() { sleep 1 } +wait_for_qdrant() { + local port=$1 + local retries=60 + while ! curl -sf "http://127.0.0.1:$port/healthz" > /dev/null 2>&1; do + retries=$((retries - 1)) + if [[ $retries -le 0 ]]; then + echo "error: qdrant did not start on port $port" >&2 + exit 1 + fi + sleep 0.5 + done +} + # --- dependency checks --- if ! command -v python3 &> /dev/null; then @@ -200,6 +220,15 @@ if [[ "$EMBER_ONLY" == "false" ]]; then fi fi +if [[ "$QDRANT" == "true" ]]; then + REQUIRED_DEPS="$REQUIRED_DEPS qdrant-client" + + if ! command -v docker &> /dev/null; then + echo "error: docker required for qdrant" >&2 + exit 1 + fi +fi + # check if all deps are available in current python; if not, use venv if ! python3 -c "import redis, numpy" 2>/dev/null; then ensure_venv @@ -207,6 +236,9 @@ if ! python3 -c "import redis, numpy" 2>/dev/null; then elif [[ "$EMBER_ONLY" == "false" ]] && ! python3 -c "import chromadb, psycopg2" 2>/dev/null; then ensure_venv pip install --quiet $REQUIRED_DEPS +elif [[ "$QDRANT" == "true" ]] && ! python3 -c "import qdrant_client" 2>/dev/null; then + ensure_venv + pip install --quiet $REQUIRED_DEPS fi # install ember-py for gRPC benchmarks @@ -268,6 +300,7 @@ BENCH_ARGS=( --ember-grpc-port "$EMBER_GRPC_PORT" --chroma-port "$CHROMA_PORT" --pgvector-port "$PGVECTOR_PORT" + --qdrant-port "$QDRANT_PORT" --sift-dir "$SIFT_DIR" ) @@ -366,6 +399,32 @@ if [[ "$EMBER_ONLY" == "false" ]]; then echo "" fi +# --- benchmark qdrant --- + +QDRANT_JSON="/tmp/bench_vector_qdrant.json" +QDRANT_RSS=0 + +if [[ "$QDRANT" == "true" ]]; then + echo "--- qdrant ---" + echo "" + echo " starting qdrant container on port $QDRANT_PORT..." + QDRANT_CONTAINER=$(docker run -d --rm \ + -p "$QDRANT_PORT:6333" \ + qdrant/qdrant \ + 2>/dev/null) + wait_for_qdrant "$QDRANT_PORT" + + python3 "$BENCH_SCRIPT" --system qdrant "${BENCH_ARGS[@]}" --output "$QDRANT_JSON" + + QDRANT_RSS=$(get_container_rss_mb "$QDRANT_CONTAINER") + echo " memory (RSS): ${QDRANT_RSS} MB" + + docker rm -f "$QDRANT_CONTAINER" > /dev/null 2>&1 || true + QDRANT_CONTAINER="" + sleep 0.5 + echo "" +fi + # --- parse results --- # extract values from JSON files using python (avoids jq dependency) @@ -438,6 +497,21 @@ if [[ "$EMBER_ONLY" == "false" ]] && [[ -f "$PGVECTOR_JSON" ]]; then P_P99=$(extract "$PGVECTOR_JSON" "query.p99_ms") fi +# qdrant results +Q_INSERT="—" +Q_QUERY="—" +Q_P50="—" +Q_P95="—" +Q_P99="—" + +if [[ "$QDRANT" == "true" ]] && [[ -f "$QDRANT_JSON" ]]; then + Q_INSERT=$(extract "$QDRANT_JSON" "insert.throughput") + Q_QUERY=$(extract "$QDRANT_JSON" "query.throughput") + Q_P50=$(extract "$QDRANT_JSON" "query.p50_ms") + Q_P95=$(extract "$QDRANT_JSON" "query.p95_ms") + Q_P99=$(extract "$QDRANT_JSON" "query.p99_ms") +fi + # --- display results --- DATE=$(date +%Y-%m-%d) @@ -467,6 +541,16 @@ if [[ "$EMBER_ONLY" == "true" ]]; then printf "$fmt\n" "query p95 (ms)" "$E_P95" printf "$fmt\n" "query p99 (ms)" "$E_P99" printf "$fmt\n" "memory (MB)" "$EMBER_RSS" +elif [[ "$QDRANT" == "true" ]]; then + fmt="%-24s %14s %14s %14s %14s" + printf "$fmt\n" "metric" "ember" "chromadb" "pgvector" "qdrant" + printf "$fmt\n" "------" "-----" "--------" "--------" "------" + printf "$fmt\n" "insert (vectors/sec)" "$E_INSERT" "$C_INSERT" "$P_INSERT" "$Q_INSERT" + printf "$fmt\n" "query (queries/sec)" "$E_QUERY" "$C_QUERY" "$P_QUERY" "$Q_QUERY" + printf "$fmt\n" "query p50 (ms)" "$E_P50" "$C_P50" "$P_P50" "$Q_P50" + printf "$fmt\n" "query p95 (ms)" "$E_P95" "$C_P95" "$P_P95" "$Q_P95" + printf "$fmt\n" "query p99 (ms)" "$E_P99" "$C_P99" "$P_P99" "$Q_P99" + printf "$fmt\n" "memory (MB)" "$EMBER_RSS" "$CHROMA_RSS" "$PGVECTOR_RSS" "$QDRANT_RSS" else fmt="%-24s %14s %14s %14s" printf "$fmt\n" "metric" "ember" "chromadb" "pgvector" @@ -506,6 +590,13 @@ if [[ "$SIFT_MODE" == "true" ]]; then printf "%-24s %14s\n" "metric" "ember" printf "%-24s %14s\n" "------" "-----" printf "%-24s %14s\n" "recall@$K" "$E_RECALL" + elif [[ "$QDRANT" == "true" ]]; then + C_RECALL=$(extract "$CHROMA_JSON" "recall.recall_at_k") + P_RECALL=$(extract "$PGVECTOR_JSON" "recall.recall_at_k") + Q_RECALL=$(extract "$QDRANT_JSON" "recall.recall_at_k") + printf "%-24s %14s %14s %14s %14s\n" "metric" "ember" "chromadb" "pgvector" "qdrant" + printf "%-24s %14s %14s %14s %14s\n" "------" "-----" "--------" "--------" "------" + printf "%-24s %14s %14s %14s %14s\n" "recall@$K" "$E_RECALL" "$C_RECALL" "$P_RECALL" "$Q_RECALL" else C_RECALL=$(extract "$CHROMA_JSON" "recall.recall_at_k") P_RECALL=$(extract "$PGVECTOR_JSON" "recall.recall_at_k") @@ -521,22 +612,26 @@ echo "" RESULT_FILE="$RESULTS_DIR/${TIMESTAMP}-vector.csv" { - echo "metric,ember,ember-grpc,chromadb,pgvector" - echo "insert_throughput,$E_INSERT,$EG_INSERT,$C_INSERT,$P_INSERT" - echo "query_throughput,$E_QUERY,$EG_QUERY,$C_QUERY,$P_QUERY" - echo "query_p50_ms,$E_P50,$EG_P50,$C_P50,$P_P50" - echo "query_p95_ms,$E_P95,$EG_P95,$C_P95,$P_P95" - echo "query_p99_ms,$E_P99,$EG_P99,$C_P99,$P_P99" - echo "memory_mb,$EMBER_RSS,—,$CHROMA_RSS,$PGVECTOR_RSS" + echo "metric,ember,ember-grpc,chromadb,pgvector,qdrant" + echo "insert_throughput,$E_INSERT,$EG_INSERT,$C_INSERT,$P_INSERT,$Q_INSERT" + echo "query_throughput,$E_QUERY,$EG_QUERY,$C_QUERY,$P_QUERY,$Q_QUERY" + echo "query_p50_ms,$E_P50,$EG_P50,$C_P50,$P_P50,$Q_P50" + echo "query_p95_ms,$E_P95,$EG_P95,$C_P95,$P_P95,$Q_P95" + echo "query_p99_ms,$E_P99,$EG_P99,$C_P99,$P_P99,$Q_P99" + echo "memory_mb,$EMBER_RSS,—,$CHROMA_RSS,$PGVECTOR_RSS,$QDRANT_RSS" if [[ "$SIFT_MODE" == "true" ]]; then E_RECALL=$(extract "$EMBER_JSON" "recall.recall_at_k") C_RECALL="—" P_RECALL="—" + Q_RECALL="—" if [[ "$EMBER_ONLY" == "false" ]]; then C_RECALL=$(extract "$CHROMA_JSON" "recall.recall_at_k") P_RECALL=$(extract "$PGVECTOR_JSON" "recall.recall_at_k") fi - echo "recall_at_$K,$E_RECALL,$C_RECALL,$P_RECALL" + if [[ "$QDRANT" == "true" ]]; then + Q_RECALL=$(extract "$QDRANT_JSON" "recall.recall_at_k") + fi + echo "recall_at_$K,$E_RECALL,$C_RECALL,$P_RECALL,$Q_RECALL" fi } > "$RESULT_FILE"