diff --git a/bench/README.md b/bench/README.md index 548abec7..9d34fe51 100644 --- a/bench/README.md +++ b/bench/README.md @@ -118,24 +118,24 @@ note: encryption only affects persistence writes. GET throughput should be uncha ember vs chromadb vs pgvector. 100k random vectors, 128 dimensions, cosine metric, k=10 kNN search. HNSW index: M=16, ef_construction=64 for all systems. tested on GCP c2-standard-8. -| metric | ember | chromadb | pgvector | -|--------|-------|----------|----------| -| insert (vectors/sec) | 917 | **3,738** | 1,562 | -| query (queries/sec) | **1,214** | 376 | 882 | -| query p99 (ms) | **1.09ms** | 2.90ms | 1.52ms | -| memory (MB) | **30 MB** | 122 MB | 178 MB | +| metric | ember | chromadb | pgvector | qdrant | +|--------|-------|----------|----------|--------| +| insert (vectors/sec) | 917 | **3,738** | 1,562 | — | +| query (queries/sec) | **1,214** | 376 | 882 | — | +| query p99 (ms) | **1.09ms** | 2.90ms | 1.52ms | — | +| memory (MB) | **30 MB** | 122 MB | 178 MB | — | ember's query throughput is 3.2x chromadb and 1.4x pgvector, with 4-6x lower memory usage. insert throughput is lower due to per-vector RESP protocol overhead — batched pipelining helps but each VADD is still a separate command. #### SIFT1M recall accuracy (128-dim, 1M vectors, 10k queries) -| metric | ember | chromadb | pgvector | -|--------|-------|----------|----------| -| recall@10 | — | — | — | -| insert (vectors/sec) | — | — | — | -| query p99 (ms) | — | — | — | +| metric | ember | chromadb | pgvector | qdrant | +|--------|-------|----------|----------|--------| +| recall@10 | — | — | — | — | +| insert (vectors/sec) | — | — | — | — | +| query p99 (ms) | — | — | — | — | -*results pending — requires a larger VM (c2-standard-16 or higher) since the 1M-vector HNSW index exceeds 16GB RAM during construction. run `bench/bench-vector.sh --sift` to populate.* +*results pending — run `bench/bench-vector.sh --sift --qdrant` on a c2-standard-8 (32GB) to populate. SIFT1M is 1M × 128 × 4B = 512MB raw data; ember HNSW peaks around 1.5GB RSS, well within 32GB.* ### scaling efficiency diff --git a/bench/bench-vector.sh b/bench/bench-vector.sh index b4b899ec..69f65140 100755 --- a/bench/bench-vector.sh +++ b/bench/bench-vector.sh @@ -11,7 +11,7 @@ # bash bench/bench-vector.sh --ember-only # ember only (no docker needed) # bash bench/bench-vector.sh --ember-grpc # include ember gRPC benchmark # bash bench/bench-vector.sh --quick # quick run (1k vectors) -# bash bench/bench-vector.sh --qdrant # include qdrant comparison +# bash bench/bench-vector.sh --no-qdrant # skip qdrant comparison # bash bench/bench-vector.sh --sift # SIFT1M recall accuracy # # environment variables: @@ -46,7 +46,7 @@ BENCH_SCRIPT="bench/bench-vector.py" SIFT_DIR="bench/vector_data" EMBER_ONLY=false -QDRANT=false +QDRANT=true EMBER_GRPC=false QUICK_MODE=false SIFT_MODE=false @@ -55,6 +55,7 @@ for arg in "$@"; do case "$arg" in --ember-only) EMBER_ONLY=true ;; --qdrant) QDRANT=true ;; + --no-qdrant) QDRANT=false ;; --ember-grpc) EMBER_GRPC=true ;; --quick) QUICK_MODE=true ;; --sift) SIFT_MODE=true ;; diff --git a/bench/setup-vm-vector.sh b/bench/setup-vm-vector.sh index e5d01091..e7b484fe 100755 --- a/bench/setup-vm-vector.sh +++ b/bench/setup-vm-vector.sh @@ -3,7 +3,7 @@ # additional VM setup for vector similarity benchmarks. # run after setup-vm.sh on a fresh ubuntu VM. # -# installs: python3 + deps, docker, chromadb/pgvector images +# installs: python3 + deps, docker, chromadb/pgvector/qdrant images # # usage: ssh user@vm 'bash -s' < ./bench/setup-vm-vector.sh @@ -14,7 +14,7 @@ echo "=== installing python dependencies ===" sudo apt-get update sudo apt-get install -y python3-pip python3-dev -pip3 install --quiet redis chromadb psycopg2-binary numpy grpcio protobuf +pip3 install --quiet redis chromadb psycopg2-binary numpy grpcio protobuf qdrant-client echo "" echo "=== installing docker ===" @@ -33,6 +33,7 @@ echo "=== pulling docker images ===" sudo docker pull chromadb/chroma sudo docker pull pgvector/pgvector:pg17 +sudo docker pull qdrant/qdrant echo "" echo "=== installing ember python client ===" @@ -49,8 +50,8 @@ cargo build --release -p ember-server --features jemalloc,vector,grpc echo "" echo "=== verifying ===" -python3 -c "import redis, chromadb, psycopg2, numpy; print('python deps: ok')" -sudo docker images | grep -E "chroma|pgvector" || true +python3 -c "import redis, chromadb, psycopg2, numpy, qdrant_client; print('python deps: ok')" +sudo docker images | grep -E "chroma|pgvector|qdrant" || true ./target/release/ember-server --help | head -3 echo "" @@ -61,4 +62,5 @@ echo " ./bench/bench-vector.sh # 100k random vectors" echo " ./bench/bench-vector.sh --ember-only # ember only (no docker)" echo " ./bench/bench-vector.sh --ember-grpc # include gRPC benchmark" echo " ./bench/bench-vector.sh --quick # quick sanity check" +echo " ./bench/bench-vector.sh --qdrant # include qdrant comparison" echo " ./bench/bench-vector.sh --sift # SIFT1M recall accuracy"