From f76d8fc69166809944fb8f2dbbf84f299f82c502 Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sat, 14 Feb 2026 19:10:59 -0500 Subject: [PATCH] bench: add grpc-based ember vector benchmarks adds EmberGrpcClient to bench-vector.py using the ember-py gRPC client for binary-packed vector transport. adds --ember-grpc flag to bench-vector.sh to run RESP and gRPC benchmarks side by side. updates setup-vm-vector.sh to install ember-py and build with grpc feature. --- bench/bench-vector.py | 37 +++++++++++++++++- bench/bench-vector.sh | 83 ++++++++++++++++++++++++++++++++++------ bench/setup-vm-vector.sh | 12 ++++-- 3 files changed, 117 insertions(+), 15 deletions(-) diff --git a/bench/bench-vector.py b/bench/bench-vector.py index 485d3f9e..85dce0be 100755 --- a/bench/bench-vector.py +++ b/bench/bench-vector.py @@ -100,6 +100,38 @@ def name(self) -> str: return "ember" +class EmberGrpcClient(VectorClient): + """ember vector client using the python gRPC client. + + sends vectors as packed floats — no string parsing overhead. + requires the ember-py package to be installed. + """ + + def __init__(self, host: str = "127.0.0.1", port: int = 6380): + from ember import EmberClient + self.client = EmberClient(f"{host}:{port}") + self.key = "bench_vectors" + + def setup(self, dim: int, metric: str = "cosine"): + self.client.flushdb() + + def insert_batch(self, ids: list, vectors: np.ndarray): + for i, vid in enumerate(ids): + vec = vectors[i].tolist() + self.client.vadd(self.key, vid, vec, metric="cosine", m=16, ef=64) + + def query(self, vector: np.ndarray, k: int) -> list: + results = self.client.vsim(self.key, vector.tolist(), count=k) + return [r[0] for r in results] + + def teardown(self): + self.client.flushdb() + self.client.close() + + def name(self) -> str: + return "ember-grpc" + + class ChromaClient(VectorClient): """chromadb client via HTTP API.""" @@ -308,7 +340,7 @@ def compute_recall(client: VectorClient, queries: np.ndarray, def main(): parser = argparse.ArgumentParser(description="vector similarity benchmark") - parser.add_argument("--system", required=True, choices=["ember", "chromadb", "pgvector"]) + parser.add_argument("--system", required=True, choices=["ember", "ember-grpc", "chromadb", "pgvector"]) parser.add_argument("--mode", default="random", choices=["random", "sift"]) parser.add_argument("--dim", type=int, default=128) parser.add_argument("--count", type=int, default=100000) @@ -316,6 +348,7 @@ def main(): parser.add_argument("--k", type=int, default=10) parser.add_argument("--batch-size", type=int, default=500) parser.add_argument("--ember-port", type=int, default=6379) + parser.add_argument("--ember-grpc-port", type=int, default=6380) parser.add_argument("--chroma-port", type=int, default=8000) parser.add_argument("--pgvector-port", type=int, default=5432) parser.add_argument("--sift-dir", default="bench/vector_data") @@ -325,6 +358,8 @@ def main(): # create client if args.system == "ember": client = EmberClient(port=args.ember_port) + elif args.system == "ember-grpc": + client = EmberGrpcClient(port=args.ember_grpc_port) elif args.system == "chromadb": client = ChromaClient(port=args.chroma_port) elif args.system == "pgvector": diff --git a/bench/bench-vector.sh b/bench/bench-vector.sh index 75752ef3..6d7d1a8a 100755 --- a/bench/bench-vector.sh +++ b/bench/bench-vector.sh @@ -9,11 +9,13 @@ # usage: # bash bench/bench-vector.sh # full comparison (100k vectors) # bash bench/bench-vector.sh --ember-only # ember only (no docker needed) +# bash bench/bench-vector.sh --ember-grpc # include ember gRPC benchmark # bash bench/bench-vector.sh --quick # quick run (1k vectors) # bash bench/bench-vector.sh --sift # SIFT1M recall accuracy # # environment variables: # EMBER_PORT ember port (default: 6379) +# EMBER_GRPC_PORT ember gRPC port (default: 6380) # CHROMA_PORT chromadb port (default: 8000) # PGVECTOR_PORT pgvector port (default: 5432) # VECTOR_COUNT base vector count (default: 100000) @@ -26,6 +28,7 @@ set -euo pipefail # --- configuration --- EMBER_PORT="${EMBER_PORT:-6379}" +EMBER_GRPC_PORT="${EMBER_GRPC_PORT:-6380}" CHROMA_PORT="${CHROMA_PORT:-8000}" PGVECTOR_PORT="${PGVECTOR_PORT:-5432}" VECTOR_COUNT="${VECTOR_COUNT:-100000}" @@ -40,12 +43,14 @@ BENCH_SCRIPT="bench/bench-vector.py" SIFT_DIR="bench/vector_data" EMBER_ONLY=false +EMBER_GRPC=false QUICK_MODE=false SIFT_MODE=false for arg in "$@"; do case "$arg" in --ember-only) EMBER_ONLY=true ;; + --ember-grpc) EMBER_GRPC=true ;; --quick) QUICK_MODE=true ;; --sift) SIFT_MODE=true ;; *) echo "unknown flag: $arg"; exit 1 ;; @@ -204,10 +209,20 @@ elif [[ "$EMBER_ONLY" == "false" ]] && ! python3 -c "import chromadb, psycopg2" pip install --quiet $REQUIRED_DEPS fi +# install ember-py for gRPC benchmarks +if [[ "$EMBER_GRPC" == "true" ]]; then + ensure_venv + pip install --quiet ./clients/ember-py +fi + # build ember if needed if [[ ! -x "$EMBER_BIN" ]]; then - echo "building ember-server with vector support..." - cargo build --release -p ember-server --features jemalloc,vector + FEATURES="jemalloc,vector" + if [[ "$EMBER_GRPC" == "true" ]]; then + FEATURES="$FEATURES,grpc" + fi + echo "building ember-server with features: $FEATURES..." + cargo build --release -p ember-server --features "$FEATURES" fi # download SIFT dataset if needed @@ -250,6 +265,7 @@ BENCH_ARGS=( --k "$K" --batch-size "$BATCH_SIZE" --ember-port "$EMBER_PORT" + --ember-grpc-port "$EMBER_GRPC_PORT" --chroma-port "$CHROMA_PORT" --pgvector-port "$PGVECTOR_PORT" --sift-dir "$SIFT_DIR" @@ -265,14 +281,29 @@ fi echo "--- ember ---" echo "" -echo " starting ember on port $EMBER_PORT..." -"$EMBER_BIN" --port "$EMBER_PORT" > /dev/null 2>&1 & +EMBER_ARGS=(--port "$EMBER_PORT") +if [[ "$EMBER_GRPC" == "true" ]]; then + EMBER_ARGS+=(--grpc-port "$EMBER_GRPC_PORT") + echo " starting ember on port $EMBER_PORT (gRPC: $EMBER_GRPC_PORT)..." +else + echo " starting ember on port $EMBER_PORT..." +fi +"$EMBER_BIN" "${EMBER_ARGS[@]}" > /dev/null 2>&1 & EMBER_PID=$! wait_for_ember "$EMBER_PORT" EMBER_JSON="/tmp/bench_vector_ember.json" python3 "$BENCH_SCRIPT" --system ember "${BENCH_ARGS[@]}" --output "$EMBER_JSON" +# also run gRPC benchmark if requested +EMBER_GRPC_JSON="/tmp/bench_vector_ember_grpc.json" +if [[ "$EMBER_GRPC" == "true" ]]; then + echo "" + echo "--- ember (gRPC) ---" + echo "" + python3 "$BENCH_SCRIPT" --system ember-grpc "${BENCH_ARGS[@]}" --output "$EMBER_GRPC_JSON" +fi + EMBER_RSS=$(get_rss_mb "$EMBER_PID") echo " memory (RSS): ${EMBER_RSS} MB" @@ -362,6 +393,21 @@ E_P50=$(extract "$EMBER_JSON" "query.p50_ms") E_P95=$(extract "$EMBER_JSON" "query.p95_ms") E_P99=$(extract "$EMBER_JSON" "query.p99_ms") +# ember gRPC results +EG_INSERT="—" +EG_QUERY="—" +EG_P50="—" +EG_P95="—" +EG_P99="—" + +if [[ "$EMBER_GRPC" == "true" ]] && [[ -f "$EMBER_GRPC_JSON" ]]; then + EG_INSERT=$(extract "$EMBER_GRPC_JSON" "insert.throughput") + EG_QUERY=$(extract "$EMBER_GRPC_JSON" "query.throughput") + EG_P50=$(extract "$EMBER_GRPC_JSON" "query.p50_ms") + EG_P95=$(extract "$EMBER_GRPC_JSON" "query.p95_ms") + EG_P99=$(extract "$EMBER_GRPC_JSON" "query.p99_ms") +fi + # chromadb results C_INSERT="—" C_QUERY="—" @@ -433,6 +479,21 @@ else printf "$fmt\n" "memory (MB)" "$EMBER_RSS" "$CHROMA_RSS" "$PGVECTOR_RSS" fi +# grpc comparison (if enabled) +if [[ "$EMBER_GRPC" == "true" ]]; then + echo "" + echo "=== ember RESP vs gRPC ===" + echo "" + fmt="%-24s %14s %14s" + printf "$fmt\n" "metric" "RESP" "gRPC" + printf "$fmt\n" "------" "----" "----" + printf "$fmt\n" "insert (vectors/sec)" "$E_INSERT" "$EG_INSERT" + printf "$fmt\n" "query (queries/sec)" "$E_QUERY" "$EG_QUERY" + printf "$fmt\n" "query p50 (ms)" "$E_P50" "$EG_P50" + printf "$fmt\n" "query p95 (ms)" "$E_P95" "$EG_P95" + printf "$fmt\n" "query p99 (ms)" "$E_P99" "$EG_P99" +fi + # recall (sift mode only) if [[ "$SIFT_MODE" == "true" ]]; then echo "" @@ -460,13 +521,13 @@ echo "" RESULT_FILE="$RESULTS_DIR/${TIMESTAMP}-vector.csv" { - echo "metric,ember,chromadb,pgvector" - echo "insert_throughput,$E_INSERT,$C_INSERT,$P_INSERT" - echo "query_throughput,$E_QUERY,$C_QUERY,$P_QUERY" - echo "query_p50_ms,$E_P50,$C_P50,$P_P50" - echo "query_p95_ms,$E_P95,$C_P95,$P_P95" - echo "query_p99_ms,$E_P99,$C_P99,$P_P99" - echo "memory_mb,$EMBER_RSS,$CHROMA_RSS,$PGVECTOR_RSS" + echo "metric,ember,ember-grpc,chromadb,pgvector" + echo "insert_throughput,$E_INSERT,$EG_INSERT,$C_INSERT,$P_INSERT" + echo "query_throughput,$E_QUERY,$EG_QUERY,$C_QUERY,$P_QUERY" + echo "query_p50_ms,$E_P50,$EG_P50,$C_P50,$P_P50" + echo "query_p95_ms,$E_P95,$EG_P95,$C_P95,$P_P95" + echo "query_p99_ms,$E_P99,$EG_P99,$C_P99,$P_P99" + echo "memory_mb,$EMBER_RSS,—,$CHROMA_RSS,$PGVECTOR_RSS" if [[ "$SIFT_MODE" == "true" ]]; then E_RECALL=$(extract "$EMBER_JSON" "recall.recall_at_k") C_RECALL="—" diff --git a/bench/setup-vm-vector.sh b/bench/setup-vm-vector.sh index 04c187af..e5d01091 100755 --- a/bench/setup-vm-vector.sh +++ b/bench/setup-vm-vector.sh @@ -14,7 +14,7 @@ echo "=== installing python dependencies ===" sudo apt-get update sudo apt-get install -y python3-pip python3-dev -pip3 install --quiet redis chromadb psycopg2-binary numpy +pip3 install --quiet redis chromadb psycopg2-binary numpy grpcio protobuf echo "" echo "=== installing docker ===" @@ -35,11 +35,16 @@ sudo docker pull chromadb/chroma sudo docker pull pgvector/pgvector:pg17 echo "" -echo "=== rebuilding ember with vector support ===" +echo "=== installing ember python client ===" cd ~/ember +pip3 install --quiet ./clients/ember-py + +echo "" +echo "=== rebuilding ember with vector + grpc support ===" + source ~/.cargo/env -cargo build --release -p ember-server --features jemalloc,vector +cargo build --release -p ember-server --features jemalloc,vector,grpc echo "" echo "=== verifying ===" @@ -54,5 +59,6 @@ echo "" echo "run benchmarks with:" echo " ./bench/bench-vector.sh # 100k random vectors" echo " ./bench/bench-vector.sh --ember-only # ember only (no docker)" +echo " ./bench/bench-vector.sh --ember-grpc # include gRPC benchmark" echo " ./bench/bench-vector.sh --quick # quick sanity check" echo " ./bench/bench-vector.sh --sift # SIFT1M recall accuracy"