bench: add grpc-based ember vector benchmarks

bench/bench-vector.py

@@ -100,6 +100,38 @@ def name(self) -> str:
         return "ember"
 
 
+class EmberGrpcClient(VectorClient):
+    """ember vector client using the python gRPC client.
+
+    sends vectors as packed floats — no string parsing overhead.
+    requires the ember-py package to be installed.
+    """
+
+    def __init__(self, host: str = "127.0.0.1", port: int = 6380):
+        from ember import EmberClient
+        self.client = EmberClient(f"{host}:{port}")
+        self.key = "bench_vectors"
+
+    def setup(self, dim: int, metric: str = "cosine"):
+        self.client.flushdb()
+
+    def insert_batch(self, ids: list, vectors: np.ndarray):
+        for i, vid in enumerate(ids):
+            vec = vectors[i].tolist()
+            self.client.vadd(self.key, vid, vec, metric="cosine", m=16, ef=64)
+
+    def query(self, vector: np.ndarray, k: int) -> list:
+        results = self.client.vsim(self.key, vector.tolist(), count=k)
+        return [r[0] for r in results]
+
+    def teardown(self):
+        self.client.flushdb()
+        self.client.close()
+
+    def name(self) -> str:
+        return "ember-grpc"
+
+
 class ChromaClient(VectorClient):
     """chromadb client via HTTP API."""
 
@@ -308,14 +340,15 @@ def compute_recall(client: VectorClient, queries: np.ndarray,
 
 def main():
     parser = argparse.ArgumentParser(description="vector similarity benchmark")
-    parser.add_argument("--system", required=True, choices=["ember", "chromadb", "pgvector"])
+    parser.add_argument("--system", required=True, choices=["ember", "ember-grpc", "chromadb", "pgvector"])
     parser.add_argument("--mode", default="random", choices=["random", "sift"])
     parser.add_argument("--dim", type=int, default=128)
     parser.add_argument("--count", type=int, default=100000)
     parser.add_argument("--queries", type=int, default=1000)
     parser.add_argument("--k", type=int, default=10)
     parser.add_argument("--batch-size", type=int, default=500)
     parser.add_argument("--ember-port", type=int, default=6379)
+    parser.add_argument("--ember-grpc-port", type=int, default=6380)
     parser.add_argument("--chroma-port", type=int, default=8000)
     parser.add_argument("--pgvector-port", type=int, default=5432)
     parser.add_argument("--sift-dir", default="bench/vector_data")
@@ -325,6 +358,8 @@ def main():
     # create client
     if args.system == "ember":
         client = EmberClient(port=args.ember_port)
+    elif args.system == "ember-grpc":
+        client = EmberGrpcClient(port=args.ember_grpc_port)
     elif args.system == "chromadb":
         client = ChromaClient(port=args.chroma_port)
     elif args.system == "pgvector":

bench/bench-vector.sh

@@ -9,11 +9,13 @@
 # usage:
 #   bash bench/bench-vector.sh              # full comparison (100k vectors)
 #   bash bench/bench-vector.sh --ember-only # ember only (no docker needed)
+#   bash bench/bench-vector.sh --ember-grpc # include ember gRPC benchmark
 #   bash bench/bench-vector.sh --quick      # quick run (1k vectors)
 #   bash bench/bench-vector.sh --sift       # SIFT1M recall accuracy
 #
 # environment variables:
 #   EMBER_PORT       ember port             (default: 6379)
+#   EMBER_GRPC_PORT  ember gRPC port        (default: 6380)
 #   CHROMA_PORT      chromadb port          (default: 8000)
 #   PGVECTOR_PORT    pgvector port          (default: 5432)
 #   VECTOR_COUNT     base vector count      (default: 100000)
@@ -26,6 +28,7 @@ set -euo pipefail
 # --- configuration ---
 
 EMBER_PORT="${EMBER_PORT:-6379}"
+EMBER_GRPC_PORT="${EMBER_GRPC_PORT:-6380}"
 CHROMA_PORT="${CHROMA_PORT:-8000}"
 PGVECTOR_PORT="${PGVECTOR_PORT:-5432}"
 VECTOR_COUNT="${VECTOR_COUNT:-100000}"
@@ -40,12 +43,14 @@ BENCH_SCRIPT="bench/bench-vector.py"
 SIFT_DIR="bench/vector_data"
 
 EMBER_ONLY=false
+EMBER_GRPC=false
 QUICK_MODE=false
 SIFT_MODE=false
 
 for arg in "$@"; do
     case "$arg" in
         --ember-only) EMBER_ONLY=true ;;
+        --ember-grpc) EMBER_GRPC=true ;;
         --quick) QUICK_MODE=true ;;
         --sift) SIFT_MODE=true ;;
         *) echo "unknown flag: $arg"; exit 1 ;;
@@ -204,10 +209,20 @@ elif [[ "$EMBER_ONLY" == "false" ]] && ! python3 -c "import chromadb, psycopg2"
     pip install --quiet $REQUIRED_DEPS
 fi
 
+# install ember-py for gRPC benchmarks
+if [[ "$EMBER_GRPC" == "true" ]]; then
+    ensure_venv
+    pip install --quiet ./clients/ember-py
+fi
+
 # build ember if needed
 if [[ ! -x "$EMBER_BIN" ]]; then
-    echo "building ember-server with vector support..."
-    cargo build --release -p ember-server --features jemalloc,vector
+    FEATURES="jemalloc,vector"
+    if [[ "$EMBER_GRPC" == "true" ]]; then
+        FEATURES="$FEATURES,grpc"
+    fi
+    echo "building ember-server with features: $FEATURES..."
+    cargo build --release -p ember-server --features "$FEATURES"
 fi
 
 # download SIFT dataset if needed
@@ -250,6 +265,7 @@ BENCH_ARGS=(
     --k "$K"
     --batch-size "$BATCH_SIZE"
     --ember-port "$EMBER_PORT"
+    --ember-grpc-port "$EMBER_GRPC_PORT"
     --chroma-port "$CHROMA_PORT"
     --pgvector-port "$PGVECTOR_PORT"
     --sift-dir "$SIFT_DIR"
@@ -265,14 +281,29 @@ fi
 
 echo "--- ember ---"
 echo ""
-echo "  starting ember on port $EMBER_PORT..."
-"$EMBER_BIN" --port "$EMBER_PORT" > /dev/null 2>&1 &
+EMBER_ARGS=(--port "$EMBER_PORT")
+if [[ "$EMBER_GRPC" == "true" ]]; then
+    EMBER_ARGS+=(--grpc-port "$EMBER_GRPC_PORT")
+    echo "  starting ember on port $EMBER_PORT (gRPC: $EMBER_GRPC_PORT)..."
+else
+    echo "  starting ember on port $EMBER_PORT..."
+fi
+"$EMBER_BIN" "${EMBER_ARGS[@]}" > /dev/null 2>&1 &
 EMBER_PID=$!
 wait_for_ember "$EMBER_PORT"
 
 EMBER_JSON="/tmp/bench_vector_ember.json"
 python3 "$BENCH_SCRIPT" --system ember "${BENCH_ARGS[@]}" --output "$EMBER_JSON"
 
+# also run gRPC benchmark if requested
+EMBER_GRPC_JSON="/tmp/bench_vector_ember_grpc.json"
+if [[ "$EMBER_GRPC" == "true" ]]; then
+    echo ""
+    echo "--- ember (gRPC) ---"
+    echo ""
+    python3 "$BENCH_SCRIPT" --system ember-grpc "${BENCH_ARGS[@]}" --output "$EMBER_GRPC_JSON"
+fi
+
 EMBER_RSS=$(get_rss_mb "$EMBER_PID")
 echo "  memory (RSS): ${EMBER_RSS} MB"
 
@@ -362,6 +393,21 @@ E_P50=$(extract "$EMBER_JSON" "query.p50_ms")
 E_P95=$(extract "$EMBER_JSON" "query.p95_ms")
 E_P99=$(extract "$EMBER_JSON" "query.p99_ms")
 
+# ember gRPC results
+EG_INSERT="—"
+EG_QUERY="—"
+EG_P50="—"
+EG_P95="—"
+EG_P99="—"
+
+if [[ "$EMBER_GRPC" == "true" ]] && [[ -f "$EMBER_GRPC_JSON" ]]; then
+    EG_INSERT=$(extract "$EMBER_GRPC_JSON" "insert.throughput")
+    EG_QUERY=$(extract "$EMBER_GRPC_JSON" "query.throughput")
+    EG_P50=$(extract "$EMBER_GRPC_JSON" "query.p50_ms")
+    EG_P95=$(extract "$EMBER_GRPC_JSON" "query.p95_ms")
+    EG_P99=$(extract "$EMBER_GRPC_JSON" "query.p99_ms")
+fi
+
 # chromadb results
 C_INSERT="—"
 C_QUERY="—"
@@ -433,6 +479,21 @@ else
     printf "$fmt\n" "memory (MB)" "$EMBER_RSS" "$CHROMA_RSS" "$PGVECTOR_RSS"
 fi
 
+# grpc comparison (if enabled)
+if [[ "$EMBER_GRPC" == "true" ]]; then
+    echo ""
+    echo "=== ember RESP vs gRPC ==="
+    echo ""
+    fmt="%-24s %14s %14s"
+    printf "$fmt\n" "metric" "RESP" "gRPC"
+    printf "$fmt\n" "------" "----" "----"
+    printf "$fmt\n" "insert (vectors/sec)" "$E_INSERT" "$EG_INSERT"
+    printf "$fmt\n" "query (queries/sec)" "$E_QUERY" "$EG_QUERY"
+    printf "$fmt\n" "query p50 (ms)" "$E_P50" "$EG_P50"
+    printf "$fmt\n" "query p95 (ms)" "$E_P95" "$EG_P95"
+    printf "$fmt\n" "query p99 (ms)" "$E_P99" "$EG_P99"
+fi
+
 # recall (sift mode only)
 if [[ "$SIFT_MODE" == "true" ]]; then
     echo ""
@@ -460,13 +521,13 @@ echo ""
 
 RESULT_FILE="$RESULTS_DIR/${TIMESTAMP}-vector.csv"
 {
-    echo "metric,ember,chromadb,pgvector"
-    echo "insert_throughput,$E_INSERT,$C_INSERT,$P_INSERT"
-    echo "query_throughput,$E_QUERY,$C_QUERY,$P_QUERY"
-    echo "query_p50_ms,$E_P50,$C_P50,$P_P50"
-    echo "query_p95_ms,$E_P95,$C_P95,$P_P95"
-    echo "query_p99_ms,$E_P99,$C_P99,$P_P99"
-    echo "memory_mb,$EMBER_RSS,$CHROMA_RSS,$PGVECTOR_RSS"
+    echo "metric,ember,ember-grpc,chromadb,pgvector"
+    echo "insert_throughput,$E_INSERT,$EG_INSERT,$C_INSERT,$P_INSERT"
+    echo "query_throughput,$E_QUERY,$EG_QUERY,$C_QUERY,$P_QUERY"
+    echo "query_p50_ms,$E_P50,$EG_P50,$C_P50,$P_P50"
+    echo "query_p95_ms,$E_P95,$EG_P95,$C_P95,$P_P95"
+    echo "query_p99_ms,$E_P99,$EG_P99,$C_P99,$P_P99"
+    echo "memory_mb,$EMBER_RSS,—,$CHROMA_RSS,$PGVECTOR_RSS"
     if [[ "$SIFT_MODE" == "true" ]]; then
         E_RECALL=$(extract "$EMBER_JSON" "recall.recall_at_k")
         C_RECALL="—"

bench/setup-vm-vector.sh

@@ -14,7 +14,7 @@ echo "=== installing python dependencies ==="
 sudo apt-get update
 sudo apt-get install -y python3-pip python3-dev
 
-pip3 install --quiet redis chromadb psycopg2-binary numpy
+pip3 install --quiet redis chromadb psycopg2-binary numpy grpcio protobuf
 
 echo ""
 echo "=== installing docker ==="
@@ -35,11 +35,16 @@ sudo docker pull chromadb/chroma
 sudo docker pull pgvector/pgvector:pg17
 
 echo ""
-echo "=== rebuilding ember with vector support ==="
+echo "=== installing ember python client ==="
 
 cd ~/ember
+pip3 install --quiet ./clients/ember-py
+
+echo ""
+echo "=== rebuilding ember with vector + grpc support ==="
+
 source ~/.cargo/env
-cargo build --release -p ember-server --features jemalloc,vector
+cargo build --release -p ember-server --features jemalloc,vector,grpc
 
 echo ""
 echo "=== verifying ==="
@@ -54,5 +59,6 @@ echo ""
 echo "run benchmarks with:"
 echo "  ./bench/bench-vector.sh               # 100k random vectors"
 echo "  ./bench/bench-vector.sh --ember-only  # ember only (no docker)"
+echo "  ./bench/bench-vector.sh --ember-grpc  # include gRPC benchmark"
 echo "  ./bench/bench-vector.sh --quick       # quick sanity check"
 echo "  ./bench/bench-vector.sh --sift        # SIFT1M recall accuracy"