diff --git a/bench/bench-grpc.py b/bench/bench-grpc.py new file mode 100755 index 00000000..69cd3518 --- /dev/null +++ b/bench/bench-grpc.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +""" +gRPC vs RESP3 benchmark harness. + +measures SET/GET throughput and latency for ember's gRPC interface +compared to the standard RESP3 protocol. called by bench-grpc.sh. + +usage: + python3 bench/bench-grpc.py --grpc-addr localhost:6380 --resp-port 6379 +""" + +import argparse +import json +import time +import sys +import os +import random +import string +import statistics + + +def random_value(size: int) -> bytes: + """generate a random byte string of the given size.""" + return bytes(random.getrandbits(8) for _ in range(size)) + + +def random_key(prefix: str, i: int) -> str: + return f"{prefix}:{i}" + + +# --------------------------------------------------------------------------- +# RESP3 benchmarks (using redis-py) +# --------------------------------------------------------------------------- + +def bench_resp_set(host, port, keys, value, warmup=1000): + """sequential SET throughput via RESP3.""" + import redis + r = redis.Redis(host=host, port=port) + + # warmup + for i in range(warmup): + r.set(f"warmup:{i}", value) + + latencies = [] + start = time.perf_counter() + for i in range(len(keys)): + t0 = time.perf_counter() + r.set(keys[i], value) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + + r.close() + return elapsed, latencies + + +def bench_resp_get(host, port, keys, warmup=1000): + """sequential GET throughput via RESP3.""" + import redis + r = redis.Redis(host=host, port=port) + + # warmup + for i in range(warmup): + r.get(f"warmup:{i}") + + latencies = [] + start = time.perf_counter() + for i in range(len(keys)): + t0 = time.perf_counter() + r.get(keys[i]) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + + r.close() + return elapsed, latencies + + +def bench_resp_pipeline(host, port, keys, value, batch_size=100): + """pipelined SET+GET throughput via RESP3.""" + import redis + r = redis.Redis(host=host, port=port) + + # SET pipeline + set_latencies = [] + set_start = time.perf_counter() + for batch_start in range(0, len(keys), batch_size): + batch_end = min(batch_start + batch_size, len(keys)) + t0 = time.perf_counter() + pipe = r.pipeline(transaction=False) + for i in range(batch_start, batch_end): + pipe.set(keys[i], value) + pipe.execute() + batch_time = time.perf_counter() - t0 + per_op = batch_time / (batch_end - batch_start) + set_latencies.extend([per_op] * (batch_end - batch_start)) + set_elapsed = time.perf_counter() - set_start + + # GET pipeline + get_latencies = [] + get_start = time.perf_counter() + for batch_start in range(0, len(keys), batch_size): + batch_end = min(batch_start + batch_size, len(keys)) + t0 = time.perf_counter() + pipe = r.pipeline(transaction=False) + for i in range(batch_start, batch_end): + pipe.get(keys[i]) + pipe.execute() + batch_time = time.perf_counter() - t0 + per_op = batch_time / (batch_end - batch_start) + get_latencies.extend([per_op] * (batch_end - batch_start)) + get_elapsed = time.perf_counter() - get_start + + r.close() + return set_elapsed, set_latencies, get_elapsed, get_latencies + + +# --------------------------------------------------------------------------- +# gRPC benchmarks (using ember-py) +# --------------------------------------------------------------------------- + +def bench_grpc_set(addr, keys, value, warmup=1000): + """sequential SET throughput via gRPC unary calls.""" + from ember import EmberClient + client = EmberClient(addr) + + # warmup + for i in range(warmup): + client.set(f"warmup:{i}", value) + + latencies = [] + start = time.perf_counter() + for i in range(len(keys)): + t0 = time.perf_counter() + client.set(keys[i], value) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + + client.close() + return elapsed, latencies + + +def bench_grpc_get(addr, keys, warmup=1000): + """sequential GET throughput via gRPC unary calls.""" + from ember import EmberClient + client = EmberClient(addr) + + # warmup + for i in range(warmup): + client.get(f"warmup:{i}") + + latencies = [] + start = time.perf_counter() + for i in range(len(keys)): + t0 = time.perf_counter() + client.get(keys[i]) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + + client.close() + return elapsed, latencies + + +# --------------------------------------------------------------------------- +# stats helpers +# --------------------------------------------------------------------------- + +def percentile(data, p): + """compute the p-th percentile of a list.""" + if not data: + return 0.0 + sorted_data = sorted(data) + k = (len(sorted_data) - 1) * (p / 100.0) + f = int(k) + c = f + 1 + if c >= len(sorted_data): + return sorted_data[f] + return sorted_data[f] + (k - f) * (sorted_data[c] - sorted_data[f]) + + +def compute_stats(elapsed, latencies, count): + """compute throughput and latency stats.""" + ops_sec = count / elapsed if elapsed > 0 else 0 + p50 = percentile(latencies, 50) * 1000 # ms + p95 = percentile(latencies, 95) * 1000 + p99 = percentile(latencies, 99) * 1000 + return { + "ops_sec": round(ops_sec), + "p50_ms": round(p50, 3), + "p95_ms": round(p95, 3), + "p99_ms": round(p99, 3), + } + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="gRPC vs RESP3 benchmark") + parser.add_argument("--resp-host", default="127.0.0.1") + parser.add_argument("--resp-port", type=int, default=6379) + parser.add_argument("--grpc-addr", default="127.0.0.1:6380") + parser.add_argument("--requests", type=int, default=100000) + parser.add_argument("--value-size", type=int, default=64) + parser.add_argument("--pipeline-batch", type=int, default=100) + parser.add_argument("--output", default=None, help="JSON output file") + args = parser.parse_args() + + count = args.requests + value = random_value(args.value_size) + keys = [random_key("bench", i) for i in range(count)] + + # pre-populate keys for GET tests + print(f" pre-populating {count} keys via RESP3...", file=sys.stderr) + import redis + r = redis.Redis(host=args.resp_host, port=args.resp_port) + pipe = r.pipeline(transaction=False) + for k in keys: + pipe.set(k, value) + pipe.execute() + r.close() + + results = {} + + # --- RESP3 sequential --- + print(" RESP3 sequential SET...", file=sys.stderr) + elapsed, latencies = bench_resp_set(args.resp_host, args.resp_port, keys, value) + results["resp_set"] = compute_stats(elapsed, latencies, count) + print(f" {results['resp_set']['ops_sec']} ops/sec", file=sys.stderr) + + print(" RESP3 sequential GET...", file=sys.stderr) + elapsed, latencies = bench_resp_get(args.resp_host, args.resp_port, keys) + results["resp_get"] = compute_stats(elapsed, latencies, count) + print(f" {results['resp_get']['ops_sec']} ops/sec", file=sys.stderr) + + # --- RESP3 pipelined --- + print(f" RESP3 pipelined (batch={args.pipeline_batch})...", file=sys.stderr) + se, sl, ge, gl = bench_resp_pipeline( + args.resp_host, args.resp_port, keys, value, args.pipeline_batch + ) + results["resp_pipeline_set"] = compute_stats(se, sl, count) + results["resp_pipeline_get"] = compute_stats(ge, gl, count) + print(f" SET: {results['resp_pipeline_set']['ops_sec']} ops/sec", file=sys.stderr) + print(f" GET: {results['resp_pipeline_get']['ops_sec']} ops/sec", file=sys.stderr) + + # --- gRPC unary --- + print(" gRPC unary SET...", file=sys.stderr) + elapsed, latencies = bench_grpc_set(args.grpc_addr, keys, value) + results["grpc_set"] = compute_stats(elapsed, latencies, count) + print(f" {results['grpc_set']['ops_sec']} ops/sec", file=sys.stderr) + + print(" gRPC unary GET...", file=sys.stderr) + elapsed, latencies = bench_grpc_get(args.grpc_addr, keys) + results["grpc_get"] = compute_stats(elapsed, latencies, count) + print(f" {results['grpc_get']['ops_sec']} ops/sec", file=sys.stderr) + + # --- output --- + results["config"] = { + "requests": count, + "value_size": args.value_size, + "pipeline_batch": args.pipeline_batch, + } + + if args.output: + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + print(f" results saved to {args.output}", file=sys.stderr) + + # return results as JSON to stdout for the shell wrapper + json.dump(results, sys.stdout, indent=2) + print() + + +if __name__ == "__main__": + main() diff --git a/bench/bench-grpc.sh b/bench/bench-grpc.sh new file mode 100755 index 00000000..4492fe44 --- /dev/null +++ b/bench/bench-grpc.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# +# gRPC vs RESP3 benchmark for standard operations (SET/GET). +# +# compares ember's gRPC interface against RESP3 for sequential and pipelined +# workloads using the ember-py gRPC client and redis-py. +# +# usage: +# bash bench/bench-grpc.sh +# bash bench/bench-grpc.sh --quick # 10k requests instead of 100k +# +# requirements: +# - ember built with: cargo build --release -p ember-server --features jemalloc,grpc +# - python3 with redis and ember-py installed +# +# environment variables: +# EMBER_PORT RESP3 port (default: 6379) +# EMBER_GRPC_PORT gRPC port (default: 6380) +# BENCH_REQUESTS requests per test (default: 100000) +# VALUE_SIZE value size in bytes (default: 64) + +set -euo pipefail + +EMBER_PORT="${EMBER_PORT:-6379}" +EMBER_GRPC_PORT="${EMBER_GRPC_PORT:-6380}" +REQUESTS="${BENCH_REQUESTS:-100000}" +VALUE_SIZE="${VALUE_SIZE:-64}" +EMBER_BIN="${EMBER_BIN:-./target/release/ember-server}" +RESULTS_DIR="bench/results" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +BENCH_SCRIPT="bench/bench-grpc.py" + +QUICK_MODE=false +for arg in "$@"; do + case "$arg" in + --quick) QUICK_MODE=true ;; + *) echo "unknown flag: $arg"; exit 1 ;; + esac +done + +if [[ "$QUICK_MODE" == "true" ]]; then + REQUESTS=10000 +fi + +# --- cleanup --- + +EMBER_PID="" + +cleanup() { + [[ -n "$EMBER_PID" ]] && kill "$EMBER_PID" 2>/dev/null && wait "$EMBER_PID" 2>/dev/null || true +} +trap cleanup EXIT + +# --- checks --- + +if [[ ! -x "$EMBER_BIN" ]]; then + echo "error: ember-server not found at $EMBER_BIN" >&2 + echo "build with: cargo build --release -p ember-server --features jemalloc,grpc" >&2 + exit 1 +fi + +if ! command -v python3 &> /dev/null; then + echo "error: python3 required" >&2 + exit 1 +fi + +# set up venv if needed +VENV_DIR=".bench-venv" + +ensure_venv() { + if [[ ! -d "$VENV_DIR" ]]; then + echo "creating python venv for benchmark dependencies..." + python3 -m venv "$VENV_DIR" + fi + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" +} + +if ! python3 -c "import redis" 2>/dev/null; then + ensure_venv + pip install --quiet redis +fi + +# install ember-py for gRPC client +ensure_venv +pip install --quiet ./clients/ember-py + +# --- start server --- + +echo "" +echo "=== gRPC vs RESP3 benchmark ===" +echo "requests: $REQUESTS" +echo "value size: ${VALUE_SIZE}B" +echo "RESP port: $EMBER_PORT" +echo "gRPC port: $EMBER_GRPC_PORT" +echo "" + +echo "starting ember (RESP: $EMBER_PORT, gRPC: $EMBER_GRPC_PORT)..." +"$EMBER_BIN" --port "$EMBER_PORT" --grpc-port "$EMBER_GRPC_PORT" > /dev/null 2>&1 & +EMBER_PID=$! + +# wait for RESP +retries=50 +while ! redis-cli -p "$EMBER_PORT" ping > /dev/null 2>&1; do + retries=$((retries - 1)) + if [[ $retries -le 0 ]]; then + echo "error: ember did not start on port $EMBER_PORT" >&2 + exit 1 + fi + sleep 0.1 +done + +# brief extra wait for gRPC to bind +sleep 1 + +echo "" + +# --- run python benchmark --- + +mkdir -p "$RESULTS_DIR" +RESULT_JSON="$RESULTS_DIR/${TIMESTAMP}-grpc.json" + +python3 "$BENCH_SCRIPT" \ + --resp-port "$EMBER_PORT" \ + --grpc-addr "127.0.0.1:$EMBER_GRPC_PORT" \ + --requests "$REQUESTS" \ + --value-size "$VALUE_SIZE" \ + --output "$RESULT_JSON" \ + > /dev/null + +# --- parse and display results --- + +extract() { + python3 -c " +import json, sys +d = json.load(open('$RESULT_JSON')) +keys = '$1'.split('.') +v = d +for k in keys: + v = v[k] +print(v) +" 2>/dev/null || echo "—" +} + +echo "" +echo "========================================================================" +echo " gRPC vs RESP3 benchmark results" +echo "========================================================================" +echo "" + +fmt="%-28s %14s %14s %14s" +printf "$fmt\n" "test" "ops/sec" "p50 (ms)" "p99 (ms)" +printf "$fmt\n" "----" "-------" "--------" "--------" +printf "$fmt\n" "RESP3 SET (sequential)" "$(extract resp_set.ops_sec)" "$(extract resp_set.p50_ms)" "$(extract resp_set.p99_ms)" +printf "$fmt\n" "RESP3 GET (sequential)" "$(extract resp_get.ops_sec)" "$(extract resp_get.p50_ms)" "$(extract resp_get.p99_ms)" +printf "$fmt\n" "RESP3 SET (pipelined)" "$(extract resp_pipeline_set.ops_sec)" "$(extract resp_pipeline_set.p50_ms)" "$(extract resp_pipeline_set.p99_ms)" +printf "$fmt\n" "RESP3 GET (pipelined)" "$(extract resp_pipeline_get.ops_sec)" "$(extract resp_pipeline_get.p50_ms)" "$(extract resp_pipeline_get.p99_ms)" +printf "$fmt\n" "gRPC SET (unary)" "$(extract grpc_set.ops_sec)" "$(extract grpc_set.p50_ms)" "$(extract grpc_set.p99_ms)" +printf "$fmt\n" "gRPC GET (unary)" "$(extract grpc_get.ops_sec)" "$(extract grpc_get.p50_ms)" "$(extract grpc_get.p99_ms)" + +echo "" +echo "results saved to $RESULT_JSON"