From c29c5b9e1e890943ffc03b035ce49e38a6a85481 Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sun, 15 Feb 2026 08:01:21 -0500 Subject: [PATCH] bench: add protobuf storage overhead benchmark new benchmark comparing PROTO.* commands against raw SET/GET: - bench-proto.sh: shell wrapper (requires --features protobuf, protoc) - bench-proto.py: python harness using redis-py - bench/proto/user.proto: simple test schema (name/age/email) measures: - PROTO.SET vs raw SET (same data, validated vs raw bytes) - PROTO.GET vs raw GET - PROTO.GETFIELD single-field read throughput - PROTO.SETFIELD single-field update throughput - overhead percentage for schema validation proto encoding is hand-rolled in python to avoid requiring the protobuf library at benchmark runtime. schema compilation uses protoc --descriptor_set_out at startup. --- bench/bench-proto.py | 285 +++++++++++++++++++++++++++++++++++++++++ bench/bench-proto.sh | 149 +++++++++++++++++++++ bench/proto/user.proto | 9 ++ 3 files changed, 443 insertions(+) create mode 100755 bench/bench-proto.py create mode 100755 bench/bench-proto.sh create mode 100644 bench/proto/user.proto diff --git a/bench/bench-proto.py b/bench/bench-proto.py new file mode 100755 index 00000000..3ce2f0ca --- /dev/null +++ b/bench/bench-proto.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +""" +protobuf storage overhead benchmark harness. + +compares PROTO.SET/GET/GETFIELD/SETFIELD against raw SET/GET to measure +the overhead of server-side schema validation and field-level access. + +called by bench-proto.sh with appropriate arguments. + +usage: + python3 bench/bench-proto.py --port 6379 --requests 100000 +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time + + +def compile_proto(proto_path): + """compile a .proto file to a FileDescriptorSet binary.""" + out = tempfile.NamedTemporaryFile(suffix=".pb", delete=False) + out.close() + subprocess.check_call([ + "protoc", + f"--descriptor_set_out={out.name}", + "--include_imports", + f"--proto_path={os.path.dirname(proto_path)}", + os.path.basename(proto_path), + ]) + with open(out.name, "rb") as f: + data = f.read() + os.unlink(out.name) + return data + + +def encode_user(name, age, email): + """encode a User message using protobuf wire format. + + hand-rolled to avoid requiring protobuf python library at runtime. + fields: name=1 (string), age=2 (int32), email=3 (string). + """ + buf = bytearray() + + # field 1: string (wire type 2 = length-delimited) + name_bytes = name.encode("utf-8") + buf.append(0x0a) # field 1, wire type 2 + buf.extend(_encode_varint(len(name_bytes))) + buf.extend(name_bytes) + + # field 2: int32 (wire type 0 = varint) + buf.append(0x10) # field 2, wire type 0 + buf.extend(_encode_varint(age)) + + # field 3: string (wire type 2 = length-delimited) + email_bytes = email.encode("utf-8") + buf.append(0x1a) # field 3, wire type 2 + buf.extend(_encode_varint(len(email_bytes))) + buf.extend(email_bytes) + + return bytes(buf) + + +def _encode_varint(value): + """encode an integer as a protobuf varint.""" + buf = bytearray() + while value > 0x7f: + buf.append((value & 0x7f) | 0x80) + value >>= 7 + buf.append(value & 0x7f) + return bytes(buf) + + +def percentile(sorted_data, p): + """compute p-th percentile from pre-sorted data.""" + if not sorted_data: + return 0.0 + k = (len(sorted_data) - 1) * (p / 100.0) + f = int(k) + c = min(f + 1, len(sorted_data) - 1) + return sorted_data[f] + (k - f) * (sorted_data[c] - sorted_data[f]) + + +def compute_stats(elapsed, latencies, count): + """compute throughput and latency stats.""" + ops_sec = count / elapsed if elapsed > 0 else 0 + sorted_lat = sorted(latencies) + p50 = percentile(sorted_lat, 50) * 1000 + p95 = percentile(sorted_lat, 95) * 1000 + p99 = percentile(sorted_lat, 99) * 1000 + return { + "ops_sec": round(ops_sec), + "p50_ms": round(p50, 3), + "p95_ms": round(p95, 3), + "p99_ms": round(p99, 3), + } + + +def bench_raw_set(r, keys, value, warmup=1000): + """raw SET throughput (no validation).""" + for i in range(warmup): + r.set(f"warmup:{i}", value) + + latencies = [] + start = time.perf_counter() + for k in keys: + t0 = time.perf_counter() + r.set(k, value) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + return elapsed, latencies + + +def bench_raw_get(r, keys, warmup=1000): + """raw GET throughput.""" + for i in range(warmup): + r.get(f"warmup:{i}") + + latencies = [] + start = time.perf_counter() + for k in keys: + t0 = time.perf_counter() + r.get(k) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + return elapsed, latencies + + +def bench_proto_set(r, keys, value, type_name, warmup=1000): + """PROTO.SET throughput (schema-validated).""" + for i in range(warmup): + r.execute_command("PROTO.SET", f"warmup:{i}", type_name, value) + + latencies = [] + start = time.perf_counter() + for k in keys: + t0 = time.perf_counter() + r.execute_command("PROTO.SET", k, type_name, value) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + return elapsed, latencies + + +def bench_proto_get(r, keys, warmup=1000): + """PROTO.GET throughput.""" + for i in range(warmup): + r.execute_command("PROTO.GET", f"warmup:{i}") + + latencies = [] + start = time.perf_counter() + for k in keys: + t0 = time.perf_counter() + r.execute_command("PROTO.GET", k) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + return elapsed, latencies + + +def bench_proto_getfield(r, keys, field, warmup=1000): + """PROTO.GETFIELD throughput (single field read).""" + for i in range(warmup): + r.execute_command("PROTO.GETFIELD", f"warmup:{i}", field) + + latencies = [] + start = time.perf_counter() + for k in keys: + t0 = time.perf_counter() + r.execute_command("PROTO.GETFIELD", k, field) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + return elapsed, latencies + + +def bench_proto_setfield(r, keys, field, value, warmup=1000): + """PROTO.SETFIELD throughput (single field update).""" + for i in range(warmup): + r.execute_command("PROTO.SETFIELD", f"warmup:{i}", field, value) + + latencies = [] + start = time.perf_counter() + for k in keys: + t0 = time.perf_counter() + r.execute_command("PROTO.SETFIELD", k, field, value) + latencies.append(time.perf_counter() - t0) + elapsed = time.perf_counter() - start + return elapsed, latencies + + +def main(): + parser = argparse.ArgumentParser(description="protobuf storage benchmark") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=6379) + parser.add_argument("--requests", type=int, default=100000) + parser.add_argument("--proto-file", default="bench/proto/user.proto") + parser.add_argument("--output", default=None, help="JSON output file") + args = parser.parse_args() + + import redis + r = redis.Redis(host=args.host, port=args.port) + + count = args.requests + type_name = "bench.User" + + # compile and register schema + print(" compiling proto schema...", file=sys.stderr) + descriptor = compile_proto(args.proto_file) + + print(" registering schema with PROTO.REGISTER...", file=sys.stderr) + r.execute_command("PROTO.REGISTER", "bench", descriptor) + + # generate test data + user_bytes = encode_user("alice", 30, "alice@example.com") + raw_keys = [f"raw:{i}" for i in range(count)] + proto_keys = [f"proto:{i}" for i in range(count)] + + results = {} + + # --- raw SET --- + print(" raw SET...", file=sys.stderr) + elapsed, latencies = bench_raw_set(r, raw_keys, user_bytes) + results["raw_set"] = compute_stats(elapsed, latencies, count) + print(f" {results['raw_set']['ops_sec']} ops/sec", file=sys.stderr) + + # --- PROTO.SET --- + print(" PROTO.SET...", file=sys.stderr) + elapsed, latencies = bench_proto_set(r, proto_keys, user_bytes, type_name) + results["proto_set"] = compute_stats(elapsed, latencies, count) + print(f" {results['proto_set']['ops_sec']} ops/sec", file=sys.stderr) + + # --- raw GET --- + print(" raw GET...", file=sys.stderr) + elapsed, latencies = bench_raw_get(r, raw_keys) + results["raw_get"] = compute_stats(elapsed, latencies, count) + print(f" {results['raw_get']['ops_sec']} ops/sec", file=sys.stderr) + + # --- PROTO.GET --- + print(" PROTO.GET...", file=sys.stderr) + elapsed, latencies = bench_proto_get(r, proto_keys) + results["proto_get"] = compute_stats(elapsed, latencies, count) + print(f" {results['proto_get']['ops_sec']} ops/sec", file=sys.stderr) + + # --- PROTO.GETFIELD --- + print(" PROTO.GETFIELD (name)...", file=sys.stderr) + elapsed, latencies = bench_proto_getfield(r, proto_keys, "name") + results["proto_getfield"] = compute_stats(elapsed, latencies, count) + print(f" {results['proto_getfield']['ops_sec']} ops/sec", file=sys.stderr) + + # --- PROTO.SETFIELD --- + print(" PROTO.SETFIELD (age)...", file=sys.stderr) + elapsed, latencies = bench_proto_setfield(r, proto_keys, "age", "31") + results["proto_setfield"] = compute_stats(elapsed, latencies, count) + print(f" {results['proto_setfield']['ops_sec']} ops/sec", file=sys.stderr) + + # --- compute overhead --- + if results["raw_set"]["ops_sec"] > 0: + results["set_overhead_pct"] = round( + (1 - results["proto_set"]["ops_sec"] / results["raw_set"]["ops_sec"]) * 100, 1 + ) + if results["raw_get"]["ops_sec"] > 0: + results["get_overhead_pct"] = round( + (1 - results["proto_get"]["ops_sec"] / results["raw_get"]["ops_sec"]) * 100, 1 + ) + + results["config"] = { + "requests": count, + "type_name": type_name, + "message_bytes": len(user_bytes), + } + + if args.output: + with open(args.output, "w") as f: + json.dump(results, f, indent=2) + print(f" results saved to {args.output}", file=sys.stderr) + + json.dump(results, sys.stdout, indent=2) + print() + + r.close() + + +if __name__ == "__main__": + main() diff --git a/bench/bench-proto.sh b/bench/bench-proto.sh new file mode 100755 index 00000000..26fbbba9 --- /dev/null +++ b/bench/bench-proto.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# +# protobuf storage overhead benchmark. +# +# compares PROTO.SET/GET/GETFIELD/SETFIELD against raw SET/GET to measure +# the cost of server-side schema validation and field-level access. +# +# usage: +# bash bench/bench-proto.sh +# bash bench/bench-proto.sh --quick # 10k requests +# +# requirements: +# - ember built with: cargo build --release -p ember-server --features jemalloc,protobuf +# - protoc on PATH (for compiling the test schema) +# - python3 with redis-py +# +# environment variables: +# EMBER_PORT ember port (default: 6379) +# BENCH_REQUESTS requests per test (default: 100000) + +set -euo pipefail + +EMBER_PORT="${EMBER_PORT:-6379}" +REQUESTS="${BENCH_REQUESTS:-100000}" +EMBER_BIN="${EMBER_BIN:-./target/release/ember-server}" +RESULTS_DIR="bench/results" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +BENCH_SCRIPT="bench/bench-proto.py" + +QUICK_MODE=false +for arg in "$@"; do + case "$arg" in + --quick) QUICK_MODE=true ;; + *) echo "unknown flag: $arg"; exit 1 ;; + esac +done + +if [[ "$QUICK_MODE" == "true" ]]; then + REQUESTS=10000 +fi + +# --- cleanup --- + +EMBER_PID="" + +cleanup() { + [[ -n "$EMBER_PID" ]] && kill "$EMBER_PID" 2>/dev/null && wait "$EMBER_PID" 2>/dev/null || true +} +trap cleanup EXIT + +# --- checks --- + +if [[ ! -x "$EMBER_BIN" ]]; then + echo "error: ember-server not found at $EMBER_BIN" >&2 + echo "build with: cargo build --release -p ember-server --features jemalloc,protobuf" >&2 + exit 1 +fi + +if ! command -v protoc &> /dev/null; then + echo "error: protoc not found. install protobuf compiler." >&2 + echo " brew install protobuf # macOS" >&2 + echo " apt install protobuf-compiler # ubuntu" >&2 + exit 1 +fi + +if ! command -v python3 &> /dev/null; then + echo "error: python3 required" >&2 + exit 1 +fi + +# set up venv if needed +VENV_DIR=".bench-venv" + +if ! python3 -c "import redis" 2>/dev/null; then + if [[ ! -d "$VENV_DIR" ]]; then + echo "creating python venv for benchmark dependencies..." + python3 -m venv "$VENV_DIR" + fi + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + pip install --quiet redis +fi + +# --- start server --- + +echo "" +echo "=== protobuf storage overhead benchmark ===" +echo "requests: $REQUESTS" +echo "port: $EMBER_PORT" +echo "" + +echo "starting ember (with protobuf support) on port $EMBER_PORT..." +"$EMBER_BIN" --port "$EMBER_PORT" > /dev/null 2>&1 & +EMBER_PID=$! + +retries=50 +while ! redis-cli -p "$EMBER_PORT" ping > /dev/null 2>&1; do + retries=$((retries - 1)) + if [[ $retries -le 0 ]]; then + echo "error: ember did not start on port $EMBER_PORT" >&2 + exit 1 + fi + sleep 0.1 +done + +echo "" + +# --- run benchmark --- + +mkdir -p "$RESULTS_DIR" +RESULT_JSON="$RESULTS_DIR/${TIMESTAMP}-proto.json" + +python3 "$BENCH_SCRIPT" \ + --port "$EMBER_PORT" \ + --requests "$REQUESTS" \ + --output "$RESULT_JSON" \ + > /dev/null + +# --- display results --- + +echo "" +echo "========================================================================" +echo " protobuf storage overhead results" +echo "========================================================================" +echo "" + +python3 -c " +import json +d = json.load(open('$RESULT_JSON')) + +fmt = '%-28s %14s %14s %14s' +print(fmt % ('test', 'ops/sec', 'p50 (ms)', 'p99 (ms)')) +print(fmt % ('----', '-------', '--------', '--------')) +print(fmt % ('raw SET', d['raw_set']['ops_sec'], d['raw_set']['p50_ms'], d['raw_set']['p99_ms'])) +print(fmt % ('PROTO.SET', d['proto_set']['ops_sec'], d['proto_set']['p50_ms'], d['proto_set']['p99_ms'])) +print(fmt % ('raw GET', d['raw_get']['ops_sec'], d['raw_get']['p50_ms'], d['raw_get']['p99_ms'])) +print(fmt % ('PROTO.GET', d['proto_get']['ops_sec'], d['proto_get']['p50_ms'], d['proto_get']['p99_ms'])) +print(fmt % ('PROTO.GETFIELD', d['proto_getfield']['ops_sec'], d['proto_getfield']['p50_ms'], d['proto_getfield']['p99_ms'])) +print(fmt % ('PROTO.SETFIELD', d['proto_setfield']['ops_sec'], d['proto_setfield']['p50_ms'], d['proto_setfield']['p99_ms'])) + +print() +if 'set_overhead_pct' in d: + print(f' SET overhead: {d[\"set_overhead_pct\"]}%') +if 'get_overhead_pct' in d: + print(f' GET overhead: {d[\"get_overhead_pct\"]}%') +" + +echo "" +echo "results saved to $RESULT_JSON" diff --git a/bench/proto/user.proto b/bench/proto/user.proto new file mode 100644 index 00000000..5d67c877 --- /dev/null +++ b/bench/proto/user.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package bench; + +message User { + string name = 1; + int32 age = 2; + string email = 3; +}