kacy · kacy · Feb 7, 2026 · Feb 7, 2026 · Feb 7, 2026
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,5 @@ perf.data
 perf.data.old
 flamegraph.svg
 criterion/
+bench/results/*.csv
+bench/results/*.json
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
 .PHONY: build release test fmt fmt-check clippy check clean docker-build \
        release-patch release-minor release-major github-release \
-       publish publish-dry-run bench bench-core bench-protocol
+       publish publish-dry-run bench bench-core bench-protocol bench-compare bench-quick
 
 # extract the workspace version from the root Cargo.toml
 VERSION = $(shell sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml)
@@ -40,6 +40,14 @@ bench-core:
 bench-protocol:
 	cargo bench -p ember-protocol
 
+bench-compare:
+	cargo build --release -p ember-server
+	bash bench/compare-redis.sh
+
+bench-quick:
+	cargo build --release -p ember-server
+	bash bench/compare-redis.sh --ember-only
+
 # --- versioning & releases ---
 #
 # usage:

diff --git a/bench/README.md b/bench/README.md
@@ -0,0 +1,59 @@
+# benchmarks
+
+performance benchmarks for ember, using `redis-benchmark` for system-level throughput/latency testing.
+
+## prerequisites
+
+- `redis-benchmark` (comes with redis: `brew install redis` on macOS)
+- `redis-server` (optional, only needed for comparison mode)
+- ember built in release mode (`cargo build --release -p ember-server`)
+
+## running
+
+```bash
+# full comparison against redis
+make bench-compare
+
+# ember only (no redis needed)
+make bench-quick
+
+# with custom parameters
+BENCH_REQUESTS=500000 BENCH_CLIENTS=100 bash bench/compare-redis.sh
+
+# JSON output for CI
+bash bench/compare-redis.sh --json
+```
+
+## what's measured
+
+| test | what it measures |
+|------|-----------------|
+| SET (3B, P=16) | peak write throughput, pipelined |
+| GET (3B, P=16) | peak read throughput, pipelined |
+| SET/GET (64B, P=16) | throughput with realistic value sizes |
+| SET/GET (1KB, P=16) | throughput with larger payloads |
+| SET/GET (3B, P=1) | single-request latency (no pipelining) |
+
+## environment variables
+
+| variable | default | description |
+|----------|---------|-------------|
+| `EMBER_PORT` | 6379 | port for ember server |
+| `REDIS_PORT` | 6399 | port for redis server |
+| `BENCH_REQUESTS` | 100000 | total requests per test |
+| `BENCH_CLIENTS` | 50 | concurrent client connections |
+| `BENCH_PIPELINE` | 16 | pipeline depth for P>1 tests |
+| `EMBER_BIN` | ./target/release/ember-server | path to ember binary |
+
+## results
+
+raw CSV results are saved to `bench/results/` with timestamps. these are gitignored to keep the repo clean — copy them elsewhere for long-term tracking.
+
+## micro-benchmarks
+
+for criterion micro-benchmarks (keyspace, engine, protocol), see the `benches/` directories in `ember-core` and `ember-protocol`:
+
+```bash
+cargo bench -p emberkv-core   # keyspace + engine benchmarks
+cargo bench -p ember-protocol # RESP3 parse/serialize benchmarks
+```
diff --git a/bench/compare-redis.sh b/bench/compare-redis.sh
@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+#
+# Runs redis-benchmark against both Ember and Redis (optionally) and
+# prints a side-by-side throughput comparison. Saves raw results to
+# bench/results/ for historical tracking.
+#
+# Usage:
+#   bash bench/compare-redis.sh              # full comparison
+#   bash bench/compare-redis.sh --ember-only # skip redis
+#   bash bench/compare-redis.sh --json       # JSON output for CI
+#
+# Environment variables:
+#   EMBER_PORT       ember server port           (default: 6379)
+#   REDIS_PORT       redis server port           (default: 6399)
+#   BENCH_REQUESTS   requests per test           (default: 100000)
+#   BENCH_CLIENTS    concurrent clients          (default: 50)
+#   BENCH_PIPELINE   pipeline depth for P>1 tests (default: 16)
+#   EMBER_BIN        path to ember-server binary (default: ./target/release/ember-server)
+
+set -euo pipefail
+
+# --- configuration ---
+
+EMBER_PORT="${EMBER_PORT:-6379}"
+REDIS_PORT="${REDIS_PORT:-6399}"
+REQUESTS="${BENCH_REQUESTS:-100000}"
+CLIENTS="${BENCH_CLIENTS:-50}"
+PIPELINE="${BENCH_PIPELINE:-16}"
+EMBER_BIN="${EMBER_BIN:-./target/release/ember-server}"
+RESULTS_DIR="bench/results"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+
+EMBER_ONLY=false
+JSON_OUTPUT=false
+
+for arg in "$@"; do
+    case "$arg" in
+        --ember-only) EMBER_ONLY=true ;;
+        --json) JSON_OUTPUT=true ;;
+        *) echo "unknown flag: $arg"; exit 1 ;;
+    esac
+done
+
+# --- helpers ---
+
+EMBER_PID=""
+REDIS_PID=""
+
+cleanup() {
+    [[ -n "$EMBER_PID" ]] && kill "$EMBER_PID" 2>/dev/null && wait "$EMBER_PID" 2>/dev/null || true
+    [[ -n "$REDIS_PID" ]] && kill "$REDIS_PID" 2>/dev/null && wait "$REDIS_PID" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+wait_for_server() {
+    local port=$1
+    local name=$2
+    local retries=30
+    while ! redis-cli -p "$port" ping > /dev/null 2>&1; do
+        retries=$((retries - 1))
+        if [[ $retries -le 0 ]]; then
+            echo "error: $name did not start on port $port" >&2
+            exit 1
+        fi
+        sleep 0.1
+    done
+}
+
+# extract requests/sec from redis-benchmark --csv output
+# format: "TEST","rps"\n e.g. "SET","123456.78"
+extract_rps() {
+    local csv_output=$1
+    local test_name=$2
+    echo "$csv_output" | grep "\"$test_name\"" | cut -d',' -f2 | tr -d '"' | cut -d'.' -f1
+}
+
+run_benchmark() {
+    local port=$1
+    local data_size=$2
+    local pipeline=$3
+    redis-benchmark -p "$port" -t set,get -n "$REQUESTS" -c "$CLIENTS" -P "$pipeline" -d "$data_size" --csv -q 2>/dev/null
+}
+
+# pre-populate keys so GET benchmarks have data to read
+populate_keys() {
+    local port=$1
+    redis-benchmark -p "$port" -t set -n "$REQUESTS" -c "$CLIENTS" -P "$PIPELINE" -d 3 -q > /dev/null 2>&1
+}
+
+# --- checks ---
+
+if ! command -v redis-benchmark &> /dev/null; then
+    echo "error: redis-benchmark not found. install redis tools first." >&2
+    echo "  brew install redis    # macOS" >&2
+    echo "  apt install redis-tools  # debian/ubuntu" >&2
+    exit 1
+fi
+
+if [[ ! -x "$EMBER_BIN" ]]; then
+    echo "building ember-server in release mode..."
+    cargo build --release -p ember-server
+fi
+
+if [[ "$EMBER_ONLY" == "false" ]] && ! command -v redis-server &> /dev/null; then
+    echo "warning: redis-server not found, running ember-only mode" >&2
+    EMBER_ONLY=true
+fi
+
+mkdir -p "$RESULTS_DIR"
+
+# --- start servers ---
+
+echo "starting ember on port $EMBER_PORT..."
+"$EMBER_BIN" --port "$EMBER_PORT" &
+EMBER_PID=$!
+wait_for_server "$EMBER_PORT" "ember"
+
+if [[ "$EMBER_ONLY" == "false" ]]; then
+    echo "starting redis on port $REDIS_PORT..."
+    redis-server --port "$REDIS_PORT" --save "" --appendonly no --loglevel warning &
+    REDIS_PID=$!
+    wait_for_server "$REDIS_PORT" "redis"
+fi
+
+# --- define test matrix ---
+# format: "label:data_size:pipeline"
+TESTS=(
+    "SET (3B, P=$PIPELINE):3:$PIPELINE"
+    "GET (3B, P=$PIPELINE):3:$PIPELINE"
+    "SET (64B, P=$PIPELINE):64:$PIPELINE"
+    "GET (64B, P=$PIPELINE):64:$PIPELINE"
+    "SET (1KB, P=$PIPELINE):1024:$PIPELINE"
+    "GET (1KB, P=$PIPELINE):1024:$PIPELINE"
+    "SET (3B, P=1):3:1"
+    "GET (3B, P=1):3:1"
+)
+
+# --- run benchmarks ---
+
+echo ""
+echo "running benchmarks ($REQUESTS requests, $CLIENTS clients)..."
+echo ""
+
+# pre-populate
+populate_keys "$EMBER_PORT"
+if [[ "$EMBER_ONLY" == "false" ]]; then
+    populate_keys "$REDIS_PORT"
+fi
+
+declare -a LABELS=()
+declare -a EMBER_RESULTS=()
+declare -a REDIS_RESULTS=()
+
+for test_spec in "${TESTS[@]}"; do
+    IFS=':' read -r label data_size pipeline <<< "$test_spec"
+    LABELS+=("$label")
+
+    # determine if this is SET or GET from the label
+    if [[ "$label" == SET* ]]; then
+        test_type="SET"
+    else
+        test_type="GET"
+    fi
+
+    # run ember
+    csv=$(run_benchmark "$EMBER_PORT" "$data_size" "$pipeline")
+    ember_rps=$(extract_rps "$csv" "$test_type")
+    EMBER_RESULTS+=("${ember_rps:-0}")
+
+    # run redis
+    if [[ "$EMBER_ONLY" == "false" ]]; then
+        csv=$(run_benchmark "$REDIS_PORT" "$data_size" "$pipeline")
+        redis_rps=$(extract_rps "$csv" "$test_type")
+        REDIS_RESULTS+=("${redis_rps:-0}")
+    fi
+done
+
+# --- output results ---
+
+if [[ "$JSON_OUTPUT" == "true" ]]; then
+    echo "{"
+    echo "  \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\","
+    echo "  \"config\": { \"requests\": $REQUESTS, \"clients\": $CLIENTS, \"pipeline\": $PIPELINE },"
+    echo "  \"ember\": {"
+    for i in "${!LABELS[@]}"; do
+        comma=""
+        [[ $i -lt $((${#LABELS[@]} - 1)) ]] && comma=","
+        echo "    \"${LABELS[$i]}\": ${EMBER_RESULTS[$i]}$comma"
+    done
+    echo "  }"
+    if [[ "$EMBER_ONLY" == "false" ]]; then
+        echo "  ,\"redis\": {"
+        for i in "${!LABELS[@]}"; do
+            comma=""
+            [[ $i -lt $((${#LABELS[@]} - 1)) ]] && comma=","
+            echo "    \"${LABELS[$i]}\": ${REDIS_RESULTS[$i]}$comma"
+        done
+        echo "  }"
+    fi
+    echo "}"
+else
+    DATE=$(date +%Y-%m-%d)
+
+    if [[ "$EMBER_ONLY" == "false" ]]; then
+        echo "ember vs redis benchmark — $DATE"
+        echo "config: $REQUESTS requests, $CLIENTS clients"
+        echo ""
+        printf "%-24s %12s %12s %8s\n" "test" "ember (rps)" "redis (rps)" "ratio"
+        printf "%-24s %12s %12s %8s\n" "----" "-----------" "-----------" "-----"
+
+        for i in "${!LABELS[@]}"; do
+            e=${EMBER_RESULTS[$i]}
+            r=${REDIS_RESULTS[$i]}
+            if [[ "$r" -gt 0 ]]; then
+                # integer ratio with one decimal: (e * 10 / r) then insert decimal
+                ratio_x10=$(( (e * 10 + r / 2) / r ))
+                ratio_int=$((ratio_x10 / 10))
+                ratio_frac=$((ratio_x10 % 10))
+                ratio="${ratio_int}.${ratio_frac}x"
+            else
+                ratio="n/a"
+            fi
+            printf "%-24s %12s %12s %8s\n" "${LABELS[$i]}" \
+                "$(printf "%'d" "$e")" \
+                "$(printf "%'d" "$r")" \
+                "$ratio"
+        done
+    else
+        echo "ember benchmark — $DATE"
+        echo "config: $REQUESTS requests, $CLIENTS clients"
+        echo ""
+        printf "%-24s %12s\n" "test" "ember (rps)"
+        printf "%-24s %12s\n" "----" "-----------"
+
+        for i in "${!LABELS[@]}"; do
+            printf "%-24s %12s\n" "${LABELS[$i]}" "$(printf "%'d" "${EMBER_RESULTS[$i]}")"
+        done
+    fi
+fi
+
+# --- save raw results ---
+
+RESULT_FILE="$RESULTS_DIR/$TIMESTAMP.csv"
+{
+    echo "test,ember_rps,redis_rps"
+    for i in "${!LABELS[@]}"; do
+        redis_val="${REDIS_RESULTS[$i]:-}"
+        echo "${LABELS[$i]},${EMBER_RESULTS[$i]},${redis_val}"
+    done
+} > "$RESULT_FILE"
+
+echo ""
+echo "raw results saved to $RESULT_FILE"
diff --git a/bench/results/.gitkeep b/bench/results/.gitkeep