chore: add benchmark scripts

scripts/README.md

@@ -0,0 +1,101 @@
+# benchmark scripts
+
+scripts for benchmarking ember against redis.
+
+## quick start (local)
+
+```bash
+# build ember
+cargo build --release --features jemalloc
+
+# run quick sanity check
+./scripts/bench-quick.sh
+
+# run full comparison (requires redis)
+./scripts/bench.sh
+```
+
+## cloud vm benchmarking
+
+for reproducible results, run on a dedicated VM (e.g., GCP c2-standard-8).
+
+### 1. create a vm
+
+```bash
+# example: GCP compute-optimized instance
+gcloud compute instances create ember-bench \
+  --zone=us-central1-a \
+  --machine-type=c2-standard-8 \
+  --image-family=ubuntu-2204-lts \
+  --image-project=ubuntu-os-cloud
+```
+
+### 2. set up the vm
+
+```bash
+# ssh and run setup script
+gcloud compute ssh ember-bench --zone=us-central1-a \
+  --command='bash -s' < ./scripts/setup-vm.sh
+```
+
+or manually:
+
+```bash
+gcloud compute ssh ember-bench --zone=us-central1-a
+
+# on the vm:
+sudo apt-get update && sudo apt-get install -y build-essential git redis-server
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+source ~/.cargo/env
+git clone https://github.com/kacy/ember.git
+cd ember
+cargo build --release --features jemalloc
+```
+
+### 3. run benchmarks
+
+```bash
+gcloud compute ssh ember-bench --zone=us-central1-a
+
+cd ember
+./scripts/bench.sh
+```
+
+### 4. cleanup
+
+```bash
+gcloud compute instances delete ember-bench --zone=us-central1-a
+```
+
+## scripts
+
+| script | description |
+|--------|-------------|
+| `bench.sh` | full benchmark suite: ember (sharded + concurrent) vs redis |
+| `bench-quick.sh` | quick sanity check (ember only, ~10 seconds) |
+| `bench-memory.sh` | memory usage comparison with 1M keys |
+| `setup-vm.sh` | install dependencies on a fresh ubuntu vm |
+
+## configuration
+
+all scripts support environment variables:
+
+```bash
+# customize benchmark parameters
+REQUESTS=1000000 THREADS=16 ./scripts/bench.sh
+
+# customize memory test
+KEY_COUNT=5000000 VALUE_SIZE=128 ./scripts/bench-memory.sh
+```
+
+## expected results
+
+on a c2-standard-8 (8 vCPU Intel Xeon @ 3.1GHz):
+
+| mode | SET (P=16) | GET (P=16) |
+|------|------------|------------|
+| ember concurrent | ~1.8M/s | ~2.5M/s |
+| ember sharded | ~900K/s | ~1.0M/s |
+| redis | ~1.0M/s | ~1.2M/s |
+
+results vary based on CPU, memory, and kernel version.

scripts/bench-memory.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# memory usage comparison between ember and redis
+#
+# usage: ./scripts/bench-memory.sh
+
+set -e
+
+KEY_COUNT=${KEY_COUNT:-1000000}
+VALUE_SIZE=${VALUE_SIZE:-64}
+
+EMBER_PORT=6380
+REDIS_PORT=6379
+
+cleanup() {
+    pkill -f "ember-server.*${EMBER_PORT}" 2>/dev/null || true
+    redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true
+}
+
+trap cleanup EXIT
+
+EMBER_BIN="./target/release/ember-server"
+if [[ ! -x "$EMBER_BIN" ]]; then
+    echo "error: ember-server not found at $EMBER_BIN"
+    echo "run: cargo build --release"
+    exit 1
+fi
+
+cleanup
+
+echo "============================================="
+echo "MEMORY BENCHMARK"
+echo "============================================="
+echo "keys:       $KEY_COUNT"
+echo "value size: ${VALUE_SIZE}B"
+echo "============================================="
+echo ""
+
+echo "starting servers..."
+$EMBER_BIN --port $EMBER_PORT --concurrent > /dev/null 2>&1 &
+EMBER_PID=$!
+redis-server --port $REDIS_PORT --daemonize yes --save "" --appendonly no > /dev/null 2>&1
+sleep 2
+
+echo ""
+echo "initial memory:"
+EMBER_INIT=$(ps -o rss= -p $EMBER_PID 2>/dev/null | awk '{print $1}')
+REDIS_INIT=$(redis-cli -p $REDIS_PORT info memory 2>/dev/null | grep used_memory: | cut -d: -f2 | tr -d '\r')
+echo "  ember: $((EMBER_INIT / 1024)) MB"
+echo "  redis: $((REDIS_INIT / 1024 / 1024)) MB"
+
+echo ""
+echo "loading $KEY_COUNT keys..."
+redis-benchmark -p $EMBER_PORT -t set -n $KEY_COUNT -r $KEY_COUNT -d $VALUE_SIZE -q 2>/dev/null
+redis-benchmark -p $REDIS_PORT -t set -n $KEY_COUNT -r $KEY_COUNT -d $VALUE_SIZE -q 2>/dev/null
+
+EMBER_KEYS=$(redis-cli -p $EMBER_PORT DBSIZE 2>/dev/null | awk '{print $2}')
+REDIS_KEYS=$(redis-cli -p $REDIS_PORT DBSIZE 2>/dev/null | awk '{print $2}')
+
+echo ""
+echo "after loading:"
+EMBER_FINAL=$(ps -o rss= -p $EMBER_PID 2>/dev/null | awk '{print $1}')
+REDIS_FINAL=$(redis-cli -p $REDIS_PORT info memory 2>/dev/null | grep used_memory: | cut -d: -f2 | tr -d '\r')
+
+EMBER_USED=$((EMBER_FINAL - EMBER_INIT))
+REDIS_USED=$((REDIS_FINAL - REDIS_INIT))
+
+echo "  ember: $((EMBER_FINAL / 1024)) MB ($EMBER_KEYS keys)"
+echo "  redis: $((REDIS_FINAL / 1024 / 1024)) MB ($REDIS_KEYS keys)"
+
+echo ""
+echo "per-key overhead:"
+if [[ $EMBER_KEYS -gt 0 ]]; then
+    echo "  ember: $((EMBER_USED * 1024 / EMBER_KEYS)) bytes/key"
+fi
+if [[ $REDIS_KEYS -gt 0 ]]; then
+    echo "  redis: $((REDIS_USED / REDIS_KEYS)) bytes/key"
+fi
+
+echo ""
+echo "done."

scripts/bench-quick.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+#
+# quick benchmark for sanity checks (ember only)
+#
+# usage: ./scripts/bench-quick.sh
+
+set -e
+
+REQUESTS=100000
+CLIENTS=50
+PIPELINE=16
+VALUE_SIZE=64
+
+PORT=6379
+
+cleanup() {
+    pkill -f "ember-server.*${PORT}" 2>/dev/null || true
+}
+
+trap cleanup EXIT
+
+EMBER_BIN="./target/release/ember-server"
+if [[ ! -x "$EMBER_BIN" ]]; then
+    echo "error: ember-server not found at $EMBER_BIN"
+    echo "run: cargo build --release"
+    exit 1
+fi
+
+cleanup
+
+echo "starting ember on port $PORT..."
+$EMBER_BIN --port $PORT > /dev/null 2>&1 &
+sleep 1
+
+redis-cli -p $PORT PING > /dev/null || { echo "ember failed to start"; exit 1; }
+
+echo ""
+echo "=== QUICK BENCHMARK ==="
+echo ""
+
+echo -n "SET (P=$PIPELINE): "
+redis-benchmark -p $PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE -q 2>/dev/null
+
+echo -n "GET (P=$PIPELINE): "
+redis-benchmark -p $PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE -q 2>/dev/null
+
+echo -n "SET (P=1):  "
+redis-benchmark -p $PORT -t set -n $REQUESTS -c $CLIENTS -P 1 -d $VALUE_SIZE -q 2>/dev/null
+
+echo -n "GET (P=1):  "
+redis-benchmark -p $PORT -t get -n $REQUESTS -c $CLIENTS -P 1 -d $VALUE_SIZE -q 2>/dev/null
+
+echo ""
+echo "done."

scripts/bench.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# full benchmark suite comparing ember (sharded + concurrent) vs redis
+#
+# usage: ./scripts/bench.sh
+#
+# requirements:
+#   - ember built with: cargo build --release --features jemalloc
+#   - redis-server and redis-benchmark installed
+#   - redis-benchmark with --threads support (redis 6+)
+
+set -e
+
+REQUESTS=${REQUESTS:-500000}
+CLIENTS=${CLIENTS:-50}
+PIPELINE=${PIPELINE:-16}
+VALUE_SIZE=${VALUE_SIZE:-64}
+THREADS=${THREADS:-8}
+
+EMBER_SHARDED_PORT=6380
+EMBER_CONCURRENT_PORT=6381
+REDIS_PORT=6379
+
+cleanup() {
+    pkill -f "ember-server.*${EMBER_SHARDED_PORT}" 2>/dev/null || true
+    pkill -f "ember-server.*${EMBER_CONCURRENT_PORT}" 2>/dev/null || true
+    redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true
+}
+
+trap cleanup EXIT
+
+echo "============================================="
+echo "EMBER BENCHMARK SUITE"
+echo "============================================="
+echo "requests:   $REQUESTS"
+echo "clients:    $CLIENTS"
+echo "pipeline:   $PIPELINE"
+echo "value size: ${VALUE_SIZE}B"
+echo "threads:    $THREADS"
+echo "============================================="
+echo ""
+
+# find ember binary
+EMBER_BIN="./target/release/ember-server"
+if [[ ! -x "$EMBER_BIN" ]]; then
+    echo "error: ember-server not found at $EMBER_BIN"
+    echo "run: cargo build --release --features jemalloc"
+    exit 1
+fi
+
+# check redis
+if ! command -v redis-server &> /dev/null; then
+    echo "error: redis-server not found"
+    exit 1
+fi
+
+cleanup
+
+echo "starting servers..."
+$EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 &
+$EMBER_BIN --port $EMBER_CONCURRENT_PORT --concurrent > /dev/null 2>&1 &
+redis-server --port $REDIS_PORT --daemonize yes --save "" --appendonly no > /dev/null 2>&1
+sleep 2
+
+# verify servers are up
+redis-cli -p $EMBER_SHARDED_PORT PING > /dev/null || { echo "ember sharded failed to start"; exit 1; }
+redis-cli -p $EMBER_CONCURRENT_PORT PING > /dev/null || { echo "ember concurrent failed to start"; exit 1; }
+redis-cli -p $REDIS_PORT PING > /dev/null || { echo "redis failed to start"; exit 1; }
+
+echo ""
+echo "=== THROUGHPUT (P=$PIPELINE, $THREADS threads) ==="
+echo ""
+
+echo "ember sharded:"
+echo -n "  SET: "
+redis-benchmark -p $EMBER_SHARDED_PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}'
+echo -n "  GET: "
+redis-benchmark -p $EMBER_SHARDED_PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}'
+
+echo ""
+echo "ember concurrent:"
+echo -n "  SET: "
+redis-benchmark -p $EMBER_CONCURRENT_PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}'
+echo -n "  GET: "
+redis-benchmark -p $EMBER_CONCURRENT_PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}'
+
+echo ""
+echo "redis:"
+echo -n "  SET: "
+redis-benchmark -p $REDIS_PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}'
+echo -n "  GET: "
+redis-benchmark -p $REDIS_PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}'
+
+echo ""
+echo "=== LATENCY (P=1, single thread) ==="
+echo ""
+
+LATENCY_REQUESTS=100000
+
+echo "ember sharded:"
+redis-benchmark -p $EMBER_SHARDED_PORT -t set -n $LATENCY_REQUESTS -c $CLIENTS -d $VALUE_SIZE 2>/dev/null | grep -E "requests per second|<="
+
+echo ""
+echo "ember concurrent:"
+redis-benchmark -p $EMBER_CONCURRENT_PORT -t set -n $LATENCY_REQUESTS -c $CLIENTS -d $VALUE_SIZE 2>/dev/null | grep -E "requests per second|<="
+
+echo ""
+echo "redis:"
+redis-benchmark -p $REDIS_PORT -t set -n $LATENCY_REQUESTS -c $CLIENTS -d $VALUE_SIZE 2>/dev/null | grep -E "requests per second|<="
+
+echo ""
+echo "=== DONE ==="