diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..74ff8f0c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,101 @@ +# benchmark scripts + +scripts for benchmarking ember against redis. + +## quick start (local) + +```bash +# build ember +cargo build --release --features jemalloc + +# run quick sanity check +./scripts/bench-quick.sh + +# run full comparison (requires redis) +./scripts/bench.sh +``` + +## cloud vm benchmarking + +for reproducible results, run on a dedicated VM (e.g., GCP c2-standard-8). + +### 1. create a vm + +```bash +# example: GCP compute-optimized instance +gcloud compute instances create ember-bench \ + --zone=us-central1-a \ + --machine-type=c2-standard-8 \ + --image-family=ubuntu-2204-lts \ + --image-project=ubuntu-os-cloud +``` + +### 2. set up the vm + +```bash +# ssh and run setup script +gcloud compute ssh ember-bench --zone=us-central1-a \ + --command='bash -s' < ./scripts/setup-vm.sh +``` + +or manually: + +```bash +gcloud compute ssh ember-bench --zone=us-central1-a + +# on the vm: +sudo apt-get update && sudo apt-get install -y build-essential git redis-server +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +source ~/.cargo/env +git clone https://github.com/kacy/ember.git +cd ember +cargo build --release --features jemalloc +``` + +### 3. run benchmarks + +```bash +gcloud compute ssh ember-bench --zone=us-central1-a + +cd ember +./scripts/bench.sh +``` + +### 4. cleanup + +```bash +gcloud compute instances delete ember-bench --zone=us-central1-a +``` + +## scripts + +| script | description | +|--------|-------------| +| `bench.sh` | full benchmark suite: ember (sharded + concurrent) vs redis | +| `bench-quick.sh` | quick sanity check (ember only, ~10 seconds) | +| `bench-memory.sh` | memory usage comparison with 1M keys | +| `setup-vm.sh` | install dependencies on a fresh ubuntu vm | + +## configuration + +all scripts support environment variables: + +```bash +# customize benchmark parameters +REQUESTS=1000000 THREADS=16 ./scripts/bench.sh + +# customize memory test +KEY_COUNT=5000000 VALUE_SIZE=128 ./scripts/bench-memory.sh +``` + +## expected results + +on a c2-standard-8 (8 vCPU Intel Xeon @ 3.1GHz): + +| mode | SET (P=16) | GET (P=16) | +|------|------------|------------| +| ember concurrent | ~1.8M/s | ~2.5M/s | +| ember sharded | ~900K/s | ~1.0M/s | +| redis | ~1.0M/s | ~1.2M/s | + +results vary based on CPU, memory, and kernel version. diff --git a/scripts/bench-memory.sh b/scripts/bench-memory.sh new file mode 100755 index 00000000..848deab8 --- /dev/null +++ b/scripts/bench-memory.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# memory usage comparison between ember and redis +# +# usage: ./scripts/bench-memory.sh + +set -e + +KEY_COUNT=${KEY_COUNT:-1000000} +VALUE_SIZE=${VALUE_SIZE:-64} + +EMBER_PORT=6380 +REDIS_PORT=6379 + +cleanup() { + pkill -f "ember-server.*${EMBER_PORT}" 2>/dev/null || true + redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true +} + +trap cleanup EXIT + +EMBER_BIN="./target/release/ember-server" +if [[ ! -x "$EMBER_BIN" ]]; then + echo "error: ember-server not found at $EMBER_BIN" + echo "run: cargo build --release" + exit 1 +fi + +cleanup + +echo "=============================================" +echo "MEMORY BENCHMARK" +echo "=============================================" +echo "keys: $KEY_COUNT" +echo "value size: ${VALUE_SIZE}B" +echo "=============================================" +echo "" + +echo "starting servers..." +$EMBER_BIN --port $EMBER_PORT --concurrent > /dev/null 2>&1 & +EMBER_PID=$! +redis-server --port $REDIS_PORT --daemonize yes --save "" --appendonly no > /dev/null 2>&1 +sleep 2 + +echo "" +echo "initial memory:" +EMBER_INIT=$(ps -o rss= -p $EMBER_PID 2>/dev/null | awk '{print $1}') +REDIS_INIT=$(redis-cli -p $REDIS_PORT info memory 2>/dev/null | grep used_memory: | cut -d: -f2 | tr -d '\r') +echo " ember: $((EMBER_INIT / 1024)) MB" +echo " redis: $((REDIS_INIT / 1024 / 1024)) MB" + +echo "" +echo "loading $KEY_COUNT keys..." +redis-benchmark -p $EMBER_PORT -t set -n $KEY_COUNT -r $KEY_COUNT -d $VALUE_SIZE -q 2>/dev/null +redis-benchmark -p $REDIS_PORT -t set -n $KEY_COUNT -r $KEY_COUNT -d $VALUE_SIZE -q 2>/dev/null + +EMBER_KEYS=$(redis-cli -p $EMBER_PORT DBSIZE 2>/dev/null | awk '{print $2}') +REDIS_KEYS=$(redis-cli -p $REDIS_PORT DBSIZE 2>/dev/null | awk '{print $2}') + +echo "" +echo "after loading:" +EMBER_FINAL=$(ps -o rss= -p $EMBER_PID 2>/dev/null | awk '{print $1}') +REDIS_FINAL=$(redis-cli -p $REDIS_PORT info memory 2>/dev/null | grep used_memory: | cut -d: -f2 | tr -d '\r') + +EMBER_USED=$((EMBER_FINAL - EMBER_INIT)) +REDIS_USED=$((REDIS_FINAL - REDIS_INIT)) + +echo " ember: $((EMBER_FINAL / 1024)) MB ($EMBER_KEYS keys)" +echo " redis: $((REDIS_FINAL / 1024 / 1024)) MB ($REDIS_KEYS keys)" + +echo "" +echo "per-key overhead:" +if [[ $EMBER_KEYS -gt 0 ]]; then + echo " ember: $((EMBER_USED * 1024 / EMBER_KEYS)) bytes/key" +fi +if [[ $REDIS_KEYS -gt 0 ]]; then + echo " redis: $((REDIS_USED / REDIS_KEYS)) bytes/key" +fi + +echo "" +echo "done." diff --git a/scripts/bench-quick.sh b/scripts/bench-quick.sh new file mode 100755 index 00000000..875b52a7 --- /dev/null +++ b/scripts/bench-quick.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# quick benchmark for sanity checks (ember only) +# +# usage: ./scripts/bench-quick.sh + +set -e + +REQUESTS=100000 +CLIENTS=50 +PIPELINE=16 +VALUE_SIZE=64 + +PORT=6379 + +cleanup() { + pkill -f "ember-server.*${PORT}" 2>/dev/null || true +} + +trap cleanup EXIT + +EMBER_BIN="./target/release/ember-server" +if [[ ! -x "$EMBER_BIN" ]]; then + echo "error: ember-server not found at $EMBER_BIN" + echo "run: cargo build --release" + exit 1 +fi + +cleanup + +echo "starting ember on port $PORT..." +$EMBER_BIN --port $PORT > /dev/null 2>&1 & +sleep 1 + +redis-cli -p $PORT PING > /dev/null || { echo "ember failed to start"; exit 1; } + +echo "" +echo "=== QUICK BENCHMARK ===" +echo "" + +echo -n "SET (P=$PIPELINE): " +redis-benchmark -p $PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE -q 2>/dev/null + +echo -n "GET (P=$PIPELINE): " +redis-benchmark -p $PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE -q 2>/dev/null + +echo -n "SET (P=1): " +redis-benchmark -p $PORT -t set -n $REQUESTS -c $CLIENTS -P 1 -d $VALUE_SIZE -q 2>/dev/null + +echo -n "GET (P=1): " +redis-benchmark -p $PORT -t get -n $REQUESTS -c $CLIENTS -P 1 -d $VALUE_SIZE -q 2>/dev/null + +echo "" +echo "done." diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100755 index 00000000..4abd5234 --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# +# full benchmark suite comparing ember (sharded + concurrent) vs redis +# +# usage: ./scripts/bench.sh +# +# requirements: +# - ember built with: cargo build --release --features jemalloc +# - redis-server and redis-benchmark installed +# - redis-benchmark with --threads support (redis 6+) + +set -e + +REQUESTS=${REQUESTS:-500000} +CLIENTS=${CLIENTS:-50} +PIPELINE=${PIPELINE:-16} +VALUE_SIZE=${VALUE_SIZE:-64} +THREADS=${THREADS:-8} + +EMBER_SHARDED_PORT=6380 +EMBER_CONCURRENT_PORT=6381 +REDIS_PORT=6379 + +cleanup() { + pkill -f "ember-server.*${EMBER_SHARDED_PORT}" 2>/dev/null || true + pkill -f "ember-server.*${EMBER_CONCURRENT_PORT}" 2>/dev/null || true + redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true +} + +trap cleanup EXIT + +echo "=============================================" +echo "EMBER BENCHMARK SUITE" +echo "=============================================" +echo "requests: $REQUESTS" +echo "clients: $CLIENTS" +echo "pipeline: $PIPELINE" +echo "value size: ${VALUE_SIZE}B" +echo "threads: $THREADS" +echo "=============================================" +echo "" + +# find ember binary +EMBER_BIN="./target/release/ember-server" +if [[ ! -x "$EMBER_BIN" ]]; then + echo "error: ember-server not found at $EMBER_BIN" + echo "run: cargo build --release --features jemalloc" + exit 1 +fi + +# check redis +if ! command -v redis-server &> /dev/null; then + echo "error: redis-server not found" + exit 1 +fi + +cleanup + +echo "starting servers..." +$EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 & +$EMBER_BIN --port $EMBER_CONCURRENT_PORT --concurrent > /dev/null 2>&1 & +redis-server --port $REDIS_PORT --daemonize yes --save "" --appendonly no > /dev/null 2>&1 +sleep 2 + +# verify servers are up +redis-cli -p $EMBER_SHARDED_PORT PING > /dev/null || { echo "ember sharded failed to start"; exit 1; } +redis-cli -p $EMBER_CONCURRENT_PORT PING > /dev/null || { echo "ember concurrent failed to start"; exit 1; } +redis-cli -p $REDIS_PORT PING > /dev/null || { echo "redis failed to start"; exit 1; } + +echo "" +echo "=== THROUGHPUT (P=$PIPELINE, $THREADS threads) ===" +echo "" + +echo "ember sharded:" +echo -n " SET: " +redis-benchmark -p $EMBER_SHARDED_PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}' +echo -n " GET: " +redis-benchmark -p $EMBER_SHARDED_PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}' + +echo "" +echo "ember concurrent:" +echo -n " SET: " +redis-benchmark -p $EMBER_CONCURRENT_PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}' +echo -n " GET: " +redis-benchmark -p $EMBER_CONCURRENT_PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}' + +echo "" +echo "redis:" +echo -n " SET: " +redis-benchmark -p $REDIS_PORT -t set -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}' +echo -n " GET: " +redis-benchmark -p $REDIS_PORT -t get -n $REQUESTS -c $CLIENTS -P $PIPELINE -d $VALUE_SIZE --threads $THREADS 2>/dev/null | grep "requests per second" | awk '{print $1}' + +echo "" +echo "=== LATENCY (P=1, single thread) ===" +echo "" + +LATENCY_REQUESTS=100000 + +echo "ember sharded:" +redis-benchmark -p $EMBER_SHARDED_PORT -t set -n $LATENCY_REQUESTS -c $CLIENTS -d $VALUE_SIZE 2>/dev/null | grep -E "requests per second|<=" + +echo "" +echo "ember concurrent:" +redis-benchmark -p $EMBER_CONCURRENT_PORT -t set -n $LATENCY_REQUESTS -c $CLIENTS -d $VALUE_SIZE 2>/dev/null | grep -E "requests per second|<=" + +echo "" +echo "redis:" +redis-benchmark -p $REDIS_PORT -t set -n $LATENCY_REQUESTS -c $CLIENTS -d $VALUE_SIZE 2>/dev/null | grep -E "requests per second|<=" + +echo "" +echo "=== DONE ===" diff --git a/scripts/setup-vm.sh b/scripts/setup-vm.sh new file mode 100755 index 00000000..45eff963 --- /dev/null +++ b/scripts/setup-vm.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# set up a fresh linux VM for benchmarking +# +# usage: ssh user@vm 'bash -s' < ./scripts/setup-vm.sh +# +# tested on ubuntu 22.04 + +set -e + +echo "=== installing dependencies ===" + +sudo apt-get update +sudo apt-get install -y build-essential git redis-server + +echo "" +echo "=== installing rust ===" + +if command -v rustup &> /dev/null; then + echo "rust already installed" + rustup update stable +else + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source ~/.cargo/env +fi + +echo "" +echo "=== cloning ember ===" + +if [[ -d ~/ember ]]; then + cd ~/ember + git pull origin main +else + git clone https://github.com/kacy/ember.git ~/ember + cd ~/ember +fi + +echo "" +echo "=== building ember ===" + +cargo build --release --features jemalloc + +echo "" +echo "=== verifying ===" + +./target/release/ember-server --help | head -5 +redis-server --version +redis-benchmark --version + +echo "" +echo "=== setup complete ===" +echo "" +echo "run benchmarks with:" +echo " ./scripts/bench.sh # full suite" +echo " ./scripts/bench-quick.sh # quick sanity check" +echo " ./scripts/bench-memory.sh # memory comparison"