From 625abb445bfbc857b8569365a5d00df9ec3a536d Mon Sep 17 00:00:00 2001 From: Kacy Fortner Date: Sun, 15 Feb 2026 22:29:43 -0500 Subject: [PATCH] fix: rewrite memory benchmark bulk loading and mode handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit three issues discovered during GCP testing: 1. redis-cli --pipe doesn't work with ember — silently drops commands. switched to a python bulk loader using redis-py pipelining. 2. hash and sorted set commands only work in sharded mode, not concurrent. skip concurrent mode for those types instead of reporting 0. 3. sorted set adds all members to one key (DBSIZE=1), making per-key division meaningless. divide by member count instead. --- bench/bench-memory.sh | 271 ++++++++++++++++++++++++------------------ 1 file changed, 156 insertions(+), 115 deletions(-) diff --git a/bench/bench-memory.sh b/bench/bench-memory.sh index c5961278..a4c85495 100755 --- a/bench/bench-memory.sh +++ b/bench/bench-memory.sh @@ -2,8 +2,8 @@ # # memory usage comparison between ember and redis across data types. # -# tests both ember modes (concurrent + sharded) and measures per-key -# memory overhead for strings, hashes, sorted sets, and optionally vectors. +# tests both ember modes (concurrent for strings, sharded for all types) +# and measures per-key/per-member memory overhead. # # usage: # ./bench/bench-memory.sh # string, hash, sorted set (ember + redis) @@ -81,6 +81,39 @@ wait_for_server() { done } +# python helper for bulk loading hash and sorted set data. +# redis-cli --pipe doesn't work reliably with ember, so we use +# redis-py pipelining instead. +BULK_LOADER=$(mktemp /tmp/bench_bulk_load_XXXXXX.py) +cat > "$BULK_LOADER" << 'PYEOF' +import redis +import sys + +port = int(sys.argv[1]) +data_type = sys.argv[2] +count = int(sys.argv[3]) + +r = redis.Redis(host="127.0.0.1", port=port) +pipe = r.pipeline(transaction=False) +batch = 1000 + +for i in range(count): + if data_type == "hash": + pipe.hset(f"hash:{i}", mapping={ + "f1": "value001", "f2": "value002", "f3": "value003", + "f4": "value004", "f5": "value005", + }) + elif data_type == "zset": + pipe.zadd("myzset", {f"member:{i}": float(i % 100000)}) + + if (i + 1) % batch == 0: + pipe.execute() + pipe = r.pipeline(transaction=False) + +pipe.execute() +r.close() +PYEOF + cleanup echo "=============================================" @@ -102,9 +135,9 @@ declare -a EMBER_CONCURRENT_BYTES=() declare -a EMBER_SHARDED_BYTES=() declare -a REDIS_BYTES=() -# --- measure ember memory for one data type --- +# --- measure memory for one data type on one server --- -measure_ember_type() { +measure_type() { local port=$1 local type=$2 local count=$3 @@ -116,19 +149,8 @@ measure_ember_type() { string) redis-benchmark -p "$port" -t set -n "$count" -r "$count" -d "$VALUE_SIZE" -q > /dev/null 2>&1 ;; - hash) - { - for i in $(seq 1 "$count"); do - echo "HSET hash:$i f1 value001 f2 value002 f3 value003 f4 value004 f5 value005" - done - } | redis-cli -p "$port" --pipe > /dev/null 2>&1 - ;; - zset) - { - for i in $(seq 1 "$count"); do - echo "ZADD myzset $((RANDOM % 100000)) member:$i" - done - } | redis-cli -p "$port" --pipe > /dev/null 2>&1 + hash|zset) + python3 "$BULK_LOADER" "$port" "$type" "$count" ;; esac @@ -136,131 +158,147 @@ measure_ember_type() { local mem_after mem_after=$(get_redis_memory "$port") - local keys - keys=$(redis-cli -p "$port" DBSIZE 2>/dev/null | awk '{print $NF}' | tr -d '\r') + + # for sorted sets, all members go into one key so DBSIZE=1. + # divide by member count instead. + local divisor + if [[ "$type" == "zset" ]]; then + divisor=$count + else + divisor=$(redis-cli -p "$port" DBSIZE 2>/dev/null | awk '{print $NF}' | tr -d '\r') + fi local used=$((mem_after - mem_before)) local bytes_per_key=0 - if [[ "${keys:-0}" -gt 0 ]] 2>/dev/null; then - bytes_per_key=$((used / keys)) + if [[ "${divisor:-0}" -gt 0 ]] 2>/dev/null; then + bytes_per_key=$((used / divisor)) fi echo "$bytes_per_key" } -# --- measure redis memory for one data type --- - -measure_redis_type() { - local port=$1 - local type=$2 - local count=$3 - - local mem_before - mem_before=$(get_redis_memory "$port") - - case "$type" in - string) - redis-benchmark -p "$port" -t set -n "$count" -r "$count" -d "$VALUE_SIZE" -q > /dev/null 2>&1 - ;; - hash) - { - for i in $(seq 1 "$count"); do - echo "HSET hash:$i f1 value001 f2 value002 f3 value003 f4 value004 f5 value005" - done - } | redis-cli -p "$port" --pipe > /dev/null 2>&1 - ;; - zset) - { - for i in $(seq 1 "$count"); do - echo "ZADD myzset $((RANDOM % 100000)) member:$i" - done - } | redis-cli -p "$port" --pipe > /dev/null 2>&1 - ;; - esac +# --- string benchmark (concurrent + sharded + redis) --- +echo "--- string (${VALUE_SIZE}B) ($STRING_KEYS keys) ---" +echo "" +TYPE_LABELS+=("string (${VALUE_SIZE}B)") + +# ember concurrent +echo " starting ember concurrent..." +$EMBER_BIN --port $EMBER_CONCURRENT_PORT --concurrent > /dev/null 2>&1 & +EC_PID=$! +wait_for_server $EMBER_CONCURRENT_PORT + +ec_bytes=$(measure_type "$EMBER_CONCURRENT_PORT" "string" "$STRING_KEYS") +EMBER_CONCURRENT_BYTES+=("$ec_bytes") +echo " ember concurrent: ${ec_bytes} B/key" + +kill "$EC_PID" 2>/dev/null && wait "$EC_PID" 2>/dev/null || true +sleep 0.5 + +# ember sharded +echo " starting ember sharded..." +$EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 & +ES_PID=$! +wait_for_server $EMBER_SHARDED_PORT + +es_bytes=$(measure_type "$EMBER_SHARDED_PORT" "string" "$STRING_KEYS") +EMBER_SHARDED_BYTES+=("$es_bytes") +echo " ember sharded: ${es_bytes} B/key" + +kill "$ES_PID" 2>/dev/null && wait "$ES_PID" 2>/dev/null || true +sleep 0.5 + +# redis +if [[ "$HAS_REDIS" == "true" ]]; then + echo " starting redis..." + redis-server --port $REDIS_PORT --save "" --appendonly no --loglevel warning > /dev/null 2>&1 & sleep 1 + wait_for_server $REDIS_PORT - local mem_after - mem_after=$(get_redis_memory "$port") - local keys - keys=$(redis-cli -p "$port" DBSIZE 2>/dev/null | awk '{print $NF}' | tr -d '\r') + r_bytes=$(measure_type "$REDIS_PORT" "string" "$STRING_KEYS") + REDIS_BYTES+=("$r_bytes") + echo " redis: ${r_bytes} B/key" - local used=$((mem_after - mem_before)) - local bytes_per_key=0 - if [[ "${keys:-0}" -gt 0 ]] 2>/dev/null; then - bytes_per_key=$((used / keys)) - fi + redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true + sleep 0.5 +else + REDIS_BYTES+=("—") +fi +echo "" - echo "$bytes_per_key" -} +# --- hash benchmark (sharded only + redis) --- +# hash/zset commands are only supported in sharded mode -# --- run benchmarks for one type across all servers --- +echo "--- hash (5 fields) ($COMPLEX_KEYS keys) ---" +echo "" +TYPE_LABELS+=("hash (5 fields)") +EMBER_CONCURRENT_BYTES+=("—") -run_type_benchmark() { - local label=$1 - local type=$2 - local count=$3 - local skip_redis=${4:-false} +echo " starting ember sharded..." +$EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 & +ES_PID=$! +wait_for_server $EMBER_SHARDED_PORT - echo "--- $label ($count keys) ---" - echo "" +es_bytes=$(measure_type "$EMBER_SHARDED_PORT" "hash" "$COMPLEX_KEYS") +EMBER_SHARDED_BYTES+=("$es_bytes") +echo " ember sharded: ${es_bytes} B/key" - TYPE_LABELS+=("$label") +kill "$ES_PID" 2>/dev/null && wait "$ES_PID" 2>/dev/null || true +sleep 0.5 - # ember concurrent - echo " starting ember concurrent..." - $EMBER_BIN --port $EMBER_CONCURRENT_PORT --concurrent > /dev/null 2>&1 & - local ec_pid=$! - wait_for_server $EMBER_CONCURRENT_PORT +if [[ "$HAS_REDIS" == "true" ]]; then + echo " starting redis..." + redis-server --port $REDIS_PORT --save "" --appendonly no --loglevel warning > /dev/null 2>&1 & + sleep 1 + wait_for_server $REDIS_PORT - local ec_bytes - ec_bytes=$(measure_ember_type "$EMBER_CONCURRENT_PORT" "$type" "$count") - EMBER_CONCURRENT_BYTES+=("$ec_bytes") - echo " ember concurrent: ${ec_bytes} bytes/key" + r_bytes=$(measure_type "$REDIS_PORT" "hash" "$COMPLEX_KEYS") + REDIS_BYTES+=("$r_bytes") + echo " redis: ${r_bytes} B/key" - kill "$ec_pid" 2>/dev/null && wait "$ec_pid" 2>/dev/null || true + redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true sleep 0.5 +else + REDIS_BYTES+=("—") +fi +echo "" - # ember sharded - echo " starting ember sharded..." - $EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 & - local es_pid=$! - wait_for_server $EMBER_SHARDED_PORT - - local es_bytes - es_bytes=$(measure_ember_type "$EMBER_SHARDED_PORT" "$type" "$count") - EMBER_SHARDED_BYTES+=("$es_bytes") - echo " ember sharded: ${es_bytes} bytes/key" +# --- sorted set benchmark (sharded only + redis) --- - kill "$es_pid" 2>/dev/null && wait "$es_pid" 2>/dev/null || true - sleep 0.5 +echo "--- sorted set ($COMPLEX_KEYS members) ---" +echo "" +TYPE_LABELS+=("sorted set") +EMBER_CONCURRENT_BYTES+=("—") - # redis - if [[ "$HAS_REDIS" == "true" ]] && [[ "$skip_redis" == "false" ]]; then - echo " starting redis..." - redis-server --port $REDIS_PORT --save "" --appendonly no --loglevel warning > /dev/null 2>&1 & - sleep 1 - wait_for_server $REDIS_PORT +echo " starting ember sharded..." +$EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 & +ES_PID=$! +wait_for_server $EMBER_SHARDED_PORT - local r_bytes - r_bytes=$(measure_redis_type "$REDIS_PORT" "$type" "$count") - REDIS_BYTES+=("$r_bytes") - echo " redis: ${r_bytes} bytes/key" +es_bytes=$(measure_type "$EMBER_SHARDED_PORT" "zset" "$COMPLEX_KEYS") +EMBER_SHARDED_BYTES+=("$es_bytes") +echo " ember sharded: ${es_bytes} B/member" - redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true - sleep 0.5 - else - REDIS_BYTES+=("—") - fi +kill "$ES_PID" 2>/dev/null && wait "$ES_PID" 2>/dev/null || true +sleep 0.5 - echo "" -} +if [[ "$HAS_REDIS" == "true" ]]; then + echo " starting redis..." + redis-server --port $REDIS_PORT --save "" --appendonly no --loglevel warning > /dev/null 2>&1 & + sleep 1 + wait_for_server $REDIS_PORT -# --- run each data type --- + r_bytes=$(measure_type "$REDIS_PORT" "zset" "$COMPLEX_KEYS") + REDIS_BYTES+=("$r_bytes") + echo " redis: ${r_bytes} B/member" -run_type_benchmark "string (${VALUE_SIZE}B)" "string" "$STRING_KEYS" -run_type_benchmark "hash (5 fields)" "hash" "$COMPLEX_KEYS" -run_type_benchmark "sorted set" "zset" "$COMPLEX_KEYS" + redis-cli -p $REDIS_PORT shutdown nosave 2>/dev/null || true + sleep 0.5 +else + REDIS_BYTES+=("—") +fi +echo "" # --- vector (ember only, requires python3) --- @@ -302,7 +340,7 @@ for i in range(count): print(f" inserted {i + 1}/{count} vectors", file=sys.stderr) PYEOF - # vector requires sharded mode (full type support) + # vector requires sharded mode echo " starting ember sharded..." $EMBER_BIN --port $EMBER_SHARDED_PORT > /dev/null 2>&1 & VEC_PID=$! @@ -319,7 +357,7 @@ PYEOF EMBER_CONCURRENT_BYTES+=("—") EMBER_SHARDED_BYTES+=("$VEC_BYTES") REDIS_BYTES+=("—") - echo " ember sharded: ${VEC_BYTES} bytes/vector" + echo " ember sharded: ${VEC_BYTES} B/vector" kill "$VEC_PID" 2>/dev/null && wait "$VEC_PID" 2>/dev/null || true rm -f "$VECTOR_HELPER" @@ -328,6 +366,9 @@ PYEOF echo "" fi +# --- cleanup temp files --- +rm -f "$BULK_LOADER" + # --- summary table --- echo "========================================================================"