diff --git a/Cargo.lock b/Cargo.lock index 64bd236b..885a472e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,6 +55,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -802,6 +803,7 @@ dependencies = [ name = "emberkv-core" version = "0.4.3" dependencies = [ + "ahash 0.8.12", "bytes", "criterion", "dashmap", diff --git a/README.md b/README.md index 4b53cb48..fbbd7efe 100644 --- a/README.md +++ b/README.md @@ -262,15 +262,15 @@ tested on GCP c2-standard-8 (8 vCPU Intel Xeon @ 3.10GHz). see [bench/README.md] | mode | vs redis | vs dragonfly | best for | |------|----------|--------------|----------| -| concurrent | **1.9-2.2x faster** | **2.0-2.7x faster**\* | simple GET/SET workloads | -| sharded | ~0.9x (channel overhead) | ~1.0-1.1x | all data types | +| concurrent | **2.3-3.0x faster** | **6.5-6.6x faster**\* | simple GET/SET workloads | +| sharded | ~1.1x (P=1), ~0.5x (P=16) | ~1.0-1.4x | all data types | \*take these comparisons with a grain of salt. ember is a small indie project; Redis and Dragonfly are battle-tested systems built by large teams over many years. see [bench/README.md](bench/README.md) for important caveats. **highlights**: -- concurrent mode: 1.90M SET/sec, 2.58M GET/sec (simple GET/SET only) -- p99 latency: 0.4ms (same as redis) -- memory: ~161 bytes/key (redis: ~105 bytes/key) +- concurrent mode: 4.56M SET/sec, 6.62M GET/sec (memtier, P=16) +- p99 latency: 0.6ms SET, 0.6ms GET (P=1, concurrent mode) +- memory: ~161 bytes/key (redis: ~95 bytes/key) ```bash ./bench/bench-quick.sh # quick sanity check @@ -284,7 +284,7 @@ ember offers two execution modes: **sharded mode** (default): thread-per-core with channel-based routing. supports all data types (lists, hashes, sets, sorted sets). has channel overhead but enables atomic multi-key operations. -**concurrent mode** (`--concurrent`): lock-free DashMap access. 2x faster than sharded mode but only supports string operations. +**concurrent mode** (`--concurrent`): lock-free DashMap access. 2-3x faster than Redis but only supports string operations. contributions welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/bench/README.md b/bench/README.md index d5467a22..99973446 100644 --- a/bench/README.md +++ b/bench/README.md @@ -4,49 +4,55 @@ performance benchmarks for ember comparing against Redis and Dragonfly. ## results summary -tested on GCP c2-standard-8 (8 vCPU Intel Xeon @ 3.10GHz), Ubuntu 22.04. +tested on GCP c2-standard-8 (8 vCPU Intel Xeon @ 3.10GHz), Ubuntu 22.04, 2.4M total requests per test. ### throughput (requests/sec) -#### redis-benchmark +#### redis-benchmark (1M requests, 50 clients, 8 threads) | test | ember concurrent | ember sharded | redis | dragonfly | |------|------------------|---------------|-------|-----------| -| SET (64B, P=16) | **1,897,056** | 911,363 | 1,011,232 | 897,714 | -| GET (64B, P=16) | **2,580,102** | 992,633 | 1,192,761 | 928,148 | -| SET (64B, P=1) | **199,600** | **199,600** | 99,800 | 199,600 | -| GET (64B, P=1) | **200,000** | **200,000** | 99,900 | 200,000 | +| SET (3B, P=16) | **1,333,333** | 799,360 | 999,000 | 798,722 | +| GET (3B, P=16) | **1,996,008** | 799,360 | 999,000 | 798,722 | +| SET (64B, P=16) | **1,331,558** | 798,722 | 799,360 | 797,448 | +| GET (64B, P=16) | **1,996,008** | 798,722 | 999,000 | 798,084 | +| SET (1KB, P=16) | **999,000** | 570,125 | 798,722 | 664,893 | +| GET (1KB, P=16) | **1,333,333** | 798,722 | 798,722 | 332,446 | +| SET (64B, P=1) | **190,331** | 173,822 | 114,246 | 210,393 | +| GET (64B, P=1) | **190,403** | 173,761 | 117,605 | 222,172 | -#### memtier_benchmark +#### memtier_benchmark (4 threads, 12 clients/thread, 50k req/client) | test | ember concurrent | ember sharded | redis | dragonfly | |------|------------------|---------------|-------|-----------| -| SET (64B, P=16) | **1,781,625** | 1,139,899 | 1,121,736 | 1,002,678 | -| GET (64B, P=16) | **2,111,459** | 1,326,564 | 1,335,645 | 1,073,396 | -| mixed 1:10 (64B, P=16) | **2,153,732** | 1,298,726 | 1,305,540 | 1,056,071 | -| mixed 1:1 (64B, P=16) | **1,964,419** | 1,228,596 | 1,238,508 | 1,023,236 | -| SET (64B, P=1) | **255,763** | 159,418 | 161,217 | 304,207 | -| GET (64B, P=1) | **264,149** | 498,078 | 262,139 | 186,270 | +| SET (64B, P=16) | **4,562,538** | 987,530 | 2,022,543 | 694,116 | +| GET (64B, P=16) | **6,624,838** | 762,401 | 2,225,336 | 1,021,986 | +| mixed 1:10 (64B, P=16) | **1,727,847** | 1,083,253 | 1,165,414 | 702,866 | +| mixed 1:1 (64B, P=16) | **1,680,994** | 1,039,764 | 1,035,432 | 959,765 | +| SET (1KB, P=16) | **933,385** | 764,879 | 626,130 | 896,308 | +| GET (1KB, P=16) | **903,312** | 724,275 | 593,297 | 337,179 | +| SET (64B, P=1) | 170,655 | **171,517** | 154,439 | 171,582 | +| GET (64B, P=1) | **188,114** | 185,528 | 160,935 | 167,916 | ### vs redis | mode | SET | GET | notes | |------|-----|-----|-------| -| ember concurrent | **1.9x** | **2.2x** | best for simple GET/SET workloads | -| ember sharded | 0.9x | 0.8x | channel overhead, but supports all data types | +| ember concurrent | **2.3x** | **3.0x** | best for simple GET/SET workloads | +| ember sharded | 0.5x | 0.3x | channel overhead, but supports all data types | ### vs dragonfly | mode | SET | GET | notes | |------|-----|-----|-------| -| ember concurrent | **2.0x** | **2.7x** | redis-benchmark, pipelined | -| ember sharded | 1.0x | 1.1x | comparable throughput | +| ember concurrent | **6.6x** | **6.5x** | memtier, pipelined | +| ember sharded | 1.4x | 0.7x | mixed results depending on workload | **important caveat**: these benchmarks should be taken with a grain of salt. ember is a small indie project built for learning and experimentation. Redis and Dragonfly are production-grade systems developed by large teams over many years, battle-tested at massive scale. dragonfly in particular offers features ember simply doesn't have: -- full Redis API compatibility (200+ commands vs ember's ~85) +- full Redis API compatibility (200+ commands vs ember's ~101) - sophisticated memory management (dashtable for ~25% of Redis memory usage) - transactional semantics (MULTI/EXEC, Lua scripting) - fork-free snapshotting @@ -55,21 +61,30 @@ dragonfly in particular offers features ember simply doesn't have: ember's concurrent mode shows higher throughput on simple GET/SET because it's architecturally minimal — essentially a concurrent hashmap with RESP3 parsing. this simplicity is a tradeoff, not an advantage. for anything resembling production use, Redis and Dragonfly are the sensible choices. ember exists primarily as a learning project and for workloads where simplicity matters more than features. -### latency (48 clients, no pipelining, memtier_benchmark) +### latency (P=16, 48 clients, memtier_benchmark) | server | p99 SET | p99 GET | |--------|---------|---------| -| ember concurrent | 0.41ms | 0.38ms | -| ember sharded | 0.56ms | 0.51ms | -| redis | 0.55ms | 0.54ms | -| dragonfly | 0.98ms | 0.98ms | +| ember concurrent | 1.56ms | 1.56ms | +| ember sharded | 2.29ms | 1.98ms | +| redis | 1.16ms | 1.19ms | +| dragonfly | 1.60ms | 1.46ms | + +### latency (P=1, 48 clients, memtier_benchmark) + +| server | p99 SET | p99 GET | +|--------|---------|---------| +| ember concurrent | 0.64ms | 0.61ms | +| ember sharded | 0.88ms | 0.83ms | +| redis | 0.58ms | 0.56ms | +| dragonfly | 1.15ms | 1.14ms | ### memory usage (~1M keys, 64B values) | server | memory | per key | |--------|--------|---------| | ember | 161 MB | ~161 bytes | -| redis | 105 MB | ~105 bytes | +| redis | 95 MB | ~95 bytes | ember uses more memory per key due to storing additional metadata for LRU eviction and expiration tracking. @@ -113,7 +128,7 @@ ember offers two modes with different tradeoffs: **concurrent mode** (`--concurrent`): - uses DashMap for lock-free access -- 1.9-2.2x faster than redis for GET/SET +- 2.3-3.0x faster than redis for GET/SET (pipelined) - only supports string operations - best for simple key-value workloads @@ -121,7 +136,7 @@ ember offers two modes with different tradeoffs: - each CPU core owns a keyspace partition - requests routed via tokio channels - supports all data types (lists, hashes, sets, sorted sets) -- ~0.9x redis throughput with pipelining, 2x faster without pipelining +- 1.1-1.2x redis throughput without pipelining, lower with heavy pipelining due to channel overhead ## running benchmarks diff --git a/bench/results/20260212-184126-gcp-c2s8.csv b/bench/results/20260212-184126-gcp-c2s8.csv new file mode 100644 index 00000000..c73ed380 --- /dev/null +++ b/bench/results/20260212-184126-gcp-c2s8.csv @@ -0,0 +1,9 @@ +test,ember_concurrent_rps,ember_sharded_rps,redis_rps,dragonfly_rps +SET (3B, P=16),1333333,799360,999000,798722 +GET (3B, P=16),1996008,799360,999000,798722 +SET (64B, P=16),1331558,798722,799360,797448 +GET (64B, P=16),1996008,798722,999000,798084 +SET (1KB, P=16),999000,570125,798722,664893 +GET (1KB, P=16),1333333,798722,798722,332446 +SET (64B, P=1),190331,173822,114246,210393 +GET (64B, P=1),190403,173761,117605,222172 diff --git a/bench/results/20260212-184406-gcp-c2s8-memtier.csv b/bench/results/20260212-184406-gcp-c2s8-memtier.csv new file mode 100644 index 00000000..49b21586 --- /dev/null +++ b/bench/results/20260212-184406-gcp-c2s8-memtier.csv @@ -0,0 +1,9 @@ +test,ember_concurrent_ops,ember_sharded_ops,redis_ops,dragonfly_ops,ec_p99,es_p99,redis_p99,dragonfly_p99 +SET (64B, P=16),4562538,987530,2022543,694116,1.559,2.287,1.159,1.599 +GET (64B, P=16),6624838,762401,2225336,1021986,1.559,1.975,1.191,1.455 +mixed 1:10 (64B, P=16),1727847,1083253,1165414,702866,1.415,1.999,1.007,1.527 +mixed 1:1 (64B, P=16),1680994,1039764,1035432,959765,1.647,2.303,1.151,1.591 +SET (1KB, P=16),933385,764879,626130,896308,3.439,4.735,1.727,2.207 +GET (1KB, P=16),903312,724275,593297,337179,1.631,2.575,2.351,4.351 +SET (64B, P=1),170655,171517,154439,171582,0.639,0.879,0.583,1.151 +GET (64B, P=1),185528,188114,160935,167916,0.607,0.831,0.559,1.135