chore: observability audit fixes and documentation updates

Dockerfile

@@ -16,5 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY --from=builder /usr/src/ember/target/release/ember-server /usr/local/bin/ember-server
 
 EXPOSE 6379
+# metrics port (set via --metrics-port at runtime)
+EXPOSE 9100
 
 ENTRYPOINT ["ember-server", "--host", "0.0.0.0"]

README.md

@@ -16,6 +16,7 @@ a low-latency, memory-efficient, distributed cache written in Rust. designed to
 - **sets** — SADD, SREM, SMEMBERS, SISMEMBER, SCARD
 - **key commands** — DEL, EXISTS, EXPIRE, TTL, PEXPIRE, PTTL, PERSIST, TYPE, SCAN
 - **server commands** — PING, ECHO, INFO, DBSIZE, FLUSHDB, BGSAVE, BGREWRITEAOF
+- **observability** — prometheus metrics (`--metrics-port`), enriched INFO with 6 sections, SLOWLOG command
 - **sharded engine** — shared-nothing, thread-per-core design with no cross-shard locking
 - **active expiration** — background sampling cleans up expired keys without client access
 - **memory limits** — per-shard byte-level accounting with configurable limits
@@ -97,6 +98,9 @@ redis-cli FLUSHDB               # => OK
 | `--data-dir` | — | directory for persistence files |
 | `--appendonly` | false | enable append-only file logging |
 | `--appendfsync` | everysec | fsync policy: `always`, `everysec`, `no` |
+| `--metrics-port` | — | prometheus metrics HTTP port (disabled when not set) |
+| `--slowlog-log-slower-than` | 10000 | log commands slower than N microseconds (-1 disables) |
+| `--slowlog-max-len` | 128 | max entries in slow log ring buffer |
 
 ## build & development
 
@@ -140,9 +144,9 @@ ember uses a shared-nothing, thread-per-core design inspired by [Dragonfly](http
 | 2 | persistence (AOF, snapshots, recovery) | ✅ complete |
 | 3 | data types (sorted sets, lists, hashes, sets) | ✅ complete |
 | 4 | clustering (raft, gossip, slots, migration) | ✅ complete |
-| 5 | developer experience (observability, CLI, clients) | 🚧 not started |
+| 5 | developer experience (observability, CLI, clients) | 🚧 in progress |
 
-**current**: 62 commands, 574 tests, ~18k lines of code
+**current**: 65 commands, 579 tests, ~18k lines of code
 
 ## security
 

crates/ember-protocol/README.md

@@ -48,7 +48,7 @@ let cmd = Command::from_frame(frame).unwrap();
 
 **keys**: `DEL`, `EXISTS`, `EXPIRE`, `PEXPIRE`, `TTL`, `PTTL`, `PERSIST`, `TYPE`, `SCAN`
 
-**server**: `PING`, `ECHO`, `DBSIZE`, `INFO`, `BGSAVE`, `BGREWRITEAOF`, `FLUSHDB`
+**server**: `PING`, `ECHO`, `DBSIZE`, `INFO`, `BGSAVE`, `BGREWRITEAOF`, `FLUSHDB`, `SLOWLOG GET`, `SLOWLOG LEN`, `SLOWLOG RESET`
 
 **cluster**: `CLUSTER INFO`, `NODES`, `SLOTS`, `KEYSLOT`, `MYID`, `MEET`, `ADDSLOTS`, `DELSLOTS`, `SETSLOT`, `FORGET`, `REPLICATE`, `FAILOVER`, `COUNTKEYSINSLOT`, `GETKEYSINSLOT`, `MIGRATE`, `ASKING`
 

crates/ember-server/README.md

@@ -4,10 +4,12 @@ the main server binary for [ember](https://github.com/kacy/ember). accepts TCP c
 
 ## what's in here
 
-- **main** — CLI arg parsing (host, port, max-memory, eviction policy, persistence config)
+- **main** — CLI arg parsing (host, port, max-memory, eviction policy, persistence, metrics, slowlog config)
 - **server** — TCP accept loop with configurable connection limits, graceful shutdown on SIGINT/SIGTERM, spawns a handler task per client
-- **connection** — per-connection event loop: read → parse frames → dispatch commands → write responses. handles idle timeouts, buffer limits, and protocol errors
+- **connection** — per-connection event loop: read → parse frames → dispatch commands → write responses. handles idle timeouts, buffer limits, and protocol errors. renders multi-section INFO and handles SLOWLOG commands
 - **config** — configuration helpers for byte sizes, eviction policies, fsync policies
+- **metrics** — prometheus exporter, per-command histogram/counter recording, background stats poller
+- **slowlog** — ring buffer for slow command logging with configurable threshold and capacity
 
 ## running
 
@@ -20,6 +22,9 @@ cargo run --release -p ember-server -- --max-memory 256M --eviction-policy allke
 
 # with AOF persistence
 cargo run --release -p ember-server -- --data-dir ./data --appendonly --appendfsync everysec
+
+# with prometheus metrics on port 9100
+cargo run --release -p ember-server -- --metrics-port 9100
 ```
 
 compatible with `redis-cli` and any RESP3 client.

crates/ember-server/src/connection.rs

@@ -40,7 +40,6 @@ pub async fn handle(
     engine: Engine,
     ctx: &Arc<ServerContext>,
     slow_log: &Arc<SlowLog>,
-    metrics_enabled: bool,
 ) -> Result<(), Box<dyn std::error::Error>> {
     // disable Nagle's algorithm — cache servers need low-latency writes,
     // and we already batch responses from pipelining into a single write
@@ -76,7 +75,7 @@ pub async fn handle(
                 Ok(Some((frame, consumed))) => {
                     let _ = buf.split_to(consumed);
                     let response =
-                        process(frame, &engine, ctx, slow_log, metrics_enabled).await;
+                        process(frame, &engine, ctx, slow_log).await;
                     response.serialize(&mut out);
                 }
                 Ok(None) => break, // need more data
@@ -97,29 +96,35 @@ pub async fn handle(
 
 /// Converts a raw frame into a command and executes it.
 ///
-/// When `metrics_enabled` is true, records per-command latency and
-/// error counters in prometheus.
+/// When metrics or slowlog are enabled, brackets the command with
+/// `Instant::now()` to measure latency. Skips timing entirely when
+/// neither feature needs it.
 async fn process(
     frame: Frame,
     engine: &Engine,
     ctx: &Arc<ServerContext>,
     slow_log: &Arc<SlowLog>,
-    metrics_enabled: bool,
 ) -> Frame {
     match Command::from_frame(frame) {
         Ok(cmd) => {
             let cmd_name = cmd.command_name();
-            let start = Instant::now();
+            let needs_timing = ctx.metrics_enabled || slow_log.is_enabled();
+            let start = if needs_timing {
+                Some(Instant::now())
+            } else {
+                None
+            };
 
             let response = execute(cmd, engine, ctx, slow_log).await;
-            let elapsed = start.elapsed();
-
             ctx.commands_processed.fetch_add(1, Ordering::Relaxed);
-            slow_log.maybe_record(elapsed, cmd_name);
 
-            if metrics_enabled {
-                let is_error = matches!(&response, Frame::Error(_));
-                crate::metrics::record_command(cmd_name, elapsed, is_error);
+            if let Some(start) = start {
+                let elapsed = start.elapsed();
+                slow_log.maybe_record(elapsed, cmd_name);
+                if ctx.metrics_enabled {
+                    let is_error = matches!(&response, Frame::Error(_));
+                    crate::metrics::record_command(cmd_name, elapsed, is_error);
+                }
             }
 
             response
@@ -1014,7 +1019,7 @@ async fn render_info(
     }
 
     if want("CLIENTS") {
-        let connected = ctx.connections_accepted.load(Ordering::Relaxed);
+        let connected = ctx.connections_active.load(Ordering::Relaxed);
         out.push_str("# Clients\r\n");
         out.push_str(&format!("connected_clients:{connected}\r\n"));
         out.push_str(&format!("max_clients:{}\r\n", ctx.max_connections));

crates/ember-server/src/server.rs

@@ -31,7 +31,9 @@ pub struct ServerContext {
     pub max_connections: usize,
     pub max_memory: Option<usize>,
     pub aof_enabled: bool,
+    pub metrics_enabled: bool,
     pub connections_accepted: AtomicU64,
+    pub connections_active: AtomicU64,
     pub commands_processed: AtomicU64,
 }
 
@@ -81,7 +83,9 @@ pub async fn run(
         max_connections: max_conn,
         max_memory,
         aof_enabled,
+        metrics_enabled,
         connections_accepted: AtomicU64::new(0),
+        connections_active: AtomicU64::new(0),
         commands_processed: AtomicU64::new(0),
     });
 
@@ -123,17 +127,18 @@ pub async fn run(
                     crate::metrics::on_connection_accepted();
                 }
                 ctx.connections_accepted.fetch_add(1, Ordering::Relaxed);
+                ctx.connections_active.fetch_add(1, Ordering::Relaxed);
 
                 let engine = engine.clone();
                 let ctx = Arc::clone(&ctx);
                 let slow_log = Arc::clone(&slow_log);
-                let metrics = metrics_enabled;
 
                 tokio::spawn(async move {
-                    if let Err(e) = connection::handle(stream, engine, &ctx, &slow_log, metrics).await {
+                    if let Err(e) = connection::handle(stream, engine, &ctx, &slow_log).await {
                         error!("connection error from {peer}: {e}");
                     }
-                    if metrics {
+                    ctx.connections_active.fetch_sub(1, Ordering::Relaxed);
+                    if ctx.metrics_enabled {
                         crate::metrics::on_connection_closed();
                     }
                     // permit is dropped here, releasing the slot

crates/ember-server/src/slowlog.rs

@@ -67,16 +67,34 @@ impl SlowLog {
         }
     }
 
+    /// Returns whether the slow log is enabled.
+    ///
+    /// Used by the connection handler to skip timing overhead entirely
+    /// when both metrics and slowlog are disabled.
+    pub fn is_enabled(&self) -> bool {
+        self.config.enabled
+    }
+
     /// Records a command if it exceeded the threshold.
     ///
     /// Called from the connection handler after each command completes.
     /// The mutex is effectively uncontended since slow commands are rare.
+    /// If the lock is poisoned (another thread panicked while holding it),
+    /// we recover by clearing the entries rather than propagating the panic.
     pub fn maybe_record(&self, duration: Duration, command: &str) {
         if !self.config.enabled || duration < self.config.slower_than {
             return;
         }
 
-        let mut inner = self.inner.lock().expect("slowlog lock poisoned");
+        let mut inner = match self.inner.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => {
+                let mut guard = poisoned.into_inner();
+                guard.entries.clear();
+                guard
+            }
+        };
+
         let id = inner.next_id;
         inner.next_id += 1;
 
@@ -96,19 +114,28 @@ impl SlowLog {
     ///
     /// If `count` is `None`, returns all entries.
     pub fn get(&self, count: Option<usize>) -> Vec<SlowLogEntry> {
-        let inner = self.inner.lock().expect("slowlog lock poisoned");
+        let inner = match self.inner.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
         let n = count.unwrap_or(inner.entries.len()).min(inner.entries.len());
         inner.entries.iter().rev().take(n).cloned().collect()
     }
 
     /// Returns the number of entries currently in the log.
     pub fn len(&self) -> usize {
-        self.inner.lock().expect("slowlog lock poisoned").entries.len()
+        match self.inner.lock() {
+            Ok(guard) => guard.entries.len(),
+            Err(poisoned) => poisoned.into_inner().entries.len(),
+        }
     }
 
     /// Clears all entries from the log.
     pub fn reset(&self) {
-        let mut inner = self.inner.lock().expect("slowlog lock poisoned");
+        let mut inner = match self.inner.lock() {
+            Ok(guard) => guard,
+            Err(poisoned) => poisoned.into_inner(),
+        };
         inner.entries.clear();
     }
 }