Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ a low-latency, memory-efficient, distributed cache written in Rust. designed to
- **list operations** — LPUSH, RPUSH, LPOP, RPOP, LRANGE, LLEN
- **sorted sets** — ZADD (with NX/XX/GT/LT/CH), ZREM, ZSCORE, ZRANK, ZRANGE, ZCARD
- **hashes** — HSET, HGET, HGETALL, HDEL, HEXISTS, HLEN, HINCRBY, HKEYS, HVALS, HMGET
- **sets** — SADD, SREM, SMEMBERS, SISMEMBER, SCARD
- **key commands** — DEL, EXISTS, EXPIRE, TTL, PEXPIRE, PTTL, PERSIST, TYPE, SCAN
- **server commands** — PING, ECHO, INFO, DBSIZE, FLUSHDB, BGSAVE, BGREWRITEAOF
- **sharded engine** — shared-nothing, thread-per-core design with no cross-shard locking
Expand Down Expand Up @@ -71,6 +72,13 @@ redis-cli HGET user:1 name # => "alice"
redis-cli HGETALL user:1 # => 1) "name" 2) "alice" 3) "age" 4) "30"
redis-cli HINCRBY user:1 age 1 # => (integer) 31

# sets
redis-cli SADD tags rust cache fast # => (integer) 3
redis-cli SMEMBERS tags # => 1) "cache" 2) "fast" 3) "rust"
redis-cli SISMEMBER tags rust # => (integer) 1
redis-cli SCARD tags # => (integer) 3
redis-cli SREM tags fast # => (integer) 1

# iteration
redis-cli SCAN 0 MATCH "user:*" COUNT 100
redis-cli DBSIZE # => (integer) 6
Expand Down
245 changes: 245 additions & 0 deletions crates/ember-core/src/keyspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1366,6 +1366,162 @@ impl Keyspace {
}
}

// -------------------------------------------------------------------------
// Set operations
// -------------------------------------------------------------------------

/// Adds one or more members to a set.
///
/// Creates the set if the key doesn't exist. Returns the number of
/// new members added (existing members don't count).
pub fn sadd(&mut self, key: &str, members: &[String]) -> Result<usize, WriteError> {
if members.is_empty() {
return Ok(0);
}

self.remove_if_expired(key);

let is_new = !self.entries.contains_key(key);

if !is_new && !matches!(self.entries[key].value, Value::Set(_)) {
return Err(WriteError::WrongType);
}

// estimate memory increase before mutating
let member_increase: usize = members
.iter()
.map(|m| m.len() + memory::HASHSET_MEMBER_OVERHEAD)
.sum();
let estimated_increase = if is_new {
memory::ENTRY_OVERHEAD + key.len() + memory::HASHSET_BASE_OVERHEAD + member_increase
} else {
member_increase
};
if !self.enforce_memory_limit(estimated_increase) {
return Err(WriteError::OutOfMemory);
}

if is_new {
let value = Value::Set(std::collections::HashSet::new());
self.memory.add(key, &value);
self.entries.insert(key.to_owned(), Entry::new(value, None));
}

let entry = self
.entries
.get_mut(key)
.expect("just inserted or verified");
let old_entry_size = memory::entry_size(key, &entry.value);

let mut added = 0;
if let Value::Set(ref mut set) = entry.value {
for member in members {
if set.insert(member.clone()) {
added += 1;
}
}
}
entry.touch();

let new_entry_size = memory::entry_size(key, &entry.value);
self.memory.adjust(old_entry_size, new_entry_size);

Ok(added)
}

/// Removes one or more members from a set.
///
/// Returns the number of members that were actually removed.
pub fn srem(&mut self, key: &str, members: &[String]) -> Result<usize, WrongType> {
if self.remove_if_expired(key) {
return Ok(0);
}

match self.entries.get(key) {
None => return Ok(0),
Some(e) => {
if !matches!(e.value, Value::Set(_)) {
return Err(WrongType);
}
}
}

let old_entry_size = memory::entry_size(key, &self.entries[key].value);
let entry = self.entries.get_mut(key).expect("verified above");

let mut removed = 0;
let is_empty = if let Value::Set(ref mut set) = entry.value {
for member in members {
if set.remove(member) {
removed += 1;
}
}
set.is_empty()
} else {
false
};

if is_empty {
self.memory.remove_with_size(old_entry_size);
self.entries.remove(key);
} else {
let new_entry_size = memory::entry_size(key, &self.entries[key].value);
self.memory.adjust(old_entry_size, new_entry_size);
}

Ok(removed)
}

/// Returns all members of a set.
pub fn smembers(&mut self, key: &str) -> Result<Vec<String>, WrongType> {
if self.remove_if_expired(key) {
return Ok(vec![]);
}
match self.entries.get_mut(key) {
None => Ok(vec![]),
Some(entry) => match &entry.value {
Value::Set(set) => {
let result = set.iter().cloned().collect();
entry.touch();
Ok(result)
}
_ => Err(WrongType),
},
}
}

/// Checks if a member exists in a set.
pub fn sismember(&mut self, key: &str, member: &str) -> Result<bool, WrongType> {
if self.remove_if_expired(key) {
return Ok(false);
}
match self.entries.get_mut(key) {
None => Ok(false),
Some(entry) => match &entry.value {
Value::Set(set) => {
let result = set.contains(member);
entry.touch();
Ok(result)
}
_ => Err(WrongType),
},
}
}

/// Returns the cardinality (number of elements) of a set.
pub fn scard(&mut self, key: &str) -> Result<usize, WrongType> {
if self.remove_if_expired(key) {
return Ok(0);
}
match self.entries.get(key) {
None => Ok(0),
Some(entry) => match &entry.value {
Value::Set(set) => Ok(set.len()),
_ => Err(WrongType),
},
}
}

/// Randomly samples up to `count` keys and removes any that have expired.
///
/// Returns the number of keys actually removed. Used by the active
Expand Down Expand Up @@ -3003,4 +3159,93 @@ mod tests {
assert!(ks.hvals("s").is_err());
assert!(ks.hmget("s", &["f".into()]).is_err());
}

// --- set tests ---

#[test]
fn sadd_creates_set() {
let mut ks = Keyspace::new();
let added = ks.sadd("s", &["a".into(), "b".into()]).unwrap();
assert_eq!(added, 2);
assert_eq!(ks.value_type("s"), "set");
}

#[test]
fn sadd_returns_new_member_count() {
let mut ks = Keyspace::new();
ks.sadd("s", &["a".into(), "b".into()]).unwrap();
// add one existing, one new
let added = ks.sadd("s", &["b".into(), "c".into()]).unwrap();
assert_eq!(added, 1); // only "c" is new
}

#[test]
fn srem_removes_members() {
let mut ks = Keyspace::new();
ks.sadd("s", &["a".into(), "b".into(), "c".into()]).unwrap();
let removed = ks.srem("s", &["a".into(), "c".into()]).unwrap();
assert_eq!(removed, 2);
assert_eq!(ks.scard("s").unwrap(), 1);
}

#[test]
fn srem_auto_deletes_empty_set() {
let mut ks = Keyspace::new();
ks.sadd("s", &["only".into()]).unwrap();
ks.srem("s", &["only".into()]).unwrap();
assert_eq!(ks.value_type("s"), "none");
}

#[test]
fn smembers_returns_all_members() {
let mut ks = Keyspace::new();
ks.sadd("s", &["a".into(), "b".into(), "c".into()]).unwrap();
let mut members = ks.smembers("s").unwrap();
members.sort();
assert_eq!(members, vec!["a", "b", "c"]);
}

#[test]
fn smembers_missing_key_returns_empty() {
let mut ks = Keyspace::new();
assert_eq!(ks.smembers("missing").unwrap(), Vec::<String>::new());
}

#[test]
fn sismember_returns_true_for_existing() {
let mut ks = Keyspace::new();
ks.sadd("s", &["member".into()]).unwrap();
assert!(ks.sismember("s", "member").unwrap());
}

#[test]
fn sismember_returns_false_for_missing() {
let mut ks = Keyspace::new();
ks.sadd("s", &["a".into()]).unwrap();
assert!(!ks.sismember("s", "missing").unwrap());
}

#[test]
fn scard_returns_count() {
let mut ks = Keyspace::new();
ks.sadd("s", &["a".into(), "b".into(), "c".into()]).unwrap();
assert_eq!(ks.scard("s").unwrap(), 3);
}

#[test]
fn scard_missing_key_returns_zero() {
let mut ks = Keyspace::new();
assert_eq!(ks.scard("missing").unwrap(), 0);
}

#[test]
fn set_on_string_key_returns_wrongtype() {
let mut ks = Keyspace::new();
ks.set("s".into(), Bytes::from("string"), None);
assert!(ks.sadd("s", &["m".into()]).is_err());
assert!(ks.srem("s", &["m".into()]).is_err());
assert!(ks.smembers("s").is_err());
assert!(ks.sismember("s", "m").is_err());
assert!(ks.scard("s").is_err());
}
}
12 changes: 12 additions & 0 deletions crates/ember-core/src/memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ pub(crate) const HASHMAP_ENTRY_OVERHEAD: usize = 64;
/// Base overhead for an empty HashMap (bucket array pointer + len + capacity).
pub(crate) const HASHMAP_BASE_OVERHEAD: usize = 48;

/// Estimated overhead per member in a HashSet.
///
/// Each member is a String (24 bytes ptr+len+cap) plus bucket overhead.
pub(crate) const HASHSET_MEMBER_OVERHEAD: usize = 40;

/// Base overhead for an empty HashSet (bucket array pointer + len + capacity).
pub(crate) const HASHSET_BASE_OVERHEAD: usize = 48;

/// Returns the byte size of a value's payload.
pub fn value_size(value: &Value) -> usize {
match value {
Expand All @@ -150,6 +158,10 @@ pub fn value_size(value: &Value) -> usize {
.sum();
HASHMAP_BASE_OVERHEAD + entry_bytes
}
Value::Set(set) => {
let member_bytes: usize = set.iter().map(|m| m.len() + HASHSET_MEMBER_OVERHEAD).sum();
HASHSET_BASE_OVERHEAD + member_bytes
}
}
}

Expand Down
41 changes: 41 additions & 0 deletions crates/ember-core/src/shard.rs
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,24 @@ pub enum ShardRequest {
key: String,
fields: Vec<String>,
},
SAdd {
key: String,
members: Vec<String>,
},
SRem {
key: String,
members: Vec<String>,
},
SMembers {
key: String,
},
SIsMember {
key: String,
member: String,
},
SCard {
key: String,
},
/// Returns the key count for this shard.
DbSize,
/// Returns keyspace stats for this shard.
Expand Down Expand Up @@ -334,6 +352,7 @@ async fn run_shard(
Value::SortedSet(ss)
}
RecoveredValue::Hash(map) => Value::Hash(map),
RecoveredValue::Set(set) => Value::Set(set),
};
keyspace.restore(entry.key, value, entry.expires_at);
}
Expand Down Expand Up @@ -648,6 +667,27 @@ fn dispatch(ks: &mut Keyspace, req: &ShardRequest) -> ShardResponse {
Ok(vals) => ShardResponse::OptionalArray(vals),
Err(_) => ShardResponse::WrongType,
},
ShardRequest::SAdd { key, members } => match ks.sadd(key, members) {
Ok(count) => ShardResponse::Len(count),
Err(WriteError::WrongType) => ShardResponse::WrongType,
Err(WriteError::OutOfMemory) => ShardResponse::OutOfMemory,
},
ShardRequest::SRem { key, members } => match ks.srem(key, members) {
Ok(count) => ShardResponse::Len(count),
Err(_) => ShardResponse::WrongType,
},
ShardRequest::SMembers { key } => match ks.smembers(key) {
Ok(members) => ShardResponse::StringArray(members),
Err(_) => ShardResponse::WrongType,
},
ShardRequest::SIsMember { key, member } => match ks.sismember(key, member) {
Ok(exists) => ShardResponse::Bool(exists),
Err(_) => ShardResponse::WrongType,
},
ShardRequest::SCard { key } => match ks.scard(key) {
Ok(count) => ShardResponse::Len(count),
Err(_) => ShardResponse::WrongType,
},
// snapshot/rewrite are handled in the main loop, not here
ShardRequest::Snapshot | ShardRequest::RewriteAof => ShardResponse::Ok,
}
Expand Down Expand Up @@ -804,6 +844,7 @@ fn write_snapshot(
SnapValue::SortedSet(members)
}
Value::Hash(map) => SnapValue::Hash(map.clone()),
Value::Set(set) => SnapValue::Set(set.clone()),
};
writer.write_entry(&SnapEntry {
key: key.to_owned(),
Expand Down
Loading