diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a6a5e2..a48224b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,27 @@ jobs: - run: cargo check --all-targets --no-default-features --features sharded-lock - run: cargo check --all-targets --features shuttle + msrv: + name: MSRV + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: taiki-e/install-action@v2 + with: + tool: cargo-msrv + # `cargo msrv verify` checks that the crate compiles with the + # `rust-version` declared in Cargo.toml. The default `cargo check` skips + # dev-dependencies, which may require a newer Rust than downstream needs. + - run: cargo msrv verify + - run: cargo msrv verify -- cargo check --features stats + - run: cargo msrv verify -- cargo check --no-default-features + - run: cargo msrv verify -- cargo check --no-default-features --features sharded-lock + test: name: Tests strategy: diff --git a/Cargo.toml b/Cargo.toml index f6a8eef..f3700d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "quick_cache" -version = "0.6.21" +version = "0.7.0" edition = "2021" description = "Lightweight and high performance concurrent cache" repository = "https://github.com/arthurprs/quick-cache" diff --git a/README.md b/README.md index 046d03c..4174151 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Lightweight and high performance concurrent cache optimized for low cache overhe * Scales well with the number of threads * Atomic operations with `get_or_insert` and `get_value_or_guard` functions * Atomic async operations with `get_or_insert_async` and `get_value_or_guard_async` functions -* Non-blocking `try_get`, `try_insert`, `try_remove`, and related methods that return an error instead of blocking: typically `Err(LockContention)`, or `Err((Key, Val))` for `try_insert`/`try_insert_with_lifecycle` so inputs are preserved +* Non-blocking methods that return immediately on lock contention. * Closure-based `entry` API for atomic inspect-and-act patterns (keep, remove, replace) * Supports item pinning * Iteration and draining diff --git a/examples/eviction_listener.rs b/examples/eviction_listener.rs index 4861993..e5e5725 100644 --- a/examples/eviction_listener.rs +++ b/examples/eviction_listener.rs @@ -7,8 +7,6 @@ struct EvictionListener(mpsc::Sender<(u64, u64)>); impl Lifecycle for EvictionListener { type RequestState = (); - fn begin_request(&self) -> Self::RequestState {} - fn on_evict(&self, _state: &mut Self::RequestState, key: u64, val: u64) { let _ = self.0.send((key, val)); } diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 0f19630..3c864f2 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "quick_cache" -version = "0.6.17" +version = "0.7.0" dependencies = [ "ahash", "equivalent", diff --git a/fuzz/fuzz_targets/fuzz_sync_cache.rs b/fuzz/fuzz_targets/fuzz_sync_cache.rs index d5765f1..16238ed 100644 --- a/fuzz/fuzz_targets/fuzz_sync_cache.rs +++ b/fuzz/fuzz_targets/fuzz_sync_cache.rs @@ -31,10 +31,6 @@ impl Weighter for MyWeighter { impl Lifecycle for MyLifecycle { type RequestState = Vec<(u16, Value)>; - fn begin_request(&self) -> Self::RequestState { - Default::default() - } - fn is_pinned(&self, _key: &u16, val: &Value) -> bool { val.pinned } @@ -104,13 +100,15 @@ fn run(input: Input) { match op { Op::Insert(k, v, pinned) => { // eprintln!("insert {k} {v}"); - let evicted = cache.insert_with_lifecycle( + let mut evicted = Vec::new(); + cache.insert_with_lifecycle( k, Value { original: v, current: v, pinned, }, + &mut evicted, ); placeholders.remove(&k); // if k is present it must have value v @@ -121,15 +119,20 @@ fn run(input: Input) { Op::Replace(k, v, pinned) => { // eprintln!("replace {k} {v}"); placeholders.remove(&k); - if let Ok(evicted) = cache.replace_with_lifecycle( - k, - Value { - original: v, - current: v, - pinned, - }, - false, - ) { + let mut evicted = Vec::new(); + if cache + .replace_with_lifecycle( + k, + Value { + original: v, + current: v, + pinned, + }, + false, + &mut evicted, + ) + .is_ok() + { // if k is present it must have value v let peek = cache.peek(&k); assert!(peek.is_none() || peek.unwrap().original == v); @@ -155,7 +158,8 @@ fn run(input: Input) { current: v, pinned, }; - if let Ok(evicted) = p.insert_with_lifecycle(value) { + let mut evicted = Vec::new(); + if p.insert_with_lifecycle(value, &mut evicted).is_ok() { let peek = cache.peek(&k); assert!(peek.is_none() || peek.unwrap().original == v); check_evicted(k, peek, evicted); diff --git a/fuzz/fuzz_targets/fuzz_unsync_cache.rs b/fuzz/fuzz_targets/fuzz_unsync_cache.rs index 6ead4d4..02dc73b 100644 --- a/fuzz/fuzz_targets/fuzz_unsync_cache.rs +++ b/fuzz/fuzz_targets/fuzz_unsync_cache.rs @@ -26,10 +26,6 @@ impl Weighter for MyWeighter { impl Lifecycle for MyLifecycle { type RequestState = Vec<(u16, Value)>; - fn begin_request(&self) -> Self::RequestState { - Default::default() - } - fn is_pinned(&self, _key: &u16, val: &Value) -> bool { val.pinned } @@ -99,13 +95,15 @@ fn run(input: Input) { match op { Op::Insert(k, v, pinned) => { // eprintln!("insert {k} {v}"); - let evicted = cache.insert_with_lifecycle( + let mut evicted = Vec::new(); + cache.insert_with_lifecycle( k, Value { original: v, current: v, pinned, }, + &mut evicted, ); // if k is present it must have value v let peek = cache.peek(&k).copied(); @@ -125,15 +123,20 @@ fn run(input: Input) { } Op::Replace(k, v, pinned) => { // eprintln!("replace {k} {v}"); - if let Ok(evicted) = cache.replace_with_lifecycle( - k, - Value { - original: v, - current: v, - pinned, - }, - false, - ) { + let mut evicted = Vec::new(); + if cache + .replace_with_lifecycle( + k, + Value { + original: v, + current: v, + pinned, + }, + false, + &mut evicted, + ) + .is_ok() + { // if k is present it must have value v let peek = cache.peek(&k).copied(); assert!(peek.is_none() || peek.unwrap().original == v); @@ -146,11 +149,15 @@ fn run(input: Input) { let (inserted, evicted) = match cache.get_ref_or_guard(&k) { Ok(_) => (false, Vec::new()), Err(g) => { - let evicted = g.insert_with_lifecycle(Value { - original: v, - current: v, - pinned, - }); + let mut evicted = Vec::new(); + g.insert_with_lifecycle( + Value { + original: v, + current: v, + pinned, + }, + &mut evicted, + ); (true, evicted) } }; diff --git a/fuzz/fuzz_targets/fuzz_unsync_cache_pinstate.rs b/fuzz/fuzz_targets/fuzz_unsync_cache_pinstate.rs index d26a3aa..68257c0 100644 --- a/fuzz/fuzz_targets/fuzz_unsync_cache_pinstate.rs +++ b/fuzz/fuzz_targets/fuzz_unsync_cache_pinstate.rs @@ -34,10 +34,6 @@ impl Weighter for MyWeighter { impl Lifecycle for MyLifecycle { type RequestState = Vec<(u16, Value)>; - fn begin_request(&self) -> Self::RequestState { - Default::default() - } - fn is_pinned(&self, _key: &u16, val: &Value) -> bool { let mut pinned = val.pinned.get(); if pinned.remaining != 0 { @@ -114,13 +110,15 @@ fn run(input: Input) { match op { Op::Insert(k, v, pinned) => { // eprintln!("insert {k} {v}"); - let evicted = cache.insert_with_lifecycle( + let mut evicted = Vec::new(); + cache.insert_with_lifecycle( k, Value { original: v, current: v, pinned: Cell::new(pinned), }, + &mut evicted, ); // if k is present it must have value v let peek = cache.peek(&k).cloned(); @@ -138,15 +136,20 @@ fn run(input: Input) { } Op::Replace(k, v, pinned) => { // eprintln!("replace {k} {v}"); - if let Ok(evicted) = cache.replace_with_lifecycle( - k, - Value { - original: v, - current: v, - pinned: Cell::new(pinned), - }, - false, - ) { + let mut evicted = Vec::new(); + if cache + .replace_with_lifecycle( + k, + Value { + original: v, + current: v, + pinned: Cell::new(pinned), + }, + false, + &mut evicted, + ) + .is_ok() + { // if k is present it must have value v let peek = cache.peek(&k).cloned(); assert!(peek.is_none() || peek.as_ref().unwrap().original == v); @@ -159,11 +162,15 @@ fn run(input: Input) { let (inserted, evicted) = match cache.get_ref_or_guard(&k) { Ok(_) => (false, Vec::new()), Err(g) => { - let evicted = g.insert_with_lifecycle(Value { - original: v, - current: v, - pinned: Cell::new(pinned), - }); + let mut evicted = Vec::new(); + g.insert_with_lifecycle( + Value { + original: v, + current: v, + pinned: Cell::new(pinned), + }, + &mut evicted, + ); (true, evicted) } }; diff --git a/src/lib.rs b/src/lib.rs index 7840608..f706b8b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -79,7 +79,7 @@ //! | `parking_lot` | ✓ | Use [parking_lot](https://crates.io/crates/parking_lot) for synchronization primitives. Mutually exclusive with `sharded-lock`. | //! | `sharded-lock` | | Use [`crossbeam_utils::sync::ShardedLock`](https://docs.rs/crossbeam-utils/latest/crossbeam_utils/sync/struct.ShardedLock.html) for synchronization primitives. Mutually exclusive with `parking_lot`. | //! | `shuttle` | | Enable [shuttle](https://crates.io/crates/shuttle) testing support for concurrency testing. | -//! | `stats` | | Enable cache statistics tracking via the `hits()` and `misses()` methods. | +//! | `stats` | | Enable cache statistics tracking via the `hits()`, `misses()`, and per-item `item_stats()` methods. Overhead: adds an 8-byte per-item access counter (`AtomicU64`) to each resident item — raising per-entry memory by up to 8 bytes, depending on layout — and performs two atomic increments per cache hit and one per miss. | #![allow(clippy::type_complexity)] #![cfg_attr(docsrs, feature(doc_cfg))] @@ -164,8 +164,31 @@ impl Weighter for UnitWeighter { /// Hooks into the lifetime of the cache items. /// /// The functions should be small and very fast, otherwise the cache performance might be negatively affected. +/// +/// # Request state +/// +/// Operations that may evict items thread a [`RequestState`](Lifecycle::RequestState) +/// through the eviction hooks. It is a per-request accumulator: the cache constructs a +/// fresh one via [`Default`], the `on_evict*`/`before_evict` hooks record into it, and it +/// is finalized by its own [`Drop`] (which, for example, releases evicted items _after_ +/// the shard lock is dropped). +/// +/// The `_with_lifecycle` cache methods take `&mut RequestState`, letting a caller drive +/// several operations against a single state — e.g. to batch the eviction work or inspect +/// evicted items before dropping it: +/// +/// ```ignore +/// let mut lcs = Default::default(); +/// cache.insert_with_lifecycle(k1, v1, &mut lcs); +/// cache.insert_with_lifecycle(k2, v2, &mut lcs); +/// // inspect `lcs` here if desired; evicted items are released when it drops +/// ``` pub trait Lifecycle { - type RequestState; + /// Per-request accumulator threaded through the eviction hooks. + /// + /// Constructed via [`Default`] at the start of each request and finalized by its + /// [`Drop`]. Keep it cheap to create and drop. + type RequestState: Default; /// Returns whether the item is pinned. Items that are pinned can't be evicted. /// Note that a pinned item can still be replaced with get_mut, insert, replace and similar APIs. @@ -181,9 +204,6 @@ pub trait Lifecycle { false } - /// Called before the insert request starts, e.g.: insert, replace. - fn begin_request(&self) -> Self::RequestState; - /// Called when a cache item is about to be evicted. /// Note that value replacement (e.g. insertions for the same key) won't call this method. /// @@ -196,17 +216,43 @@ pub trait Lifecycle { fn before_evict(&self, state: &mut Self::RequestState, key: &Key, val: &mut Val) {} /// Called when an item is evicted. - fn on_evict(&self, state: &mut Self::RequestState, key: Key, val: Val); - - /// Called after a request finishes, e.g.: insert, replace. /// - /// Notes: - /// This will _not_ be called when using `_with_lifecycle` apis, which will return the RequestState instead. - /// This will _not_ be called if the request errored (e.g. a replace didn't find a value to replace). - /// If needed, Drop for RequestState can be used to detect these cases. + /// To distinguish evictions from the hot vs cold queues, override + /// [`Lifecycle::on_evict_hot`] and/or [`Lifecycle::on_evict_cold`] instead; + /// they default to delegating here. + /// + /// If none of `on_evict`, `on_evict_hot`, or `on_evict_cold` is overridden, + /// eviction notifications are silently dropped. + /// + /// Note: items that are rejected without ever being admitted to the cache + /// (oversized inserts and oversized placeholder values) are routed through + /// [`Lifecycle::on_evict_cold`], which by default reaches this method. #[allow(unused_variables)] #[inline] - fn end_request(&self, state: Self::RequestState) {} + fn on_evict(&self, state: &mut Self::RequestState, key: Key, val: Val) {} + + /// Called when an item is evicted from the cold queue. + /// + /// By default delegates to [`Lifecycle::on_evict`]. + /// + /// Note: items that are rejected without ever being admitted to the cache + /// (oversized inserts and oversized placeholder values) are also reported + /// via this method. + #[inline] + fn on_evict_cold(&self, state: &mut Self::RequestState, key: Key, val: Val) { + self.on_evict(state, key, val) + } + + /// Called when an item is evicted from the hot queue. + /// + /// By default delegates to [`Lifecycle::on_evict`]. + /// + /// Note: rejected (never-admitted) items are reported via + /// [`Lifecycle::on_evict_cold`], not this method. + #[inline] + fn on_evict_hot(&self, state: &mut Self::RequestState, key: Key, val: Val) { + self.on_evict(state, key, val) + } } /// The memory used by the cache @@ -225,6 +271,25 @@ impl MemoryUsed { } } +/// Per-item statistics returned by `item_stats`. +/// +/// Only available with the `stats` feature enabled. Enabling that feature adds an +/// 8-byte per-item access counter to each resident item (raising per-entry memory +/// by up to 8 bytes, depending on layout) and performs two atomic increments per +/// cache hit and one per miss. +#[cfg(feature = "stats")] +#[non_exhaustive] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct ItemStats { + /// Number of times the item has been accessed (read) since it became resident. + /// + /// Incremented on every cache hit (`get`/`get_mut`/`get_value_or_guard`/`entry`). + /// Unlike the internal eviction counter, this is monotonic per residency and is + /// not bounded by the eviction policy. It resets to zero if the slot is reused for + /// a new value (e.g. after eviction and re-insertion). + pub access_count: u64, +} + #[cfg(test)] mod tests { use std::{ diff --git a/src/linked_slab.rs b/src/linked_slab.rs index bd840da..728a25d 100644 --- a/src/linked_slab.rs +++ b/src/linked_slab.rs @@ -26,6 +26,20 @@ impl LinkedSlab { } } + /// Reserves capacity for at least `additional` more entries to be inserted. + /// + /// This pre-allocates the backing storage so that subsequent `insert`s don't + /// trigger a reallocation (and the associated copy) of the whole slab. + pub fn reserve(&mut self, additional: usize) { + self.entries.reserve(additional); + } + + /// The number of entries the slab can hold without reallocating. + #[cfg(test)] + pub fn capacity(&self) -> usize { + self.entries.capacity() + } + #[cfg(fuzzing)] pub fn len(&self) -> usize { self.entries.iter().filter(|e| e.item.is_some()).count() @@ -247,6 +261,28 @@ impl LinkedSlab { /// It should be noted that if cache key or value is some type like `Vec`, /// the memory allocated in the heap will not be counted. pub fn memory_used(&self) -> usize { - self.entries.len() * size_of::>() + self.entries.len() * std::mem::size_of::>() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reserve_avoids_realloc() { + let mut slab: LinkedSlab = LinkedSlab::with_capacity(0); + slab.reserve(1000); + let cap = slab.entries.capacity(); + assert!(cap >= 1000); + // Inserting up to the reserved capacity must not reallocate. + for i in 0..cap as u64 { + slab.insert(i); + } + assert_eq!( + slab.entries.capacity(), + cap, + "slab reallocated despite reserve" + ); } } diff --git a/src/shard.rs b/src/shard.rs index ec40b94..7b110a1 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -79,6 +79,11 @@ pub struct Resident { value: Val, state: ResidentState, referenced: AtomicU16, + /// Number of times this item has been accessed (read) since it became resident. + /// Incremented wherever a cache hit is recorded, unlike `referenced` which is + /// bounded by the eviction policy. Reset whenever the slot is reused for a new key. + #[cfg(feature = "stats")] + access_count: AtomicU64, } impl Clone for Resident { @@ -89,6 +94,8 @@ impl Clone for Resident { value: self.value.clone(), state: self.state, referenced: self.referenced.load(atomic::Ordering::Relaxed).into(), + #[cfg(feature = "stats")] + access_count: self.access_count.load(atomic::Ordering::Relaxed).into(), } } } @@ -176,12 +183,22 @@ macro_rules! record_hit { ($self: expr) => {{ $self.hits.fetch_add(1, atomic::Ordering::Relaxed); }}; + ($self: expr, $resident: expr) => {{ + $self.hits.fetch_add(1, atomic::Ordering::Relaxed); + $resident + .access_count + .fetch_add(1, atomic::Ordering::Relaxed); + }}; } #[cfg(feature = "stats")] macro_rules! record_hit_mut { ($self: expr) => {{ *$self.hits.get_mut() += 1; }}; + ($self: expr, $resident: expr) => {{ + *$self.hits.get_mut() += 1; + *$resident.access_count.get_mut() += 1; + }}; } #[cfg(feature = "stats")] macro_rules! record_miss { @@ -195,14 +212,15 @@ macro_rules! record_miss_mut { *$self.misses.get_mut() += 1; }}; } - #[cfg(not(feature = "stats"))] macro_rules! record_hit { ($self: expr) => {{}}; + ($self: expr, $resident: expr) => {{}}; } #[cfg(not(feature = "stats"))] macro_rules! record_hit_mut { ($self: expr) => {{}}; + ($self: expr, $resident: expr) => {{}}; } #[cfg(not(feature = "stats"))] macro_rules! record_miss { @@ -223,6 +241,7 @@ impl CacheShard(&self, hash: u64, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + let (_, resident) = self.search_resident(hash, key)?; + Some(crate::ItemStats { + access_count: resident.access_count.load(atomic::Ordering::Relaxed), + }) + } + pub fn peek_mut(&mut self, hash: u64, key: &Q) -> Option> where Q: Hash + Equivalent + ?Sized, @@ -783,7 +819,8 @@ impl< if self.num_non_resident > self.capacity_non_resident { self.advance_ghost(); } - self.lifecycle.on_evict(lcs, evicted.key, evicted.value); + self.lifecycle + .on_evict_cold(lcs, evicted.key, evicted.value); return true; } } @@ -835,7 +872,7 @@ impl< unsafe { core::hint::unreachable_unchecked() }; }; self.hot_head = next; - self.lifecycle.on_evict(lcs, evicted.key, evicted.value); + self.lifecycle.on_evict_hot(lcs, evicted.key, evicted.value); self.map_remove(hash, idx); } return true; @@ -900,6 +937,8 @@ impl< value, state: enter_state, referenced: referenced.into(), + #[cfg(feature = "stats")] + access_count: Default::default(), }), ); match evicted { @@ -920,7 +959,15 @@ impl< } else if evicted_weight != 0 && weight == 0 { *list_head = self.entries.unlink(idx); } - self.lifecycle.on_evict(lcs, evicted.key, evicted.value); + match enter_state { + ResidentState::Hot => { + self.lifecycle.on_evict_hot(lcs, evicted.key, evicted.value) + } + ResidentState::Cold => { + self.lifecycle + .on_evict_cold(lcs, evicted.key, evicted.value) + } + } } Entry::Ghost(_) => { self.weight_hot += weight; @@ -1020,6 +1067,8 @@ impl< value, state: placeholder_hot, referenced: (referenced as u16).into(), + #[cfg(feature = "stats")] + access_count: Default::default(), }); let list_head = if placeholder_hot == ResidentState::Hot { @@ -1052,7 +1101,7 @@ impl< ) -> Result<(), Val> { self.entries.remove(placeholder.idx()); self.map_remove(placeholder.hash(), placeholder.idx()); - self.lifecycle.on_evict(lcs, key, value); + self.lifecycle.on_evict_cold(lcs, key, value); Ok(()) } @@ -1102,6 +1151,8 @@ impl< value, state, referenced: Default::default(), + #[cfg(feature = "stats")] + access_count: Default::default(), })); if weight != 0 { *list_head = Some(self.entries.link(idx, *list_head)); @@ -1120,15 +1171,19 @@ impl< strategy: InsertStrategy, ) -> Result<(), (Key, Val)> { // Make sure to remove any existing entry - if let Some((idx, _)) = self.search_resident(hash, &key) { + if let Some((idx, resident)) = self.search_resident(hash, &key) { + let prev_state = resident.state; if let Some((ek, ev)) = self.remove_internal(hash, idx) { - self.lifecycle.on_evict(lcs, ek, ev); + match prev_state { + ResidentState::Hot => self.lifecycle.on_evict_hot(lcs, ek, ev), + ResidentState::Cold => self.lifecycle.on_evict_cold(lcs, ek, ev), + } } } if matches!(strategy, InsertStrategy::Replace { .. }) { return Err((key, value)); } - self.lifecycle.on_evict(lcs, key, value); + self.lifecycle.on_evict_cold(lcs, key, value); Ok(()) } @@ -1146,7 +1201,7 @@ impl< if *resident.referenced.get_mut() < MAX_F { *resident.referenced.get_mut() += 1; } - record_hit_mut!(self); + record_hit_mut!(self, resident); unsafe { // Rustc gets insanely confused returning references from mut borrows // Safety: value will have the same lifetime as `resident` @@ -1197,7 +1252,6 @@ impl< return match action { EntryAction::Retain(t) => { - record_hit_mut!(self); let Some((Entry::Resident(resident), _)) = self.entries.get_mut(idx) else { // SAFETY: we had a mut reference to the Resident under `idx` until the previous line unsafe { unreachable_unchecked() }; @@ -1205,6 +1259,7 @@ impl< if *resident.referenced.get_mut() < MAX_F { *resident.referenced.get_mut() += 1; } + record_hit_mut!(self, resident); EntryOrPlaceholder::Kept(t) } EntryAction::Remove => { @@ -1307,7 +1362,7 @@ impl< } } - pub fn set_capacity(&mut self, new_weight_capacity: u64) { + pub fn set_capacity(&mut self, new_weight_capacity: u64, lcs: &mut L::RequestState) { // Guard against division by zero when old capacity is 0 (produces inf/NaN ratios) if self.weight_capacity == 0 { self.weight_capacity = new_weight_capacity; @@ -1326,11 +1381,7 @@ impl< } // Evict items if we're over the new capacity - let mut lcs = self.lifecycle.begin_request(); - while self.weight_hot + self.weight_cold > self.weight_capacity - && self.advance_cold(&mut lcs) - {} - self.lifecycle.end_request(lcs); + while self.weight_hot + self.weight_cold > self.weight_capacity && self.advance_cold(lcs) {} // Trim ghost entries if needed while self.num_non_resident > self.capacity_non_resident { self.advance_ghost(); @@ -1406,17 +1457,60 @@ impl, B, L, Plh: SharedPlaceholder> mod tests { use super::*; + #[cfg(not(feature = "stats"))] + #[test] + fn reserve_caps_ghost_headroom() { + // A small reserve on a shard with a large estimated capacity (hence a + // large `capacity_non_resident`) must not over-allocate the slab by the + // full ghost cap; the ghost headroom is bounded by `additional`. + let mut shard = CacheShard::< + u64, + u64, + crate::UnitWeighter, + crate::DefaultHashBuilder, + crate::sync::DefaultLifecycle, + crate::sync_placeholder::SharedPlaceholder, + >::new( + DEFAULT_HOT_ALLOCATION, + 0.5, // ghost_allocation -> capacity_non_resident = 500_000 + 1_000_000, // estimated_items_capacity + u64::MAX, // weight_capacity + crate::UnitWeighter, + crate::DefaultHashBuilder::default(), + crate::sync::DefaultLifecycle::default(), + ); + assert_eq!(shard.capacity_non_resident, 500_000); + shard.reserve(100); + // Ghost headroom is min(additional, capacity_non_resident) = 100, so the + // slab reserves ~200 entries, not 500_000+. + assert!( + shard.entries.capacity() < 1_000, + "slab over-allocated: {}", + shard.entries.capacity() + ); + } + #[test] fn entry_overhead() { use std::mem::size_of; + // 8 bytes from the linked slab, 8 bytes from the entry enum. + // `stats` adds an 8-byte `access_count` to each `Resident` (24 -> 32 bytes). + // Whether the slab entry grows then depends on enum discriminant/niche layout: + // the sync entry's discriminant moves into the `Arc` niche, cancelling the + // growth (stays 32), while the unsync entry loses its niche and grows by 8. + // (Layout-dependent and not guaranteed stable across rustc versions.) assert_eq!( size_of::>>() - size_of::<[u64; 2]>(), - 16 // 8 bytes from linked slab, 8 bytes from entry + 16 ); + #[cfg(not(feature = "stats"))] + let unsync_overhead = 16; + #[cfg(feature = "stats")] + let unsync_overhead = 24; assert_eq!( size_of::>() - size_of::<[u64; 2]>(), - 16 // 8 bytes from linked slab, 8 bytes from entry + unsync_overhead ); } } diff --git a/src/sync.rs b/src/sync.rs index 3e0aa34..42da03a 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -59,7 +59,6 @@ pub struct Cache< hash_builder: B, shards: Box<[RwLock>>]>, shards_mask: u64, - lifecycle: L, } impl Cache { @@ -76,15 +75,6 @@ impl Cache { } impl + Clone> Cache { - /// Creates a new cache with a custom [`Weighter`]. - /// - /// - `estimated_items_capacity` — expected number of items the cache will hold, - /// roughly `weight_capacity / average_item_weight`. - /// - `weight_capacity` — total weight the cache may hold across all shards. - /// - `weighter` — determines the weight of each key–value pair. - /// - /// Use [`Cache::new`] when each item has unit weight (i.e. when you only care - /// about item count, not size). pub fn with_weighter( estimated_items_capacity: usize, weight_capacity: u64, @@ -179,7 +169,6 @@ impl< shards: shards.into_boxed_slice(), hash_builder, shards_mask: num_shards - 1, - lifecycle, } } @@ -370,6 +359,35 @@ impl< } } + /// Returns per-item statistics for `key`, or `None` if the key is not present. + /// Like peeks, this does not alter the key "hotness" or its access count. + #[cfg(feature = "stats")] + pub fn item_stats(&self, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + let (shard, hash) = self.shard_for(key)?; + shard.read().item_stats(hash, key) + } + + /// Attempts to return per-item statistics for `key`. + /// Like peeks, this does not alter the key "hotness" or its access count. + /// Returns `Ok(Some(stats))` if the key is present, `Ok(None)` if absent, + /// or `Err(LockContention)` if the shard lock could not be acquired without blocking. + #[cfg(feature = "stats")] + pub fn try_item_stats(&self, key: &Q) -> Result, LockContention> + where + Q: Hash + Equivalent + ?Sized, + { + let Some((shard, hash)) = self.shard_for(key) else { + return Ok(None); + }; + match shard.try_read() { + Some(guard) => Ok(guard.item_stats(hash, key)), + None => Err(LockContention), + } + } + /// Remove an item from the cache whose key is `key`. /// Returns the removed entry, if any. pub fn remove(&self, key: &Q) -> Option<(Key, Val)> @@ -416,36 +434,32 @@ impl< /// /// Returns `Ok` if the entry was admitted and `Err(_)` if it wasn't. pub fn replace(&self, key: Key, value: Val, soft: bool) -> Result<(), (Key, Val)> { - let lcs = self.replace_with_lifecycle(key, value, soft)?; - self.lifecycle.end_request(lcs); - Ok(()) + let mut lcs = Default::default(); + self.replace_with_lifecycle(key, value, soft, &mut lcs) } /// Inserts an item in the cache, but _only_ if an entry with key `key` already exists, - /// returning the lifecycle request state. - /// + /// recording any evicted items into the given lifecycle request state. /// If `soft` is set, the replace operation won't affect the "hotness" of the entry, /// even if the value is replaced. /// - /// Returns `Ok(lcs)` with the lifecycle request state if the entry was replaced. - /// Returns `Err((key, value))` if no entry existed for `key` (inputs are returned - /// so the caller can retry or discard them). The caller is responsible for passing - /// the returned state to [`Lifecycle::end_request`]. + /// Returns `Ok` if the entry was admitted and `Err(_)` if it wasn't. /// - /// Prefer [`replace`](Self::replace) unless you need manual control over lifecycle - /// request lifetime. + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// The same `&mut lcs` can be threaded through multiple operations to batch the + /// eviction work. Evicted items are released when `lcs` is dropped. pub fn replace_with_lifecycle( &self, key: Key, value: Val, soft: bool, - ) -> Result { - let mut lcs = self.lifecycle.begin_request(); + lcs: &mut L::RequestState, + ) -> Result<(), (Key, Val)> { let (shard, hash) = self.shard_for(&key).unwrap(); shard .write() - .insert(&mut lcs, hash, key, value, InsertStrategy::Replace { soft })?; - Ok(lcs) + .insert(lcs, hash, key, value, InsertStrategy::Replace { soft })?; + Ok(()) } /// Retains only the items specified by the predicate. @@ -462,34 +476,26 @@ impl< /// Inserts an item in the cache with key `key`. pub fn insert(&self, key: Key, value: Val) { - let lcs = self.insert_with_lifecycle(key, value); - self.lifecycle.end_request(lcs); + let mut lcs = Default::default(); + self.insert_with_lifecycle(key, value, &mut lcs); } - /// Inserts an item in the cache with key `key`, returning the lifecycle request state. - /// - /// Unlike [`insert`](Self::insert), the lifecycle request is **not** ended automatically. - /// The caller is responsible for passing the returned state to [`Lifecycle::end_request`] - /// when it is ready to drop any evicted items outside the shard lock. - /// - /// This is useful when coalescing multiple insertions into a single lifecycle request, - /// or when you need to control exactly when evicted items are dropped. - pub fn insert_with_lifecycle(&self, key: Key, value: Val) -> L::RequestState { - let mut lcs = self.lifecycle.begin_request(); - self.insert_with_state(key, value, &mut lcs); - lcs + /// Attempts to insert an item in the cache with key `key` without blocking. + /// Returns `Ok(())` if the item was inserted, or `Err((key, value))` if the shard lock + /// could not be acquired without blocking. Lock contention is the only failure + /// mode: the inputs are returned so the caller can retry or discard them. + pub fn try_insert(&self, key: Key, value: Val) -> Result<(), (Key, Val)> { + let mut lcs = Default::default(); + self.try_insert_with_lifecycle(key, value, &mut lcs) } - /// Inserts an item in the cache with key `key` using an existing lifecycle request state. + /// Inserts an item in the cache with key `key`, recording any evicted items into the + /// given lifecycle request state. /// - /// `lcs` must have been obtained from a prior [`Lifecycle::begin_request`] call and - /// must **not** have been passed to [`Lifecycle::end_request`] yet. Any items evicted - /// by this insert are recorded into `lcs` and will be dropped when `end_request` is - /// eventually called. - /// - /// Prefer [`insert`](Self::insert) for the common case where you do not need to - /// manage the lifecycle state manually. - pub fn insert_with_state(&self, key: Key, value: Val, lcs: &mut L::RequestState) { + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// The same `&mut lcs` can be threaded through multiple operations to batch the + /// eviction work. Evicted items are released when `lcs` is dropped. + pub fn insert_with_lifecycle(&self, key: Key, value: Val, lcs: &mut L::RequestState) { let (shard, hash) = self.shard_for(&key).unwrap(); let result = shard .write() @@ -498,42 +504,19 @@ impl< debug_assert!(result.is_ok()); } - /// Attempts to insert an item in the cache with key `key` without blocking. + /// Attempts to insert an item in the cache with key `key` without blocking, recording + /// any evicted items into the given lifecycle request state. /// Returns `Ok(())` if the item was inserted, or `Err((key, value))` if the shard lock - /// could not be acquired without blocking. Lock contention is the only failure - /// mode: the inputs are returned so the caller can retry or discard them. - pub fn try_insert(&self, key: Key, value: Val) -> Result<(), (Key, Val)> { - let lcs = self.try_insert_with_lifecycle(key, value)?; - self.lifecycle.end_request(lcs); - Ok(()) - } - - /// Attempts to insert an item in the cache with key `key` without blocking. - /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, - /// or `Err((key, value))` if the shard lock could not be acquired without blocking. - /// Lock contention is the only failure mode: the inputs are returned so the - /// caller can retry or discard them. + /// could not be acquired without blocking. Lock contention is the only failure mode: + /// the inputs are returned so the caller can retry or discard them. + /// + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// The same `&mut lcs` can be threaded through multiple operations to batch the + /// eviction work. Evicted items are released when `lcs` is dropped. pub fn try_insert_with_lifecycle( &self, key: Key, value: Val, - ) -> Result { - // Tradeoff: begin_request is called before acquiring the shard lock to avoid holding - // the lock during potentially expensive lifecycle initialization. - let mut lcs = self.lifecycle.begin_request(); - self.try_insert_with_state(key, value, &mut lcs)?; - Ok(lcs) - } - - /// Attempts to insert an item in the cache with key `key` without blocking. - /// Returns `Ok(lcs)` with the lifecycle request state if the item was inserted, - /// or `Err((key, value))` if the shard lock could not be acquired without blocking. - /// Lock contention is the only failure mode: the inputs are returned so the - /// caller can retry or discard them. - pub fn try_insert_with_state( - &self, - key: Key, - value: Val, lcs: &mut L::RequestState, ) -> Result<(), (Key, Val)> { let (shard, hash) = self.shard_for(&key).unwrap(); @@ -599,7 +582,9 @@ impl< let shard_weight_cap = new_weight_capacity.saturating_add(self.shards.len() as u64 - 1) / self.shards.len() as u64; for shard in &*self.shards { - shard.write().set_capacity(shard_weight_cap); + let mut lcs = Default::default(); + // `lcs` drops after this statement's lock guard, releasing evicted items outside the lock. + shard.write().set_capacity(shard_weight_cap, &mut lcs); } } @@ -626,21 +611,12 @@ impl< if let Some(v) = shard.read().get(hash, key) { return GuardResult::Value(v.clone()); } - PlaceholderGuard::join(&self.lifecycle, shard, hash, key, timeout) + PlaceholderGuard::join(shard, hash, key, timeout) } - /// Gets an item from the cache with key `key`, or inserts one produced by `with`. - /// - /// If the key is already present, the cached value is returned without calling `with`. - /// Otherwise, the cache is locked for this key (other callers to `get_value_or_guard` - /// or the `get_or_insert` family will block until the value is populated), `with` is - /// called to produce the value, and the result is inserted and returned. - /// - /// `with` may return an error, in which case nothing is inserted and the error is - /// propagated. The placeholder is dropped so waiting callers can retry. + /// Gets or inserts an item in the cache with key `key`. /// - /// See also [`get_value_or_guard`](Self::get_value_or_guard) for more control over - /// the placeholder lifecycle. + /// See also `get_value_or_guard` and `get_value_or_guard_async`. pub fn get_or_insert_with( &self, key: &Q, @@ -679,7 +655,7 @@ impl< if let Some(v) = shard.read().get(hash, key) { return Ok(v.clone()); } - match JoinFuture::new(&self.lifecycle, shard, hash, key).await { + match JoinFuture::new(shard, hash, key).await { JoinResult::Filled(Some(shared)) => { // SAFETY: Filled means the value was set by the loader. return Ok(unsafe { shared.value().unwrap_unchecked().clone() }); @@ -691,12 +667,7 @@ impl< } } - /// Gets an item from the cache with key `key`, or inserts one produced by `with`. - /// - /// Async counterpart of [`get_or_insert_with`](Self::get_or_insert_with). The `with` - /// future is only polled when no value exists for `key`. While `with` is being awaited, - /// other tasks looking up the same key will suspend and resume once the value is - /// inserted (or the guard is dropped without inserting). + /// Gets or inserts an item in the cache with key `key`. pub async fn get_or_insert_async( &self, key: &Q, @@ -790,21 +761,16 @@ impl< EntryOrPlaceholder::Replaced(shared, old_val) => { drop(shard_guard); return EntryResult::Replaced( - PlaceholderGuard::start_loading(&self.lifecycle, shard, shared), + PlaceholderGuard::start_loading(shard, shared), old_val, ); } EntryOrPlaceholder::NewPlaceholder(shared) => { drop(shard_guard); - return EntryResult::Vacant(PlaceholderGuard::start_loading( - &self.lifecycle, - shard, - shared, - )); + return EntryResult::Vacant(PlaceholderGuard::start_loading(shard, shared)); } EntryOrPlaceholder::ExistingPlaceholder(shared) => { match PlaceholderGuard::wait_for_placeholder( - &self.lifecycle, shard, shard_guard, shared, @@ -849,16 +815,14 @@ impl< EntryOrPlaceholder::Replaced(shared, old_val) => { drop(shard_guard); Ok(EntryResult::Replaced( - PlaceholderGuard::start_loading(&self.lifecycle, shard, shared), + PlaceholderGuard::start_loading(shard, shared), old_val, )) } EntryOrPlaceholder::NewPlaceholder(shared) => { drop(shard_guard); Ok(EntryResult::Vacant(PlaceholderGuard::start_loading( - &self.lifecycle, - shard, - shared, + shard, shared, ))) } EntryOrPlaceholder::ExistingPlaceholder(_) => Err(()), @@ -866,7 +830,7 @@ impl< }; match result { Ok(entry_result) => return entry_result, - Err(()) => match JoinFuture::new(&self.lifecycle, shard, hash, key).await { + Err(()) => match JoinFuture::new(shard, hash, key).await { JoinResult::Filled(_) => continue, JoinResult::Guard(g) => return EntryResult::Vacant(g), JoinResult::Timeout => unsafe { unreachable_unchecked() }, @@ -1004,11 +968,6 @@ impl Lifecycle for DefaultLifecycle { // overhead (e.g. a vector) for this default lifecycle. type RequestState = [Option<(Key, Val)>; 2]; - #[inline] - fn begin_request(&self) -> Self::RequestState { - [None, None] - } - #[inline] fn on_evict(&self, state: &mut Self::RequestState, key: Key, val: Val) { if std::mem::needs_drop::<(Key, Val)>() { @@ -1024,6 +983,7 @@ impl Lifecycle for DefaultLifecycle { #[cfg(test)] mod tests { use super::*; + use crate::shard::SharedPlaceholder as _; use std::{ sync::{Arc, Barrier}, thread, @@ -1767,6 +1727,29 @@ mod tests { assert!(cache.try_peek(&1).is_err()); } + #[cfg(feature = "stats")] + #[test] + fn test_item_stats() { + let cache = Cache::new(100); + // Missing key has no stats. + assert!(cache.item_stats(&1).is_none()); + + cache.insert(1, 10); + // Insert alone is not a hit. + assert_eq!(cache.item_stats(&1).map(|s| s.access_count), Some(0)); + + // Each get increments the per-item access count. + cache.get(&1); + cache.get(&1); + cache.get(&1); + assert_eq!(cache.item_stats(&1).map(|s| s.access_count), Some(3)); + + // Peeking (including item_stats itself) does not alter the count. + cache.peek(&1); + let _ = cache.item_stats(&1); + assert_eq!(cache.item_stats(&1).map(|s| s.access_count), Some(3)); + } + #[test] fn test_try_remove() { let cache = Cache::new(100); @@ -1815,18 +1798,74 @@ mod tests { #[test] fn test_try_insert_with_lifecycle() { let cache = Cache::new(100); + let mut lcs = Default::default(); - // Successful insert returns the lifecycle request state. - let result = cache.try_insert_with_lifecycle(1, 10); - assert!(result.is_ok()); - let lcs = result.ok().unwrap(); - cache.lifecycle.end_request(lcs); + // Successful insert records evictions into the provided request state. + assert_eq!(cache.try_insert_with_lifecycle(1, 10, &mut lcs), Ok(())); assert_eq!(cache.get(&1), Some(10)); + // The same request state can be threaded through several operations. + assert_eq!(cache.try_insert_with_lifecycle(2, 20, &mut lcs), Ok(())); + assert_eq!(cache.get(&2), Some(20)); + // Contended when a read lock is held. let guards: Vec<_> = cache.shards.iter().map(|s| s.read()).collect(); - assert_eq!(cache.try_insert_with_lifecycle(2, 20), Err((2, 20))); + assert_eq!( + cache.try_insert_with_lifecycle(3, 30, &mut lcs), + Err((3, 30)) + ); drop(guards); - assert_eq!(cache.get(&2), None); + assert_eq!(cache.get(&3), None); + } + + #[test] + fn test_guard_leak() { + let cache: Cache = Cache::new(8); + let guard1 = match cache.get_value_or_guard(&1, None) { + GuardResult::Guard(g) => g, + _ => panic!("expected guard"), + }; + let idx1 = guard1.shared().idx(); + drop(guard1); + let guard2 = match cache.get_value_or_guard(&1, None) { + GuardResult::Guard(g) => g, + _ => panic!("expected guard"), + }; + let idx2 = guard2.shared().idx(); + drop(guard2); + assert_eq!(idx1, idx2); + } + + // A real insert overwrites the placeholder in place, reusing its slab slot as + // a Resident. Dropping the now-stale guard must not free that slot, otherwise + // the live entry is evicted while the map still references it. + #[test] + fn test_guard_drop_after_overwrite_insert() { + let cache: Cache = Cache::new(8); + let guard = match cache.get_value_or_guard(&1, None) { + GuardResult::Guard(g) => g, + _ => panic!("expected guard"), + }; + cache.insert(1, 100); + assert_eq!(cache.get(&1), Some(100)); + drop(guard); + assert_eq!(cache.get(&1), Some(100)); + } + + // A remove frees the placeholder's slab slot, which a later insert reuses for a + // different key. Dropping the original guard must not free that slot again, or + // it evicts the unrelated key. + #[test] + fn test_guard_drop_after_remove_and_reuse() { + let cache: Cache = Cache::new(8); + let guard = match cache.get_value_or_guard(&1, None) { + GuardResult::Guard(g) => g, + _ => panic!("expected guard"), + }; + cache.remove(&1); + cache.insert(2, 222); + assert_eq!(cache.get(&2), Some(222)); + drop(guard); + assert_eq!(cache.get(&2), Some(222)); } } diff --git a/src/sync_placeholder.rs b/src/sync_placeholder.rs index 8f09d11..23cb867 100644 --- a/src/sync_placeholder.rs +++ b/src/sync_placeholder.rs @@ -87,12 +87,18 @@ enum LoadingState { } pub struct PlaceholderGuard<'a, Key, Val, We, B, L> { - lifecycle: &'a L, shard: &'a RwLock>>, shared: SharedPlaceholder, inserted: bool, } +#[cfg(test)] +impl<'a, Key, Val, We, B, L> PlaceholderGuard<'a, Key, Val, We, B, L> { + pub fn shared(&self) -> &SharedPlaceholder { + &self.shared + } +} + #[derive(Debug)] enum Waiter { Thread { @@ -189,7 +195,6 @@ pub enum EntryResult<'a, Key, Val, We, B, L, T> { impl<'a, Key, Val, We, B, L> PlaceholderGuard<'a, Key, Val, We, B, L> { #[inline] pub fn start_loading( - lifecycle: &'a L, shard: &'a RwLock>>, shared: SharedPlaceholder, ) -> Self { @@ -198,7 +203,6 @@ impl<'a, Key, Val, We, B, L> PlaceholderGuard<'a, Key, Val, We, B, L> { LoadingState::Loading )); PlaceholderGuard { - lifecycle, shard, shared, inserted: false, @@ -209,7 +213,6 @@ impl<'a, Key, Val, We, B, L> PlaceholderGuard<'a, Key, Val, We, B, L> { // or a guard if the caller got the guard. #[inline] fn handle_notification( - lifecycle: &'a L, shard: &'a RwLock>>, shared: SharedPlaceholder, ) -> Result, PlaceholderGuard<'a, Key, Val, We, B, L>> { @@ -218,7 +221,7 @@ impl<'a, Key, Val, We, B, L> PlaceholderGuard<'a, Key, Val, We, B, L> { if shared.value().is_some() { Ok(shared) } else { - Err(PlaceholderGuard::start_loading(lifecycle, shard, shared)) + Err(PlaceholderGuard::start_loading(shard, shared)) } } @@ -258,7 +261,6 @@ impl< > PlaceholderGuard<'a, Key, Val, We, B, L> { pub fn join( - lifecycle: &'a L, shard: &'a RwLock>>, hash: u64, key: &Q, @@ -271,12 +273,12 @@ impl< let shared = match shard_guard.get_or_placeholder(hash, key) { Ok((_, v)) => return GuardResult::Value(v.clone()), Err((shared, true)) => { - return GuardResult::Guard(Self::start_loading(lifecycle, shard, shared)); + return GuardResult::Guard(Self::start_loading(shard, shared)); } Err((shared, false)) => shared, }; let mut deadline = timeout.map(Ok); - match Self::wait_for_placeholder(lifecycle, shard, shard_guard, shared, deadline.as_mut()) { + match Self::wait_for_placeholder(shard, shard_guard, shared, deadline.as_mut()) { JoinResult::Filled(shared) => unsafe { // SAFETY: Filled means the value was set by the loader. GuardResult::Value(shared.unwrap_unchecked().value().unwrap_unchecked().clone()) @@ -295,7 +297,6 @@ impl< /// call. On first use the duration is converted in-place to `Err(instant)` so that /// callers that retry (e.g. `entry`) preserve the original deadline across calls. pub(crate) fn wait_for_placeholder( - lifecycle: &'a L, shard: &'a RwLock>>, shard_guard: RwLockWriteGuard<'a, CacheShard>>, shared: SharedPlaceholder, @@ -338,7 +339,7 @@ impl< if let Some(instant) = deadline { let remaining = instant.saturating_duration_since(Instant::now()); if remaining.is_zero() { - return Self::join_timeout(lifecycle, shard, shared, parked_thread, ¬ified); + return Self::join_timeout(shard, shared, parked_thread, ¬ified); } #[cfg(not(fuzzing))] thread::park_timeout(remaining); @@ -347,7 +348,7 @@ impl< thread::park(); } if notified.load(Ordering::Acquire) { - return match Self::handle_notification(lifecycle, shard, shared) { + return match Self::handle_notification(shard, shared) { Ok(shared) => JoinResult::Filled(Some(shared)), Err(g) => JoinResult::Guard(g), }; @@ -357,7 +358,6 @@ impl< #[cold] fn join_timeout( - lifecycle: &'a L, shard: &'a RwLock>>>, shared: Arc>, // when timeout is zero, the thread may have not been added to the waiters list @@ -368,7 +368,7 @@ impl< match state.loading { LoadingState::Loading if notified.load(Ordering::Acquire) => { drop(state); // Drop state guard to avoid a deadlock with start_loading - JoinResult::Guard(PlaceholderGuard::start_loading(lifecycle, shard, shared)) + JoinResult::Guard(PlaceholderGuard::start_loading(shard, shared)) } LoadingState::Loading => { if parked_thread.is_some() { @@ -407,18 +407,24 @@ impl< /// A placeholder can be removed as a result of a `remove` call /// or a non-placeholder `insert` with the same key. pub fn insert(self, value: Val) -> Result<(), Val> { - let lifecycle = self.lifecycle; - let lcs = self.insert_with_lifecycle(value)?; - lifecycle.end_request(lcs); - Ok(()) + let mut lcs = Default::default(); + self.insert_with_lifecycle(value, &mut lcs) } - /// Inserts the value into the placeholder + /// Inserts the value into the placeholder, recording any evicted items into the given + /// lifecycle request state. /// /// Returns Err if the placeholder isn't in the cache anymore. /// A placeholder can be removed as a result of a `remove` call /// or a non-placeholder `insert` with the same key. - pub fn insert_with_lifecycle(mut self, value: Val) -> Result { + /// + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// Evicted items are released when `lcs` is dropped. + pub fn insert_with_lifecycle( + mut self, + value: Val, + lcs: &mut L::RequestState, + ) -> Result<(), Val> { unsafe { self.shared.value.set(value.clone()).unwrap_unchecked() }; let referenced; { @@ -439,11 +445,10 @@ impl< // - the placeholder will be removed here, if it still exists self.inserted = true; - let mut lcs = self.lifecycle.begin_request(); self.shard .write() - .replace_placeholder(&mut lcs, &self.shared, referenced, value)?; - Ok(lcs) + .replace_placeholder(lcs, &self.shared, referenced, value)?; + Ok(()) } } @@ -503,7 +508,6 @@ impl std::fmt::Debug for PlaceholderGuard<'_, Key, Val, We, /// be moved after the first poll, keeping that pointer valid. The pointer is /// cleaned up in `drop_pending_waiter` before the struct is destroyed. pub(crate) struct JoinFuture<'a, 'b, Q: ?Sized, Key, Val, We, B, L> { - lifecycle: &'a L, shard: &'a RwLock>>, hash: u64, key: &'b Q, @@ -523,13 +527,11 @@ enum JoinFutureState { impl<'a, 'b, Q: ?Sized, Key, Val, We, B, L> JoinFuture<'a, 'b, Q, Key, Val, We, B, L> { pub(crate) fn new( - lifecycle: &'a L, shard: &'a RwLock>>, hash: u64, key: &'b Q, ) -> Self { Self { - lifecycle, shard, hash, key, @@ -554,7 +556,7 @@ impl JoinFuture<'_, '_, Q, Key, Val, We, B, L> { // The write guard was abandoned elsewhere, this future was notified but didn't get polled. // So we get and drop the guard here to handle the side effects. drop(state); // Drop state guard to avoid a deadlock with start_loading - let _ = PlaceholderGuard::start_loading(self.lifecycle, self.shard, shared); + let _ = PlaceholderGuard::start_loading(self.shard, shared); } LoadingState::Loading => { // Remove ourselves from the waiters list @@ -600,7 +602,6 @@ impl< // fields. The `notified` field's address (registered in the waiter list) stays // stable because Pin guarantees the future won't be moved. let this = unsafe { self.get_unchecked_mut() }; - let lifecycle = this.lifecycle; let shard = this.shard; match &mut this.state { JoinFutureState::Created => { @@ -614,7 +615,7 @@ impl< this.state = JoinFutureState::Done; drop(shard_guard); Poll::Ready(JoinResult::Guard(PlaceholderGuard::start_loading( - lifecycle, shard, shared, + shard, shared, ))) } Err((shared, false)) => { @@ -673,12 +674,10 @@ impl< else { unsafe { unreachable_unchecked() } }; - Poll::Ready( - match PlaceholderGuard::handle_notification(lifecycle, shard, shared) { - Ok(shared) => JoinResult::Filled(Some(shared)), - Err(g) => JoinResult::Guard(g), - }, - ) + Poll::Ready(match PlaceholderGuard::handle_notification(shard, shared) { + Ok(shared) => JoinResult::Filled(Some(shared)), + Err(g) => JoinResult::Guard(g), + }) } JoinFutureState::Done => panic!("Polled after ready"), } diff --git a/src/unsync.rs b/src/unsync.rs index 66ee954..7e9a5c7 100644 --- a/src/unsync.rs +++ b/src/unsync.rs @@ -175,6 +175,16 @@ impl, B: BuildHasher, L: Lifecycle(&self, key: &Q) -> Option + where + Q: Hash + Equivalent + ?Sized, + { + self.shard.item_stats(self.shard.hash(key), key) + } + /// Peeks an item from the cache. Contrary to gets, peeks don't alter the key "hotness". /// /// Note: Leaking the returned RefMut might cause cache weight tracking to be inaccurate. @@ -212,31 +222,35 @@ impl, B: BuildHasher, L: Lifecycle Result<(), (Key, Val)> { - let lcs = self.replace_with_lifecycle(key, value, soft)?; - self.shard.lifecycle.end_request(lcs); - Ok(()) + let mut lcs = Default::default(); + self.replace_with_lifecycle(key, value, soft, &mut lcs) } - /// Replaces an item in the cache, but only if it already exists. + /// Replaces an item in the cache, but only if it already exists, recording any evicted + /// items into the given lifecycle request state. /// If `soft` is set, the replace operation won't affect the "hotness" of the key, /// even if the value is replaced. /// /// Returns `Ok` if the entry was admitted and `Err(_)` if it wasn't. + /// + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// The same `&mut lcs` can be threaded through multiple operations to batch the + /// eviction work. Evicted items are released when `lcs` is dropped. pub fn replace_with_lifecycle( &mut self, key: Key, value: Val, soft: bool, - ) -> Result { - let mut lcs = self.shard.lifecycle.begin_request(); + lcs: &mut L::RequestState, + ) -> Result<(), (Key, Val)> { self.shard.insert( - &mut lcs, + lcs, self.shard.hash(&key), key, value, InsertStrategy::Replace { soft }, )?; - Ok(lcs) + Ok(()) } /// Retains only the items specified by the predicate. @@ -265,9 +279,8 @@ impl, B: BuildHasher, L: Lifecycle idx, Err((plh, _)) => { let v = with()?; - let mut lcs = self.shard.lifecycle.begin_request(); + let mut lcs = Default::default(); let replaced = self.shard.replace_placeholder(&mut lcs, &plh, false, v); - self.shard.lifecycle.end_request(lcs); debug_assert!(replaced.is_ok(), "unsync replace_placeholder can't fail"); plh.idx } @@ -291,10 +304,9 @@ impl, B: BuildHasher, L: Lifecycle idx, Err((plh, _)) => { let v = with()?; - let mut lcs = self.shard.lifecycle.begin_request(); + let mut lcs = Default::default(); let replaced = self.shard.replace_placeholder(&mut lcs, &plh, false, v); debug_assert!(replaced.is_ok(), "unsync replace_placeholder can't fail"); - self.shard.lifecycle.end_request(lcs); plh.idx } }; @@ -349,15 +361,19 @@ impl, B: BuildHasher, L: Lifecycle L::RequestState { - let mut lcs = self.shard.lifecycle.begin_request(); + /// Inserts an item in the cache with key `key`, recording any evicted items into the + /// given lifecycle request state. + /// + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// The same `&mut lcs` can be threaded through multiple operations to batch the + /// eviction work. Evicted items are released when `lcs` is dropped. + pub fn insert_with_lifecycle(&mut self, key: Key, value: Val, lcs: &mut L::RequestState) { let result = self.shard.insert( - &mut lcs, + lcs, self.shard.hash(&key), key, value, @@ -365,7 +381,6 @@ impl, B: BuildHasher, L: Lifecycle, B: BuildHasher, L: Lifecycle Clone for DefaultLifecycle { impl Lifecycle for DefaultLifecycle { type RequestState = (); - - #[inline] - fn begin_request(&self) -> Self::RequestState {} - - #[inline] - fn on_evict(&self, _state: &mut Self::RequestState, _key: Key, _val: Val) {} } #[derive(Debug, Clone)] @@ -465,29 +475,22 @@ impl, B: BuildHasher, L: Lifecycle L::RequestState { - self.insert_internal(value, true).unwrap() - } - - #[inline] - fn insert_internal(mut self, value: Val, return_lcs: bool) -> Option { - let mut lcs = self.cache.shard.lifecycle.begin_request(); - let replaced = - self.cache - .shard - .replace_placeholder(&mut lcs, &self.placeholder, false, value); + /// Inserts the value into the placeholder, recording any evicted items into the given + /// lifecycle request state. + /// + /// `lcs` is a [`Lifecycle::RequestState`]; construct one with `Default::default()`. + /// Evicted items are released when `lcs` is dropped. + pub fn insert_with_lifecycle(mut self, value: Val, lcs: &mut L::RequestState) { + let replaced = self + .cache + .shard + .replace_placeholder(lcs, &self.placeholder, false, value); debug_assert!(replaced.is_ok(), "unsync replace_placeholder can't fail"); self.inserted = true; - if return_lcs { - Some(lcs) - } else { - self.cache.shard.lifecycle.end_request(lcs); - None - } } }