From 5cd887e53c895950e2d34d652a2cbf829f959956 Mon Sep 17 00:00:00 2001 From: ArtemNikit1n Date: Sun, 23 Nov 2025 13:27:40 +0300 Subject: [PATCH 1/5] feat: Add ClusteringMeasurements --- src/lib.rs | 2 +- src/system/mod.rs | 2 +- src/system/scrub.rs | 152 ++++++++++++++++++++++++++++++++++++++---- src/system/storage.rs | 2 +- 4 files changed, 143 insertions(+), 15 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8d702bf6..f0a7d8f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; pub use system::database::{Database, IterableDatabase}; -pub use system::scrub::{CopyScrubber, Scrub, ScrubMeasurements}; +pub use system::scrub::{DumbScrubber, CopyScrubber, Scrub, ScrubMeasurements}; pub use system::storage::{Data, DataContainer}; pub use system::{create_cdc_filesystem, FileSystem}; diff --git a/src/system/mod.rs b/src/system/mod.rs index c37f09d3..6dc03ee7 100644 --- a/src/system/mod.rs +++ b/src/system/mod.rs @@ -242,7 +242,7 @@ where /// otherwise it returns [`ErrorKind::InvalidInput`][io::ErrorKind::InvalidInput]. /// /// For more info check [`Scrub`][Scrub] trait and its [`scrub`][Scrub::scrub] method. - pub fn scrub(&mut self) -> io::Result { + pub fn scrub(&mut self) -> io::Result> { self.storage.scrub() } diff --git a/src/system/scrub.rs b/src/system/scrub.rs index 3957022b..eac9c79e 100644 --- a/src/system/scrub.rs +++ b/src/system/scrub.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::io; use std::time::{Duration, Instant}; @@ -57,7 +58,7 @@ where /// We should be able to iterate over the `database` to process all chunks we had stored before. /// The [IntoIterator] trait should be implemented for `database`, but it should not be a big concern, because the only structure that should be implemented /// for the algorithm is the scrubber itself. `database` should be considered a given entity, along with the `target_map`. - fn scrub<'a>(&mut self, database: &mut B, target_map: &mut T) -> io::Result + fn scrub<'a>(&mut self, database: &mut B, target_map: &mut T) -> io::Result> where Hash: 'a, Key: 'a; @@ -67,48 +68,102 @@ where /// /// Contains information about the amount of data processed by the scrubber (in bytes), /// time spent on scrubbing, -/// and the amount of data left untouched. -#[derive(Debug, Default, PartialEq, Eq, Copy, Clone)] -pub struct ScrubMeasurements { +/// the amount of data left untouched and clustering. +#[derive(Debug, Default, PartialEq, Eq, Clone)] +pub struct ScrubMeasurements { /// How much data was processed by the scrubber (in bytes). pub processed_data: usize, /// Time spent on scrubbing. pub running_time: Duration, /// The amount of data left untouched (in bytes). pub data_left: usize, + /// All information about clusterization: + /// 1. Total cluster size (number of vertices). + /// 2. Number of clusters (total number of parent vertices). + /// 3. The number of vertices within a single cluster. + /// 4. Distance to the parent vertex. + /// 5. Distance between clusters (between parent vertices). + /// 6. Deduplication coefficient for each cluster. + pub clusterization_report: ClusteringMeasurements, +} + +#[derive(Debug, Default, PartialEq, Eq, Clone)] +pub struct ClusteringMeasurements { + /// Number of vertices (chunks). + total_cluster_size: usize, + /// Total number of parent vertices. + number_of_clusters: usize, + /// The number of vertices within a single cluster. + /// It contains the hash values of the parent vertices as keys. + number_of_vertices_in_cluster: HashMap, + /// Distance to the parent vertex. + /// It contains the hash values of the parent vertices as keys. + distance_to_vertices_in_cluster: HashMap>, + /// Distance between clusters (between parent vertices). + /// The key is the parent in the cluster. The distance is calculated to the other parents. + distance_to_other_clusters: HashMap>, } pub struct CopyScrubber; pub struct DumbScrubber; -impl Scrub for CopyScrubber -where - Hash: ChunkHash, - B: IterableDatabase>, - T: Database>, -{ - fn scrub<'a>(&mut self, database: &mut B, target: &mut T) -> io::Result +impl CopyScrubber { + pub fn scrub<'a, Hash, B, T>(&mut self, database: &mut B, target: &mut T) -> io::Result> where Hash: 'a, + Hash: ChunkHash + Metric, + B: IterableDatabase>, + T: Database>, { + let mut total_cluster_size = 0; + let mut number_of_vertices_in_cluster = HashMap::new(); + let mut distance_to_other_clusters = HashMap::new(); + let mut parent_vertices: Vec = Vec::new(); let now = Instant::now(); let mut processed_data = 0; for (hash, container) in database.iterator_mut() { match container.extract() { Data::Chunk(chunk) => { target.insert(hash.clone(), chunk.clone())?; + total_cluster_size += 1; processed_data += chunk.len(); + number_of_vertices_in_cluster.insert(hash.clone(), 1); + parent_vertices.push(hash.clone()); } Data::TargetChunk(_) => (), } container.make_target(vec![hash.clone()]); } + + for i in 0..parent_vertices.len() { + let mut distances = Vec::new(); + + for j in 0..parent_vertices.len() { + if i != j { + let distance = parent_vertices[i].distance(&parent_vertices[j]); + distances.push(distance); + } + } + + distance_to_other_clusters.insert(parent_vertices[i].clone(), distances); + } + let running_time = now.elapsed(); + let number_of_clusters = total_cluster_size; + let distance_to_vertices_in_cluster = HashMap::new(); + let clusterization_report = ClusteringMeasurements { + total_cluster_size, + number_of_clusters, + number_of_vertices_in_cluster, + distance_to_vertices_in_cluster, + distance_to_other_clusters, + }; Ok(ScrubMeasurements { processed_data, running_time, data_left: 0, + clusterization_report, }) } } @@ -119,7 +174,7 @@ where B: IterableDatabase>, T: Database>, { - fn scrub<'a>(&mut self, _database: &mut B, _target: &mut T) -> io::Result + fn scrub<'a>(&mut self, _database: &mut B, _target: &mut T) -> io::Result> where Hash: 'a, Key: 'a, @@ -127,3 +182,76 @@ where Ok(ScrubMeasurements::default()) } } + +pub trait Metric { + /// Calculates the distance between `self` and `other'. + fn distance(&self, other: &T) -> usize; +} + +impl Metric for Vec { + fn distance(&self, other: &Self) -> usize { + self.len().abs_diff(other.len()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::system::HashMap; + use crate::DataContainer; + + fn create_test_data() -> Vec<(Vec, Vec)> { + vec![ + (b"chunk1".to_vec(), b"content1".to_vec()), + (b"chunk2".to_vec(), b"content2".to_vec()), + (b"chunk3".to_vec(), b"content3".to_vec()), + (b"duplicate_chunk".to_vec(), b"same_content".to_vec()), + (b"another_duplicate".to_vec(), b"same_content".to_vec()), + ] + } + + #[test] + fn scrub_should_return_correct_scrub_measurements_for_copy_scrubber() { + let test_data = create_test_data(); + let mut total_data_size = 0; + + let mut database: HashMap, DataContainer>> = HashMap::new(); + let test_data_len = test_data.len(); + for (hash, chunk) in test_data { + total_data_size += chunk.len(); + database.insert( + hash.clone(), + DataContainer::from(chunk), + ); + } + + let mut target_map: HashMap, Vec> = HashMap::new(); + let mut scrubber = CopyScrubber; + let scrub_report = scrubber.scrub(&mut database, &mut target_map).unwrap(); + + assert_eq!(scrub_report.processed_data, total_data_size); + assert!(scrub_report.running_time > Duration::from_secs(0)); + assert_eq!(scrub_report.data_left, 0); + + let cluster_report = &scrub_report.clusterization_report; + assert_eq!(cluster_report.total_cluster_size, test_data_len); + assert_eq!(cluster_report.number_of_clusters, test_data_len); + assert!(cluster_report.number_of_vertices_in_cluster.values().all(|&v| v == 1)); + assert!(cluster_report.distance_to_vertices_in_cluster.is_empty()); + assert!(cluster_report.distance_to_other_clusters.values().all(|v| v.len() == test_data_len - 1)); + } + + #[test] + fn scrub_should_handle_empty_database() { + let mut scrubber = CopyScrubber; + let mut database: HashMap, DataContainer>> = HashMap::new(); + let mut target_map: HashMap, Vec> = HashMap::new(); + + let scrub_report = scrubber.scrub(&mut database, &mut target_map).unwrap(); + + assert_eq!(scrub_report.processed_data, 0); + assert_eq!(scrub_report.data_left, 0); + assert_eq!(scrub_report.clusterization_report.total_cluster_size, 0); + assert!(target_map.is_empty()); + } +} \ No newline at end of file diff --git a/src/system/storage.rs b/src/system/storage.rs index 1010557a..0679decc 100644 --- a/src/system/storage.rs +++ b/src/system/storage.rs @@ -177,7 +177,7 @@ where } } - pub fn scrub(&mut self) -> io::Result { + pub fn scrub(&mut self) -> io::Result> { self.scrubber .as_mut() .ok_or_else(|| { From 1181bdc2eaca19334a7c1a963a46712058612ed6 Mon Sep 17 00:00:00 2001 From: ArtemNikit1n Date: Tue, 25 Nov 2025 22:07:50 +0300 Subject: [PATCH 2/5] fix: Remove type Hash in ScrubMeasurments --- src/system/mod.rs | 2 +- src/system/scrub.rs | 51 ++++++++++++++++++------------------------- src/system/storage.rs | 2 +- 3 files changed, 23 insertions(+), 32 deletions(-) diff --git a/src/system/mod.rs b/src/system/mod.rs index 6dc03ee7..c37f09d3 100644 --- a/src/system/mod.rs +++ b/src/system/mod.rs @@ -242,7 +242,7 @@ where /// otherwise it returns [`ErrorKind::InvalidInput`][io::ErrorKind::InvalidInput]. /// /// For more info check [`Scrub`][Scrub] trait and its [`scrub`][Scrub::scrub] method. - pub fn scrub(&mut self) -> io::Result> { + pub fn scrub(&mut self) -> io::Result { self.storage.scrub() } diff --git a/src/system/scrub.rs b/src/system/scrub.rs index eac9c79e..a084b5ad 100644 --- a/src/system/scrub.rs +++ b/src/system/scrub.rs @@ -58,7 +58,7 @@ where /// We should be able to iterate over the `database` to process all chunks we had stored before. /// The [IntoIterator] trait should be implemented for `database`, but it should not be a big concern, because the only structure that should be implemented /// for the algorithm is the scrubber itself. `database` should be considered a given entity, along with the `target_map`. - fn scrub<'a>(&mut self, database: &mut B, target_map: &mut T) -> io::Result> + fn scrub<'a>(&mut self, database: &mut B, target_map: &mut T) -> io::Result where Hash: 'a, Key: 'a; @@ -70,7 +70,7 @@ where /// time spent on scrubbing, /// the amount of data left untouched and clustering. #[derive(Debug, Default, PartialEq, Eq, Clone)] -pub struct ScrubMeasurements { +pub struct ScrubMeasurements { /// How much data was processed by the scrubber (in bytes). pub processed_data: usize, /// Time spent on scrubbing. @@ -84,42 +84,44 @@ pub struct ScrubMeasurements { /// 4. Distance to the parent vertex. /// 5. Distance between clusters (between parent vertices). /// 6. Deduplication coefficient for each cluster. - pub clusterization_report: ClusteringMeasurements, + pub clusterization_report: ClusteringMeasurements, } #[derive(Debug, Default, PartialEq, Eq, Clone)] -pub struct ClusteringMeasurements { +pub struct ClusteringMeasurements { /// Number of vertices (chunks). total_cluster_size: usize, /// Total number of parent vertices. number_of_clusters: usize, /// The number of vertices within a single cluster. /// It contains the hash values of the parent vertices as keys. - number_of_vertices_in_cluster: HashMap, + number_of_vertices_in_cluster: HashMap, /// Distance to the parent vertex. /// It contains the hash values of the parent vertices as keys. - distance_to_vertices_in_cluster: HashMap>, + distance_to_vertices_in_cluster: HashMap>, /// Distance between clusters (between parent vertices). /// The key is the parent in the cluster. The distance is calculated to the other parents. - distance_to_other_clusters: HashMap>, + distance_to_other_clusters: HashMap>, } pub struct CopyScrubber; pub struct DumbScrubber; -impl CopyScrubber { - pub fn scrub<'a, Hash, B, T>(&mut self, database: &mut B, target: &mut T) -> io::Result> +impl Scrub for CopyScrubber +where + Hash: ChunkHash, + B: IterableDatabase>, + T: Database>, +{ + fn scrub<'a>(&mut self, database: &mut B, target: &mut T) -> io::Result where Hash: 'a, - Hash: ChunkHash + Metric, - B: IterableDatabase>, - T: Database>, { - let mut total_cluster_size = 0; + let mut total_cluster_size: usize = 0; let mut number_of_vertices_in_cluster = HashMap::new(); let mut distance_to_other_clusters = HashMap::new(); - let mut parent_vertices: Vec = Vec::new(); + let mut parent_vertices: Vec = Vec::new(); let now = Instant::now(); let mut processed_data = 0; for (hash, container) in database.iterator_mut() { @@ -128,8 +130,8 @@ impl CopyScrubber { target.insert(hash.clone(), chunk.clone())?; total_cluster_size += 1; processed_data += chunk.len(); - number_of_vertices_in_cluster.insert(hash.clone(), 1); - parent_vertices.push(hash.clone()); + number_of_vertices_in_cluster.insert(total_cluster_size as u32, 1); + parent_vertices.push(total_cluster_size); } Data::TargetChunk(_) => (), } @@ -141,12 +143,12 @@ impl CopyScrubber { for j in 0..parent_vertices.len() { if i != j { - let distance = parent_vertices[i].distance(&parent_vertices[j]); + let distance = parent_vertices[i].abs_diff(parent_vertices[j]); distances.push(distance); } } - distance_to_other_clusters.insert(parent_vertices[i].clone(), distances); + distance_to_other_clusters.insert(parent_vertices[i] as u32, distances); } let running_time = now.elapsed(); @@ -174,7 +176,7 @@ where B: IterableDatabase>, T: Database>, { - fn scrub<'a>(&mut self, _database: &mut B, _target: &mut T) -> io::Result> + fn scrub<'a>(&mut self, _database: &mut B, _target: &mut T) -> io::Result where Hash: 'a, Key: 'a, @@ -183,17 +185,6 @@ where } } -pub trait Metric { - /// Calculates the distance between `self` and `other'. - fn distance(&self, other: &T) -> usize; -} - -impl Metric for Vec { - fn distance(&self, other: &Self) -> usize { - self.len().abs_diff(other.len()) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/system/storage.rs b/src/system/storage.rs index 0679decc..1010557a 100644 --- a/src/system/storage.rs +++ b/src/system/storage.rs @@ -177,7 +177,7 @@ where } } - pub fn scrub(&mut self) -> io::Result> { + pub fn scrub(&mut self) -> io::Result { self.scrubber .as_mut() .ok_or_else(|| { From 0af47424e3d801bff664fd09a0f849ec0868388c Mon Sep 17 00:00:00 2001 From: ArtemNikit1n Date: Wed, 26 Nov 2025 22:09:15 +0300 Subject: [PATCH 3/5] fix: Add pub access to the ScrubMeasurements fields. Fix doc and style --- src/system/scrub.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/system/scrub.rs b/src/system/scrub.rs index a084b5ad..d7efc03d 100644 --- a/src/system/scrub.rs +++ b/src/system/scrub.rs @@ -68,7 +68,7 @@ where /// /// Contains information about the amount of data processed by the scrubber (in bytes), /// time spent on scrubbing, -/// the amount of data left untouched and clustering. +/// the amount of data left untouched and clustering information. #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct ScrubMeasurements { /// How much data was processed by the scrubber (in bytes). @@ -90,18 +90,18 @@ pub struct ScrubMeasurements { #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct ClusteringMeasurements { /// Number of vertices (chunks). - total_cluster_size: usize, + pub total_cluster_size: usize, /// Total number of parent vertices. - number_of_clusters: usize, + pub number_of_clusters: usize, /// The number of vertices within a single cluster. /// It contains the hash values of the parent vertices as keys. - number_of_vertices_in_cluster: HashMap, + pub number_of_vertices_in_cluster: HashMap, /// Distance to the parent vertex. /// It contains the hash values of the parent vertices as keys. - distance_to_vertices_in_cluster: HashMap>, + pub distance_to_vertices_in_cluster: HashMap>, /// Distance between clusters (between parent vertices). /// The key is the parent in the cluster. The distance is calculated to the other parents. - distance_to_other_clusters: HashMap>, + pub distance_to_other_clusters: HashMap>, } pub struct CopyScrubber; @@ -124,6 +124,7 @@ where let mut parent_vertices: Vec = Vec::new(); let now = Instant::now(); let mut processed_data = 0; + for (hash, container) in database.iterator_mut() { match container.extract() { Data::Chunk(chunk) => { From d6bbd7d6f81d4919e950c27cfe290f2fc3df006f Mon Sep 17 00:00:00 2001 From: ArtemNikit1n Date: Wed, 26 Nov 2025 22:16:16 +0300 Subject: [PATCH 4/5] feat: Add dedup retio for each cluster --- src/system/scrub.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/system/scrub.rs b/src/system/scrub.rs index d7efc03d..d1b07d83 100644 --- a/src/system/scrub.rs +++ b/src/system/scrub.rs @@ -69,7 +69,7 @@ where /// Contains information about the amount of data processed by the scrubber (in bytes), /// time spent on scrubbing, /// the amount of data left untouched and clustering information. -#[derive(Debug, Default, PartialEq, Eq, Clone)] +#[derive(Debug, Default, PartialEq, Clone)] pub struct ScrubMeasurements { /// How much data was processed by the scrubber (in bytes). pub processed_data: usize, @@ -87,7 +87,7 @@ pub struct ScrubMeasurements { pub clusterization_report: ClusteringMeasurements, } -#[derive(Debug, Default, PartialEq, Eq, Clone)] +#[derive(Debug, Default, PartialEq, Clone)] pub struct ClusteringMeasurements { /// Number of vertices (chunks). pub total_cluster_size: usize, @@ -102,12 +102,13 @@ pub struct ClusteringMeasurements { /// Distance between clusters (between parent vertices). /// The key is the parent in the cluster. The distance is calculated to the other parents. pub distance_to_other_clusters: HashMap>, + /// Deduplication coefficient for each cluster. + pub cluster_dedup_ratio: HashMap } pub struct CopyScrubber; pub struct DumbScrubber; - impl Scrub for CopyScrubber where Hash: ChunkHash, @@ -122,6 +123,7 @@ where let mut number_of_vertices_in_cluster = HashMap::new(); let mut distance_to_other_clusters = HashMap::new(); let mut parent_vertices: Vec = Vec::new(); + let cluster_dedup_ratio = HashMap::new(); let now = Instant::now(); let mut processed_data = 0; @@ -161,6 +163,7 @@ where number_of_vertices_in_cluster, distance_to_vertices_in_cluster, distance_to_other_clusters, + cluster_dedup_ratio, }; Ok(ScrubMeasurements { processed_data, @@ -231,6 +234,7 @@ mod tests { assert!(cluster_report.number_of_vertices_in_cluster.values().all(|&v| v == 1)); assert!(cluster_report.distance_to_vertices_in_cluster.is_empty()); assert!(cluster_report.distance_to_other_clusters.values().all(|v| v.len() == test_data_len - 1)); + assert!(cluster_report.cluster_dedup_ratio.values().all(|&v| v == 0.0)); } #[test] From 74667d165cb9acb725510708bb6149a4f49e7d1e Mon Sep 17 00:00:00 2001 From: ArtemNikit1n Date: Wed, 26 Nov 2025 23:15:25 +0300 Subject: [PATCH 5/5] style: fix ci (Formatting) --- src/lib.rs | 2 +- src/system/scrub.rs | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f0a7d8f9..e9b612f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; pub use system::database::{Database, IterableDatabase}; -pub use system::scrub::{DumbScrubber, CopyScrubber, Scrub, ScrubMeasurements}; +pub use system::scrub::{CopyScrubber, DumbScrubber, Scrub, ScrubMeasurements}; pub use system::storage::{Data, DataContainer}; pub use system::{create_cdc_filesystem, FileSystem}; diff --git a/src/system/scrub.rs b/src/system/scrub.rs index d1b07d83..19d4e143 100644 --- a/src/system/scrub.rs +++ b/src/system/scrub.rs @@ -103,7 +103,7 @@ pub struct ClusteringMeasurements { /// The key is the parent in the cluster. The distance is calculated to the other parents. pub distance_to_other_clusters: HashMap>, /// Deduplication coefficient for each cluster. - pub cluster_dedup_ratio: HashMap + pub cluster_dedup_ratio: HashMap, } pub struct CopyScrubber; @@ -214,10 +214,7 @@ mod tests { let test_data_len = test_data.len(); for (hash, chunk) in test_data { total_data_size += chunk.len(); - database.insert( - hash.clone(), - DataContainer::from(chunk), - ); + database.insert(hash.clone(), DataContainer::from(chunk)); } let mut target_map: HashMap, Vec> = HashMap::new(); @@ -231,10 +228,19 @@ mod tests { let cluster_report = &scrub_report.clusterization_report; assert_eq!(cluster_report.total_cluster_size, test_data_len); assert_eq!(cluster_report.number_of_clusters, test_data_len); - assert!(cluster_report.number_of_vertices_in_cluster.values().all(|&v| v == 1)); + assert!(cluster_report + .number_of_vertices_in_cluster + .values() + .all(|&v| v == 1)); assert!(cluster_report.distance_to_vertices_in_cluster.is_empty()); - assert!(cluster_report.distance_to_other_clusters.values().all(|v| v.len() == test_data_len - 1)); - assert!(cluster_report.cluster_dedup_ratio.values().all(|&v| v == 0.0)); + assert!(cluster_report + .distance_to_other_clusters + .values() + .all(|v| v.len() == test_data_len - 1)); + assert!(cluster_report + .cluster_dedup_ratio + .values() + .all(|&v| v == 0.0)); } #[test] @@ -250,4 +256,4 @@ mod tests { assert_eq!(scrub_report.clusterization_report.total_cluster_size, 0); assert!(target_map.is_empty()); } -} \ No newline at end of file +}