Add simulator benchmarks.

Cargo.toml

@@ -34,6 +34,7 @@ rand = "0.10.1"
 rand_distr = "0.6.0"
 tempfile = "3.27.0"
 chacha20 = "0.10.0"
+criterion = "0.8.2"
 
 [profile.release]
 lto = true

crates/chaff-capture/src/trace.rs

@@ -67,7 +67,8 @@ impl TraceBuilder {
     pub fn record(&mut self, dir: Direction, time: u64, size: u32) {
         self.directions.push(dir);
         #[expect(clippy::cast_possible_truncation)]
-        self.timing_deltas.push((time - self.last_ts) as u32);
+        self.timing_deltas
+            .push(time.saturating_sub(self.last_ts) as u32);
         self.sizes.push(size);
         self.last_ts = time;
     }

crates/chaff-cli/src/main.rs

@@ -162,7 +162,18 @@ fn run() -> Result<(), CliError> {
                     dataset_type,
                 } => {
                     let input_dataset = parse_dataset(&dataset_type, &input)?;
-                    simulate::run_dataset(&input_dataset, &output, &machine)
+                    let overheads = simulate::run_dataset(&input_dataset, &output, &machine)?;
+                    if let Some(overheads) = overheads {
+                        println!(
+                            "Overall time overhead: {}μs",
+                            overheads.time_abs().unwrap_or_default().as_micros()
+                        );
+                        println!(
+                            "Overall bandwidth overhead: {} packets",
+                            overheads.bandwidth_abs()
+                        );
+                    }
+                    Ok(())
                 }
             }
         }

crates/chaff-cli/src/subcommands/simulate.rs

@@ -54,7 +54,7 @@ pub fn run_dataset(
     dataset: &Dataset,
     output: &Option<PathBuf>,
     machine: &Machine,
-) -> Result<(), CliError> {
+) -> Result<Option<SimulatorOverheads>, CliError> {
     let input_data = dataset.get_dataset();
     let mut output_dataset_builder = DatasetBuilder::new(dataset.get_pad_to());
 
@@ -69,7 +69,7 @@ pub fn run_dataset(
     let work_queue = Arc::new(Mutex::new(tasks.into_iter()));
     let (tx, rx) = mpsc::channel();
 
-    thread::scope(|s| {
+    let overheads = thread::scope(|s| {
         for _ in 0..num_threads {
             let thread_tx = tx.clone();
             let thread_machine = machine.clone();
@@ -111,10 +111,7 @@ pub fn run_dataset(
             overheads.push(overhead);
         }
 
-        let overheads_total = SimulatorOverheads::total_from(overheads);
-        if let Some(total) = overheads_total {
-            println!("{total}");
-        }
+        SimulatorOverheads::total_from(overheads)
     });
 
     if let Some(output) = output {
@@ -125,5 +122,5 @@ pub fn run_dataset(
         output_dataset_builder.build().dump_to(output)?;
     }
 
-    Ok(())
+    Ok(overheads)
 }

crates/chaff-sim/Cargo.toml

@@ -23,3 +23,11 @@ rand.workspace = true
 
 [dev-dependencies]
 chacha20 = { workspace = true, features = ["rng"] }
+chaff-datasets = { version = "0.1.0", path = "../chaff-datasets/" }
+chaff-machines = { version = "0.1.0", path = "../chaff-machines/" }
+chaff-cli = { version = "0.1.0", path = "../chaff-cli/" }
+criterion.workspace = true
+
+[[bench]]
+name = "tiktok_undefended"
+harness = false

crates/chaff-sim/README.md

@@ -1,3 +1,11 @@
 # Chaff Simulator
 
 See documentation and tests for usage.
+
+## Benchmarks
+
+To run benchmarks, you must first download the `Undefended.zip` dataset from [here](https://zenodo.org/records/11631265), unzip it to `WORKSPACE_ROOT/data/tik_tok_undefended`[^1], then use the `chaff-cli` to convert it to chaff dataset format with the `dataset-convert` subcommand into `WORKSPACE_ROOT/data/tik_tok_undefended.chaff`.
+
+Then, you can run `cargo bench` to run all benchmarks.
+
+[^1]: The `WORKSPACE_ROOT` is where the main `README.md` lives.

crates/chaff-sim/benches/tiktok_undefended.rs

@@ -0,0 +1,76 @@
+//! A benchmark which tests simulation performance on a few machines with
+//! the Tik-Tok unprocessed dataset.
+
+#![expect(clippy::unwrap_used)]
+
+use ::chaff::{action::IntegratorAction, event::Event, machine};
+use chaff_cli::subcommands::simulate;
+use chaff_datasets::parsers::chaff;
+use chaff_machines::constant;
+use criterion::{Criterion, criterion_group, criterion_main};
+use std::{hint::black_box, path::Path};
+
+fn constant(c: &mut Criterion) {
+    let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let dataset_dir = workspace_dir
+        .parent()
+        .and_then(Path::parent)
+        .unwrap()
+        .join("data/tik_tok_undefended.chaff");
+    let dataset = chaff::try_parse(dataset_dir).unwrap();
+    let machine = constant::construct();
+
+    c.bench_function("tiktok undefended const", |b| {
+        b.iter(|| simulate::run_dataset(black_box(&dataset), &None, black_box(&machine)));
+    });
+}
+
+fn no_op(c: &mut Criterion) {
+    let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let dataset_dir = workspace_dir
+        .parent()
+        .and_then(Path::parent)
+        .unwrap()
+        .join("data/tik_tok_undefended.chaff");
+    let dataset = chaff::try_parse(dataset_dir).unwrap();
+    let machine = machine! {
+        queues: [],
+        state init {},
+    }
+    .unwrap();
+
+    c.bench_function("tiktok undefended no_op", |b| {
+        b.iter(|| simulate::run_dataset(black_box(&dataset), &None, black_box(&machine)));
+    });
+}
+
+fn double(c: &mut Criterion) {
+    let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let dataset_dir = workspace_dir
+        .parent()
+        .and_then(Path::parent)
+        .unwrap()
+        .join("data/tik_tok_undefended.chaff");
+    let dataset = chaff::try_parse(dataset_dir).unwrap();
+    let machine = machine! {
+        queues: [],
+        state double {
+            action: IntegratorAction::SendDecoy,
+            transitions: [
+                Event::SendNormal => double,
+            ],
+        },
+    }
+    .unwrap();
+
+    c.bench_function("tiktok undefended double", |b| {
+        b.iter(|| simulate::run_dataset(black_box(&dataset), &None, black_box(&machine)));
+    });
+}
+
+criterion_group!(
+    name = tiktok_undefended;
+    config = Criterion::default().sample_size(10);
+    targets = constant, no_op, double
+);
+criterion_main!(tiktok_undefended);