diff --git a/Cargo.lock b/Cargo.lock index cf5ce0f..47e870a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.102" @@ -88,12 +109,24 @@ dependencies = [ "syn", ] +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.60" @@ -187,9 +220,40 @@ dependencies = [ "chacha20", "chaff", "chaff-capture", + "chaff-cli", + "chaff-datasets", + "chaff-machines", + "criterion", "rand", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.5.1" @@ -201,6 +265,31 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cpufeatures" version = "0.3.0" @@ -210,6 +299,72 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.2.1" @@ -219,6 +374,12 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "equivalent" version = "1.0.2" @@ -288,6 +449,17 @@ dependencies = [ "wasip3", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -354,12 +526,31 @@ dependencies = [ "num-traits", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "leb128fmt" version = "0.1.0" @@ -454,6 +645,22 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "pcap" version = "2.4.0" @@ -475,6 +682,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -545,6 +780,26 @@ dependencies = [ "rand", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.12.3" @@ -587,6 +842,21 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "semver" version = "1.0.28" @@ -600,6 +870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", + "serde_derive", ] [[package]] @@ -665,6 +936,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" @@ -713,6 +994,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -731,6 +1022,51 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +dependencies = [ + "unicode-ident", +] + [[package]] name = "wasm-encoder" version = "0.244.0" @@ -765,6 +1101,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -781,6 +1127,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -948,6 +1303,26 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index f6bd612..e861e43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ rand = "0.10.1" rand_distr = "0.6.0" tempfile = "3.27.0" chacha20 = "0.10.0" +criterion = "0.8.2" [profile.release] lto = true diff --git a/crates/chaff-capture/src/trace.rs b/crates/chaff-capture/src/trace.rs index ab87b95..8838690 100644 --- a/crates/chaff-capture/src/trace.rs +++ b/crates/chaff-capture/src/trace.rs @@ -67,7 +67,8 @@ impl TraceBuilder { pub fn record(&mut self, dir: Direction, time: u64, size: u32) { self.directions.push(dir); #[expect(clippy::cast_possible_truncation)] - self.timing_deltas.push((time - self.last_ts) as u32); + self.timing_deltas + .push(time.saturating_sub(self.last_ts) as u32); self.sizes.push(size); self.last_ts = time; } diff --git a/crates/chaff-cli/src/main.rs b/crates/chaff-cli/src/main.rs index da95982..fc53dc9 100644 --- a/crates/chaff-cli/src/main.rs +++ b/crates/chaff-cli/src/main.rs @@ -162,7 +162,18 @@ fn run() -> Result<(), CliError> { dataset_type, } => { let input_dataset = parse_dataset(&dataset_type, &input)?; - simulate::run_dataset(&input_dataset, &output, &machine) + let overheads = simulate::run_dataset(&input_dataset, &output, &machine)?; + if let Some(overheads) = overheads { + println!( + "Overall time overhead: {}μs", + overheads.time_abs().unwrap_or_default().as_micros() + ); + println!( + "Overall bandwidth overhead: {} packets", + overheads.bandwidth_abs() + ); + } + Ok(()) } } } diff --git a/crates/chaff-cli/src/subcommands/simulate.rs b/crates/chaff-cli/src/subcommands/simulate.rs index e2df0d6..85f2e76 100644 --- a/crates/chaff-cli/src/subcommands/simulate.rs +++ b/crates/chaff-cli/src/subcommands/simulate.rs @@ -54,7 +54,7 @@ pub fn run_dataset( dataset: &Dataset, output: &Option, machine: &Machine, -) -> Result<(), CliError> { +) -> Result, CliError> { let input_data = dataset.get_dataset(); let mut output_dataset_builder = DatasetBuilder::new(dataset.get_pad_to()); @@ -69,7 +69,7 @@ pub fn run_dataset( let work_queue = Arc::new(Mutex::new(tasks.into_iter())); let (tx, rx) = mpsc::channel(); - thread::scope(|s| { + let overheads = thread::scope(|s| { for _ in 0..num_threads { let thread_tx = tx.clone(); let thread_machine = machine.clone(); @@ -111,10 +111,7 @@ pub fn run_dataset( overheads.push(overhead); } - let overheads_total = SimulatorOverheads::total_from(overheads); - if let Some(total) = overheads_total { - println!("{total}"); - } + SimulatorOverheads::total_from(overheads) }); if let Some(output) = output { @@ -125,5 +122,5 @@ pub fn run_dataset( output_dataset_builder.build().dump_to(output)?; } - Ok(()) + Ok(overheads) } diff --git a/crates/chaff-sim/Cargo.toml b/crates/chaff-sim/Cargo.toml index a2e7151..c0a2344 100644 --- a/crates/chaff-sim/Cargo.toml +++ b/crates/chaff-sim/Cargo.toml @@ -23,3 +23,11 @@ rand.workspace = true [dev-dependencies] chacha20 = { workspace = true, features = ["rng"] } +chaff-datasets = { version = "0.1.0", path = "../chaff-datasets/" } +chaff-machines = { version = "0.1.0", path = "../chaff-machines/" } +chaff-cli = { version = "0.1.0", path = "../chaff-cli/" } +criterion.workspace = true + +[[bench]] +name = "tiktok_undefended" +harness = false diff --git a/crates/chaff-sim/README.md b/crates/chaff-sim/README.md index dc0b566..d329f90 100644 --- a/crates/chaff-sim/README.md +++ b/crates/chaff-sim/README.md @@ -1,3 +1,11 @@ # Chaff Simulator See documentation and tests for usage. + +## Benchmarks + +To run benchmarks, you must first download the `Undefended.zip` dataset from [here](https://zenodo.org/records/11631265), unzip it to `WORKSPACE_ROOT/data/tik_tok_undefended`[^1], then use the `chaff-cli` to convert it to chaff dataset format with the `dataset-convert` subcommand into `WORKSPACE_ROOT/data/tik_tok_undefended.chaff`. + +Then, you can run `cargo bench` to run all benchmarks. + +[^1]: The `WORKSPACE_ROOT` is where the main `README.md` lives. diff --git a/crates/chaff-sim/benches/tiktok_undefended.rs b/crates/chaff-sim/benches/tiktok_undefended.rs new file mode 100644 index 0000000..dbc3960 --- /dev/null +++ b/crates/chaff-sim/benches/tiktok_undefended.rs @@ -0,0 +1,76 @@ +//! A benchmark which tests simulation performance on a few machines with +//! the Tik-Tok unprocessed dataset. + +#![expect(clippy::unwrap_used)] + +use ::chaff::{action::IntegratorAction, event::Event, machine}; +use chaff_cli::subcommands::simulate; +use chaff_datasets::parsers::chaff; +use chaff_machines::constant; +use criterion::{Criterion, criterion_group, criterion_main}; +use std::{hint::black_box, path::Path}; + +fn constant(c: &mut Criterion) { + let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let dataset_dir = workspace_dir + .parent() + .and_then(Path::parent) + .unwrap() + .join("data/tik_tok_undefended.chaff"); + let dataset = chaff::try_parse(dataset_dir).unwrap(); + let machine = constant::construct(); + + c.bench_function("tiktok undefended const", |b| { + b.iter(|| simulate::run_dataset(black_box(&dataset), &None, black_box(&machine))); + }); +} + +fn no_op(c: &mut Criterion) { + let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let dataset_dir = workspace_dir + .parent() + .and_then(Path::parent) + .unwrap() + .join("data/tik_tok_undefended.chaff"); + let dataset = chaff::try_parse(dataset_dir).unwrap(); + let machine = machine! { + queues: [], + state init {}, + } + .unwrap(); + + c.bench_function("tiktok undefended no_op", |b| { + b.iter(|| simulate::run_dataset(black_box(&dataset), &None, black_box(&machine))); + }); +} + +fn double(c: &mut Criterion) { + let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let dataset_dir = workspace_dir + .parent() + .and_then(Path::parent) + .unwrap() + .join("data/tik_tok_undefended.chaff"); + let dataset = chaff::try_parse(dataset_dir).unwrap(); + let machine = machine! { + queues: [], + state double { + action: IntegratorAction::SendDecoy, + transitions: [ + Event::SendNormal => double, + ], + }, + } + .unwrap(); + + c.bench_function("tiktok undefended double", |b| { + b.iter(|| simulate::run_dataset(black_box(&dataset), &None, black_box(&machine))); + }); +} + +criterion_group!( + name = tiktok_undefended; + config = Criterion::default().sample_size(10); + targets = constant, no_op, double +); +criterion_main!(tiktok_undefended);