From 258f9112871570bd9d65359bd814d033374d302e Mon Sep 17 00:00:00 2001 From: Aria Shrimpton Date: Fri, 1 Mar 2024 14:41:13 +0000 Subject: print all observations when running cost model benchmarks --- src/crates/benchmarker/src/container.rs | 14 +++--- src/crates/benchmarker/src/indexable.rs | 8 ++-- src/crates/benchmarker/src/lib.rs | 68 ++++++----------------------- src/crates/benchmarker/src/mapping.rs | 14 +++--- src/crates/benchmarker/src/stack.rs | 14 +++--- src/crates/candelabra/src/cost/benchmark.rs | 20 ++------- src/crates/candelabra/src/cost/fit.rs | 2 +- src/crates/cli/src/estimate.rs | 2 +- src/crates/cli/src/select.rs | 2 +- 9 files changed, 42 insertions(+), 102 deletions(-) (limited to 'src') diff --git a/src/crates/benchmarker/src/container.rs b/src/crates/benchmarker/src/container.rs index 82fa6f2..470d3b3 100644 --- a/src/crates/benchmarker/src/container.rs +++ b/src/crates/benchmarker/src/container.rs @@ -1,7 +1,7 @@ use primrose_library::traits::Container; use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng, Rng}; -use crate::{benchmark_op, print_result}; +use crate::{benchmark_op, print_results}; pub fn benchmark_container(rng: &mut StdRng, ns: &[usize]) where @@ -33,11 +33,9 @@ where ); // Since we've repeated n times in each run - results.min_nanos /= n as f64; - results.avg_nanos /= n as f64; - results.max_nanos /= n as f64; + results.iter_mut().for_each(|t| *t = *t / n as f64); - print_result("insert", n, results); + print_results("insert", n, &results); } fn scenario_contains(rng: &mut StdRng, n: usize) @@ -70,7 +68,7 @@ where }, ); - print_result("contains", n, results); + print_results("contains", n, &results); } fn scenario_remove(rng: &mut StdRng, n: usize) @@ -103,7 +101,7 @@ where }, ); - print_result("remove", n, results); + print_results("remove", n, &results); } fn scenario_clear(rng: &mut StdRng, n: usize) @@ -126,5 +124,5 @@ where }, ); - print_result("clear", n, results); + print_results("clear", n, &results); } diff --git a/src/crates/benchmarker/src/indexable.rs b/src/crates/benchmarker/src/indexable.rs index e00a8bc..6fff998 100644 --- a/src/crates/benchmarker/src/indexable.rs +++ b/src/crates/benchmarker/src/indexable.rs @@ -3,7 +3,7 @@ use std::hint::black_box; use primrose_library::traits::{Container, Indexable}; use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng, Rng}; -use crate::{benchmark_op, print_result}; +use crate::{benchmark_op, print_results}; pub fn benchmark_indexable(rng: &mut StdRng, ns: &[usize]) where @@ -39,7 +39,7 @@ where }, ); - print_result("first", n, results); + print_results("first", n, &results); } fn scenario_last(rng: &mut StdRng, n: usize) @@ -63,7 +63,7 @@ where }, ); - print_result("last", n, results); + print_results("last", n, &results); } fn scenario_nth(rng: &mut StdRng, n: usize) @@ -87,5 +87,5 @@ where }, ); - print_result("nth", n, results); + print_results("nth", n, &results); } diff --git a/src/crates/benchmarker/src/lib.rs b/src/crates/benchmarker/src/lib.rs index ed2e85c..7f1a0f8 100644 --- a/src/crates/benchmarker/src/lib.rs +++ b/src/crates/benchmarker/src/lib.rs @@ -18,16 +18,16 @@ pub use stack::*; const WARM_UP_TIME: Duration = Duration::from_millis(500); const MEASUREMENT_TIME: Duration = Duration::from_secs(1); -struct BenchmarkResult { - min_nanos: f64, - avg_nanos: f64, - max_nanos: f64, +pub type BenchmarkResult = f64; + +fn print_results(op: &str, n: usize, measurements: &[BenchmarkResult]) { + measurements.iter().for_each(|m| print_result(op, n, *m)); } fn print_result(op: &str, n: usize, measurement: BenchmarkResult) { println!( "{}/{} time: [{:.3} ns {:.3} ns {:.3} ns]", - op, n, measurement.min_nanos, measurement.avg_nanos, measurement.max_nanos + op, n, measurement, measurement, measurement ) } @@ -40,23 +40,22 @@ fn print_result(op: &str, n: usize, measurement: BenchmarkResult) { fn benchmark_op( mut setup: impl FnMut() -> T, mut op: impl FnMut(&mut T) -> R, -) -> BenchmarkResult { - let mut times = 0; - let mut min = f64::MAX; - let mut max = f64::MIN; - let mut sum = 0.0; - +) -> Vec { let warmup_end = Instant::now() + WARM_UP_TIME; + let mut warmup_n = 0; // Run warmup while Instant::now() < warmup_end { let mut target = black_box(setup()); black_box(op(&mut target)); + warmup_n += 1; } + let to_run = (warmup_n * (MEASUREMENT_TIME.as_millis() / WARM_UP_TIME.as_millis())) as usize; + let mut times = Vec::with_capacity(to_run); + // Benchmarking loop - let loop_end = Instant::now() + MEASUREMENT_TIME; - while Instant::now() < loop_end { + for _ in 0..to_run { let mut target = black_box(setup()); let start = Instant::now(); @@ -64,47 +63,8 @@ fn benchmark_op( let end = Instant::now(); drop(target); - let duration = (end - start).as_nanos() as f64; - - min = min.min(duration); - max = max.max(duration); - sum += duration; - times += 1; + times.push((end - start).as_nanos() as f64); } - BenchmarkResult { - min_nanos: min, - max_nanos: max, - avg_nanos: sum / times as f64, - } -} - -#[cfg(test)] -mod tests { - use super::benchmark_op; - use std::time::Duration; - - #[test] - fn benchmark_op_resets_properly() { - benchmark_op( - || false, - |b| { - assert!(!(*b)); - *b = true; - }, - ); - } - - #[test] - fn benchmark_op_times_properly() { - let results = benchmark_op(|| (), |_| std::thread::sleep(Duration::from_millis(5))); - - let avg_millis = results.avg_nanos / (10.0_f64).powi(6); - dbg!(avg_millis); - - assert!( - (avg_millis - 5.0).abs() < 0.1, - "sleeping for 5ms takes roughly 5ms" - ) - } + times } diff --git a/src/crates/benchmarker/src/mapping.rs b/src/crates/benchmarker/src/mapping.rs index 0932a50..4b29a27 100644 --- a/src/crates/benchmarker/src/mapping.rs +++ b/src/crates/benchmarker/src/mapping.rs @@ -5,7 +5,7 @@ use rand::{ Rng, }; -use crate::{benchmark_op, print_result}; +use crate::{benchmark_op, print_results}; pub fn benchmark_mapping(rng: &mut StdRng, ns: &[usize]) where @@ -46,11 +46,9 @@ where ); // Since we've repeated n times in each run - results.min_nanos /= n as f64; - results.avg_nanos /= n as f64; - results.max_nanos /= n as f64; + results.iter_mut().for_each(|t| *t = *t / n as f64); - print_result("insert", n, results); + print_results("insert", n, &results); } fn scenario_contains(rng: &mut StdRng, n: usize) @@ -82,7 +80,7 @@ where |(c, chosen)| c.contains(chosen), ); - print_result("contains", n, results); + print_results("contains", n, &results); } fn scenario_remove(rng: &mut StdRng, n: usize) @@ -114,7 +112,7 @@ where |(c, chosen)| c.remove(chosen), ); - print_result("remove", n, results); + print_results("remove", n, &results); } fn scenario_clear(rng: &mut StdRng, n: usize) @@ -136,5 +134,5 @@ where |c| c.clear(), ); - print_result("clear", n, results); + print_results("clear", n, &results); } diff --git a/src/crates/benchmarker/src/stack.rs b/src/crates/benchmarker/src/stack.rs index c5cf913..58fe6f2 100644 --- a/src/crates/benchmarker/src/stack.rs +++ b/src/crates/benchmarker/src/stack.rs @@ -3,7 +3,7 @@ use std::hint::black_box; use primrose_library::traits::{Container, Stack}; use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng, Rng}; -use crate::{benchmark_op, print_result}; +use crate::{benchmark_op, print_results}; pub fn benchmark_stack(rng: &mut StdRng, ns: &[usize]) where @@ -33,11 +33,9 @@ where ); // Since we've repeated n times in each run - results.min_nanos /= n as f64; - results.avg_nanos /= n as f64; - results.max_nanos /= n as f64; + results.iter_mut().for_each(|t| *t = *t / n as f64); - print_result("push", n, results); + print_results("push", n, &results); } fn scenario_drain(rng: &mut StdRng, n: usize) @@ -62,9 +60,7 @@ where ); // Since we've repeated n times in each run - results.min_nanos /= n as f64; - results.avg_nanos /= n as f64; - results.max_nanos /= n as f64; + results.iter_mut().for_each(|t| *t = *t / n as f64); - print_result("pop", n, results); + print_results("pop", n, &results); } diff --git a/src/crates/candelabra/src/cost/benchmark.rs b/src/crates/candelabra/src/cost/benchmark.rs index a5ca042..6769ee1 100644 --- a/src/crates/candelabra/src/cost/benchmark.rs +++ b/src/crates/candelabra/src/cost/benchmark.rs @@ -45,17 +45,7 @@ pub type OpName = String; pub type Observation = (usize, BenchmarkResult); /// Results for a single benchmark -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct BenchmarkResult { - /// The minimum cost - pub min: Cost, - - /// The maximum cost - pub max: Cost, - - /// The average (mean) cost - pub avg: Cost, -} +pub type BenchmarkResult = f64; /// Run benchmarks for the given container type, returning the results. /// Panics if the given name is not in the library specs. @@ -141,11 +131,9 @@ pub(crate) fn parse_criterion_output( .strip_suffix(']')? .split(' '); - let result = BenchmarkResult { - min: parse_time_str(timings.next()?, timings.next()?)?, - avg: parse_time_str(timings.next()?, timings.next()?)?, - max: parse_time_str(timings.next()?, timings.next()?)?, - }; + let _min = parse_time_str(timings.next()?, timings.next()?)?; + let result = parse_time_str(timings.next()?, timings.next()?)?; + // let _max = parse_time_str(timings.next()?, timings.next()?)?; Some((name, result)) }) diff --git a/src/crates/candelabra/src/cost/fit.rs b/src/crates/candelabra/src/cost/fit.rs index dada377..6160842 100644 --- a/src/crates/candelabra/src/cost/fit.rs +++ b/src/crates/candelabra/src/cost/fit.rs @@ -97,7 +97,7 @@ impl Estimator { let xs = results.iter().map(|(n, _)| *n as f64).collect::>(); let ys = OVector::::from_iterator( results.len(), - results.iter().map(|(_, results)| results.avg), + results.iter().map(|(_, results)| *results), ); (xs, ys) diff --git a/src/crates/cli/src/estimate.rs b/src/crates/cli/src/estimate.rs index d559b62..13af9a7 100644 --- a/src/crates/cli/src/estimate.rs +++ b/src/crates/cli/src/estimate.rs @@ -119,7 +119,7 @@ impl State { .collect(), )? .into_iter() - .map(|(name, results)| (name, once(("avg", results.avg)))), + .map(|(name, results)| (name, once(("avg", results)))), ); Ok(()) diff --git a/src/crates/cli/src/select.rs b/src/crates/cli/src/select.rs index d4efc2b..9785c75 100644 --- a/src/crates/cli/src/select.rs +++ b/src/crates/cli/src/select.rs @@ -92,7 +92,7 @@ impl State { for (assignment, benchmark_results) in assignments_results.iter() { let mut record = vec![assignment.to_string()]; for key in header.iter().skip(1) { - record.push(format!("{:?}", benchmark_results.get(*key).unwrap().avg)); + record.push(format!("{:?}", benchmark_results.get(*key).unwrap())); } builder.push_record(record); } -- cgit v1.2.3