print all observations when running cost model benchmarks

9 files changed, 42 insertions, 102 deletions

diff --git a/src/crates/benchmarker/src/container.rs b/src/crates/benchmarker/src/container.rs
index 82fa6f2..470d3b3 100644
--- a/src/crates/benchmarker/src/container.rs
+++ b/src/crates/benchmarker/src/container.rs
@@ -1,7 +1,7 @@
 use primrose_library::traits::Container;
 use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng, Rng};
 
-use crate::{benchmark_op, print_result};
+use crate::{benchmark_op, print_results};
 
 pub fn benchmark_container<T, E>(rng: &mut StdRng, ns: &[usize])
 where
@@ -33,11 +33,9 @@ where
     );
 
     // Since we've repeated n times in each run
-    results.min_nanos /= n as f64;
-    results.avg_nanos /= n as f64;
-    results.max_nanos /= n as f64;
+    results.iter_mut().for_each(|t| *t = *t / n as f64);
 
-    print_result("insert", n, results);
+    print_results("insert", n, &results);
 }
 
 fn scenario_contains<T, E>(rng: &mut StdRng, n: usize)
@@ -70,7 +68,7 @@ where
         },
     );
 
-    print_result("contains", n, results);
+    print_results("contains", n, &results);
 }
 
 fn scenario_remove<T, E>(rng: &mut StdRng, n: usize)
@@ -103,7 +101,7 @@ where
         },
     );
 
-    print_result("remove", n, results);
+    print_results("remove", n, &results);
 }
 
 fn scenario_clear<T, E>(rng: &mut StdRng, n: usize)
@@ -126,5 +124,5 @@ where
         },
     );
 
-    print_result("clear", n, results);
+    print_results("clear", n, &results);
 }
diff --git a/src/crates/benchmarker/src/indexable.rs b/src/crates/benchmarker/src/indexable.rs
index e00a8bc..6fff998 100644
--- a/src/crates/benchmarker/src/indexable.rs
+++ b/src/crates/benchmarker/src/indexable.rs
@@ -3,7 +3,7 @@ use std::hint::black_box;
 use primrose_library::traits::{Container, Indexable};
 use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng, Rng};
 
-use crate::{benchmark_op, print_result};
+use crate::{benchmark_op, print_results};
 
 pub fn benchmark_indexable<T, E>(rng: &mut StdRng, ns: &[usize])
 where
@@ -39,7 +39,7 @@ where
         },
     );
 
-    print_result("first", n, results);
+    print_results("first", n, &results);
 }
 
 fn scenario_last<T, E>(rng: &mut StdRng, n: usize)
@@ -63,7 +63,7 @@ where
         },
     );
 
-    print_result("last", n, results);
+    print_results("last", n, &results);
 }
 
 fn scenario_nth<T, E>(rng: &mut StdRng, n: usize)
@@ -87,5 +87,5 @@ where
         },
     );
 
-    print_result("nth", n, results);
+    print_results("nth", n, &results);
 }
diff --git a/src/crates/benchmarker/src/lib.rs b/src/crates/benchmarker/src/lib.rs
index ed2e85c..7f1a0f8 100644
--- a/src/crates/benchmarker/src/lib.rs
+++ b/src/crates/benchmarker/src/lib.rs
@@ -18,16 +18,16 @@ pub use stack::*;
 const WARM_UP_TIME: Duration = Duration::from_millis(500);
 const MEASUREMENT_TIME: Duration = Duration::from_secs(1);
 
-struct BenchmarkResult {
-    min_nanos: f64,
-    avg_nanos: f64,
-    max_nanos: f64,
+pub type BenchmarkResult = f64;
+
+fn print_results(op: &str, n: usize, measurements: &[BenchmarkResult]) {
+    measurements.iter().for_each(|m| print_result(op, n, *m));
 }
 
 fn print_result(op: &str, n: usize, measurement: BenchmarkResult) {
     println!(
         "{}/{} time: [{:.3} ns {:.3} ns {:.3} ns]",
-        op, n, measurement.min_nanos, measurement.avg_nanos, measurement.max_nanos
+        op, n, measurement, measurement, measurement
     )
 }
 
@@ -40,23 +40,22 @@ fn print_result(op: &str, n: usize, measurement: BenchmarkResult) {
 fn benchmark_op<T, R>(
     mut setup: impl FnMut() -> T,
     mut op: impl FnMut(&mut T) -> R,
-) -> BenchmarkResult {
-    let mut times = 0;
-    let mut min = f64::MAX;
-    let mut max = f64::MIN;
-    let mut sum = 0.0;
-
+) -> Vec<BenchmarkResult> {
     let warmup_end = Instant::now() + WARM_UP_TIME;
+    let mut warmup_n = 0;
 
     // Run warmup
     while Instant::now() < warmup_end {
         let mut target = black_box(setup());
         black_box(op(&mut target));
+        warmup_n += 1;
     }
 
+    let to_run = (warmup_n * (MEASUREMENT_TIME.as_millis() / WARM_UP_TIME.as_millis())) as usize;
+    let mut times = Vec::with_capacity(to_run);
+
     // Benchmarking loop
-    let loop_end = Instant::now() + MEASUREMENT_TIME;
-    while Instant::now() < loop_end {
+    for _ in 0..to_run {
         let mut target = black_box(setup());
 
         let start = Instant::now();
@@ -64,47 +63,8 @@ fn benchmark_op<T, R>(
         let end = Instant::now();
         drop(target);
 
-        let duration = (end - start).as_nanos() as f64;
-
-        min = min.min(duration);
-        max = max.max(duration);
-        sum += duration;
-        times += 1;
+        times.push((end - start).as_nanos() as f64);
     }
 
-    BenchmarkResult {
-        min_nanos: min,
-        max_nanos: max,
-        avg_nanos: sum / times as f64,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::benchmark_op;
-    use std::time::Duration;
-
-    #[test]
-    fn benchmark_op_resets_properly() {
-        benchmark_op(
-            || false,
-            |b| {
-                assert!(!(*b));
-                *b = true;
-            },
-        );
-    }
-
-    #[test]
-    fn benchmark_op_times_properly() {
-        let results = benchmark_op(|| (), |_| std::thread::sleep(Duration::from_millis(5)));
-
-        let avg_millis = results.avg_nanos / (10.0_f64).powi(6);
-        dbg!(avg_millis);
-
-        assert!(
-            (avg_millis - 5.0).abs() < 0.1,
-            "sleeping for 5ms takes roughly 5ms"
-        )
-    }
+    times
 }
diff --git a/src/crates/benchmarker/src/mapping.rs b/src/crates/benchmarker/src/mapping.rs
index 0932a50..4b29a27 100644
--- a/src/crates/benchmarker/src/mapping.rs
+++ b/src/crates/benchmarker/src/mapping.rs
@@ -5,7 +5,7 @@ use rand::{
     Rng,
 };
 
-use crate::{benchmark_op, print_result};
+use crate::{benchmark_op, print_results};
 
 pub fn benchmark_mapping<T, K, V>(rng: &mut StdRng, ns: &[usize])
 where
@@ -46,11 +46,9 @@ where
     );
 
     // Since we've repeated n times in each run
-    results.min_nanos /= n as f64;
-    results.avg_nanos /= n as f64;
-    results.max_nanos /= n as f64;
+    results.iter_mut().for_each(|t| *t = *t / n as f64);
 
-    print_result("insert", n, results);
+    print_results("insert", n, &results);
 }
 
 fn scenario_contains<T, K, V>(rng: &mut StdRng, n: usize)
@@ -82,7 +80,7 @@ where
         |(c, chosen)| c.contains(chosen),
     );
 
-    print_result("contains", n, results);
+    print_results("contains", n, &results);
 }
 
 fn scenario_remove<T, K, V>(rng: &mut StdRng, n: usize)
@@ -114,7 +112,7 @@ where
         |(c, chosen)| c.remove(chosen),
     );
 
-    print_result("remove", n, results);
+    print_results("remove", n, &results);
 }
 
 fn scenario_clear<T, K, V>(rng: &mut StdRng, n: usize)
@@ -136,5 +134,5 @@ where
         |c| c.clear(),
     );
 
-    print_result("clear", n, results);
+    print_results("clear", n, &results);
 }
diff --git a/src/crates/benchmarker/src/stack.rs b/src/crates/benchmarker/src/stack.rs
index c5cf913..58fe6f2 100644
--- a/src/crates/benchmarker/src/stack.rs
+++ b/src/crates/benchmarker/src/stack.rs
@@ -3,7 +3,7 @@ use std::hint::black_box;
 use primrose_library::traits::{Container, Stack};
 use rand::{distributions::Standard, prelude::Distribution, rngs::StdRng, Rng};
 
-use crate::{benchmark_op, print_result};
+use crate::{benchmark_op, print_results};
 
 pub fn benchmark_stack<T, E>(rng: &mut StdRng, ns: &[usize])
 where
@@ -33,11 +33,9 @@ where
     );
 
     // Since we've repeated n times in each run
-    results.min_nanos /= n as f64;
-    results.avg_nanos /= n as f64;
-    results.max_nanos /= n as f64;
+    results.iter_mut().for_each(|t| *t = *t / n as f64);
 
-    print_result("push", n, results);
+    print_results("push", n, &results);
 }
 
 fn scenario_drain<T, E>(rng: &mut StdRng, n: usize)
@@ -62,9 +60,7 @@ where
     );
 
     // Since we've repeated n times in each run
-    results.min_nanos /= n as f64;
-    results.avg_nanos /= n as f64;
-    results.max_nanos /= n as f64;
+    results.iter_mut().for_each(|t| *t = *t / n as f64);
 
-    print_result("pop", n, results);
+    print_results("pop", n, &results);
 }
diff --git a/src/crates/candelabra/src/cost/benchmark.rs b/src/crates/candelabra/src/cost/benchmark.rs
index a5ca042..6769ee1 100644
--- a/src/crates/candelabra/src/cost/benchmark.rs
+++ b/src/crates/candelabra/src/cost/benchmark.rs
@@ -45,17 +45,7 @@ pub type OpName = String;
 pub type Observation = (usize, BenchmarkResult);
 
 /// Results for a single benchmark
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct BenchmarkResult {
-    /// The minimum cost
-    pub min: Cost,
-
-    /// The maximum cost
-    pub max: Cost,
-
-    /// The average (mean) cost
-    pub avg: Cost,
-}
+pub type BenchmarkResult = f64;
 
 /// Run benchmarks for the given container type, returning the results.
 /// Panics if the given name is not in the library specs.
@@ -141,11 +131,9 @@ pub(crate) fn parse_criterion_output(
                 .strip_suffix(']')?
                 .split(' ');
 
-            let result = BenchmarkResult {
-                min: parse_time_str(timings.next()?, timings.next()?)?,
-                avg: parse_time_str(timings.next()?, timings.next()?)?,
-                max: parse_time_str(timings.next()?, timings.next()?)?,
-            };
+            let _min = parse_time_str(timings.next()?, timings.next()?)?;
+            let result = parse_time_str(timings.next()?, timings.next()?)?;
+            // let _max = parse_time_str(timings.next()?, timings.next()?)?;
 
             Some((name, result))
         })
diff --git a/src/crates/candelabra/src/cost/fit.rs b/src/crates/candelabra/src/cost/fit.rs
index dada377..6160842 100644
--- a/src/crates/candelabra/src/cost/fit.rs
+++ b/src/crates/candelabra/src/cost/fit.rs
@@ -97,7 +97,7 @@ impl Estimator {
         let xs = results.iter().map(|(n, _)| *n as f64).collect::<Vec<_>>();
         let ys = OVector::<f64, Dyn>::from_iterator(
             results.len(),
-            results.iter().map(|(_, results)| results.avg),
+            results.iter().map(|(_, results)| *results),
         );
 
         (xs, ys)
diff --git a/src/crates/cli/src/estimate.rs b/src/crates/cli/src/estimate.rs
index d559b62..13af9a7 100644
--- a/src/crates/cli/src/estimate.rs
+++ b/src/crates/cli/src/estimate.rs
@@ -119,7 +119,7 @@ impl State {
                         .collect(),
                 )?
                 .into_iter()
-                .map(|(name, results)| (name, once(("avg", results.avg)))),
+                .map(|(name, results)| (name, once(("avg", results)))),
         );
 
         Ok(())
diff --git a/src/crates/cli/src/select.rs b/src/crates/cli/src/select.rs
index d4efc2b..9785c75 100644
--- a/src/crates/cli/src/select.rs
+++ b/src/crates/cli/src/select.rs
@@ -92,7 +92,7 @@ impl State {
             for (assignment, benchmark_results) in assignments_results.iter() {
                 let mut record = vec![assignment.to_string()];
                 for key in header.iter().skip(1) {
-                    record.push(format!("{:?}", benchmark_results.get(*key).unwrap().avg));
+                    record.push(format!("{:?}", benchmark_results.get(*key).unwrap()));
                 }
                 builder.push_record(record);
             }