refactor(benchmarker): store n alongside each result for flexibility

this allows us additional flexibility in our benchmarks, for example
we could benchmark long runs of an operation starting at some n.

we're not doing this right now though

6 files changed, 240 insertions, 219 deletions

diff --git a/primrose/crates/candelabra-benchmarker/src/bench.rs b/primrose/crates/candelabra-benchmarker/src/bench.rs
index 971c504..9c4cf74 100644
--- a/primrose/crates/candelabra-benchmarker/src/bench.rs
+++ b/primrose/crates/candelabra-benchmarker/src/bench.rs
@@ -4,7 +4,7 @@ use std::{
     time::{Duration, Instant},
 };
 
-use crate::RunResults;
+use crate::BenchmarkResult;
 
 /// Benchmark an operation for approx 5 seconds, returning the results.
 ///
@@ -16,7 +16,7 @@ pub fn benchmark_op<T>(
     mut setup: impl FnMut() -> T,
     mut op: impl FnMut(&mut T),
     mut undo: impl FnMut(&mut T),
-) -> RunResults {
+) -> BenchmarkResult {
     // let loop_end = Instant::now() + Duration::from_secs(5);
     let loop_end = Instant::now() + Duration::from_millis(100);
 
@@ -37,7 +37,7 @@ pub fn benchmark_op<T>(
         times += 1;
     }
 
-    RunResults {
+    BenchmarkResult {
         times,
         min,
         max,
diff --git a/primrose/crates/candelabra-benchmarker/src/container.rs b/primrose/crates/candelabra-benchmarker/src/container.rs
index 670e566..39e2288 100644
--- a/primrose/crates/candelabra-benchmarker/src/container.rs
+++ b/primrose/crates/candelabra-benchmarker/src/container.rs
@@ -4,27 +4,27 @@ use log::debug;
 use primrose_library::traits::Container;
 use rand::{distributions::Standard, prelude::Distribution, random, thread_rng, Rng};
 
-use crate::{bench::benchmark_op, RunResults, SingleNResults};
+use crate::{bench::benchmark_op, Observation, Results};
 
 /// Benchmark [`primrose_library::traits::Container`] operations
 pub trait ContainerExt<E> {
     /// Benchmark at a single `n`.
-    fn benchmark_container_at(n: usize) -> SingleNResults;
+    fn benchmark_container_at(n: usize) -> Results;
 
     /// Benchmark `len` at a single `n`.
-    fn benchmark_container_len(n: usize) -> RunResults;
+    fn benchmark_container_len(n: usize) -> Observation;
 
     /// Benchmark `contains` at a single `n`.
-    fn benchmark_container_contains(n: usize) -> RunResults;
+    fn benchmark_container_contains(n: usize) -> Observation;
 
     /// Benchmark `insert` at a single `n`.
-    fn benchmark_container_insert(n: usize) -> RunResults;
+    fn benchmark_container_insert(n: usize) -> Observation;
 
     /// Benchmark `clear` at a single `n`.
-    fn benchmark_container_clear(n: usize) -> RunResults;
+    fn benchmark_container_clear(n: usize) -> Observation;
 
     /// Benchmark `remove` at a single `n`.
-    fn benchmark_container_remove(n: usize) -> RunResults;
+    fn benchmark_container_remove(n: usize) -> Observation;
 }
 
 impl<T, E> ContainerExt<E> for T
@@ -33,77 +33,92 @@ where
     E: Clone,
     Standard: Distribution<E>,
 {
-    fn benchmark_container_at(n: usize) -> SingleNResults {
+    fn benchmark_container_at(n: usize) -> Results {
         let mut by_op = HashMap::new();
 
         debug!("Benchmarking {} at n = {}", type_name::<T>(), n);
 
         debug!("...len");
-        by_op.insert("len".to_string(), Self::benchmark_container_len(n));
+        by_op.insert("len".to_string(), vec![Self::benchmark_container_len(n)]);
         debug!("...contains");
         by_op.insert(
             "contains".to_string(),
-            Self::benchmark_container_contains(n),
+            vec![Self::benchmark_container_contains(n)],
         );
         debug!("...insert");
-        by_op.insert("insert".to_string(), Self::benchmark_container_insert(n));
+        by_op.insert(
+            "insert".to_string(),
+            vec![Self::benchmark_container_insert(n)],
+        );
         debug!("...clear");
-        by_op.insert("clear".to_string(), Self::benchmark_container_clear(n));
+        by_op.insert(
+            "clear".to_string(),
+            vec![Self::benchmark_container_clear(n)],
+        );
         debug!("...remove");
-        by_op.insert("remove".to_string(), Self::benchmark_container_remove(n));
+        by_op.insert(
+            "remove".to_string(),
+            vec![Self::benchmark_container_remove(n)],
+        );
         debug!("--- done!");
 
-        SingleNResults { by_op }
+        Results { by_op }
     }
 
-    fn benchmark_container_contains(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                // TODO: maybe we should actually just test the worst case? (at the end)
-                // we also don't actually test misses yet.
-
-                let mut rng = thread_rng();
-                let mut c = T::default();
-
-                // decide where the element that we will search for will be
-                let pivot = rng.gen_range(0..n);
-
-                // insert the element at pivot, and keep track of what it is
-                for _ in 0..pivot {
-                    c.insert(random());
-                }
-                let chosen = rng.gen();
-                c.insert(chosen.clone());
-                for _ in pivot..n {
-                    c.insert(random());
-                }
-
-                (c, chosen)
-            },
-            |(c, search)| {
-                c.contains(search);
-            },
-            |_| (),
+    fn benchmark_container_contains(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    // TODO: maybe we should actually just test the worst case? (at the end)
+                    // we also don't actually test misses yet.
+
+                    let mut rng = thread_rng();
+                    let mut c = T::default();
+
+                    // decide where the element that we will search for will be
+                    let pivot = rng.gen_range(0..n);
+
+                    // insert the element at pivot, and keep track of what it is
+                    for _ in 0..pivot {
+                        c.insert(random());
+                    }
+                    let chosen = rng.gen();
+                    c.insert(chosen.clone());
+                    for _ in pivot..n {
+                        c.insert(random());
+                    }
+
+                    (c, chosen)
+                },
+                |(c, search)| {
+                    c.contains(search);
+                },
+                |_| (),
+            ),
         )
     }
 
-    fn benchmark_container_len(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut c = T::default();
-                for _ in 0..n {
-                    c.insert(random());
-                }
-                c
-            },
-            |c| {
-                c.len();
-            },
-            |_| (),
+    fn benchmark_container_len(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut c = T::default();
+                    for _ in 0..n {
+                        c.insert(random());
+                    }
+                    c
+                },
+                |c| {
+                    c.len();
+                },
+                |_| (),
+            ),
         )
     }
 
-    fn benchmark_container_insert(n: usize) -> RunResults {
+    fn benchmark_container_insert(n: usize) -> Observation {
         let setup_closure = || {
             let mut c = T::default();
             for _ in 0..n {
@@ -111,17 +126,20 @@ where
             }
             c
         };
-        benchmark_op(
-            setup_closure,
-            |c| {
-                // TODO: rng generation could throw off benchmarks
-                c.insert(random());
-            },
-            |c| *c = setup_closure(),
+        (
+            n,
+            benchmark_op(
+                setup_closure,
+                |c| {
+                    // TODO: rng generation could throw off benchmarks
+                    c.insert(random());
+                },
+                |c| *c = setup_closure(),
+            ),
         )
     }
 
-    fn benchmark_container_clear(n: usize) -> RunResults {
+    fn benchmark_container_clear(n: usize) -> Observation {
         let setup_closure = || {
             let mut c = T::default();
             for _ in 0..n {
@@ -129,40 +147,46 @@ where
             }
             c
         };
-        benchmark_op(
-            setup_closure,
-            |c| {
-                c.clear();
-            },
-            |c| *c = setup_closure(),
+        (
+            n,
+            benchmark_op(
+                setup_closure,
+                |c| {
+                    c.clear();
+                },
+                |c| *c = setup_closure(),
+            ),
         )
     }
 
-    fn benchmark_container_remove(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut rng = thread_rng();
-                let mut c = T::default();
-
-                // decide where the element that we will remove will be
-                let pivot = rng.gen_range(0..n);
-
-                // insert the element at pivot, and keep track of what it is
-                for _ in 0..pivot {
-                    c.insert(random());
-                }
-                let chosen = rng.gen();
-                c.insert(chosen.clone());
-                for _ in pivot..n {
-                    c.insert(random());
-                }
-
-                (c, chosen)
-            },
-            |(c, chosen)| {
-                c.remove(chosen.clone());
-            },
-            |(c, chosen)| c.insert(chosen.clone()),
+    fn benchmark_container_remove(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut rng = thread_rng();
+                    let mut c = T::default();
+
+                    // decide where the element that we will remove will be
+                    let pivot = rng.gen_range(0..n);
+
+                    // insert the element at pivot, and keep track of what it is
+                    for _ in 0..pivot {
+                        c.insert(random());
+                    }
+                    let chosen = rng.gen();
+                    c.insert(chosen.clone());
+                    for _ in pivot..n {
+                        c.insert(random());
+                    }
+
+                    (c, chosen)
+                },
+                |(c, chosen)| {
+                    c.remove(chosen.clone());
+                },
+                |(c, chosen)| c.insert(chosen.clone()),
+            ),
         )
     }
 }
diff --git a/primrose/crates/candelabra-benchmarker/src/indexable.rs b/primrose/crates/candelabra-benchmarker/src/indexable.rs
index eadb397..066ad8a 100644
--- a/primrose/crates/candelabra-benchmarker/src/indexable.rs
+++ b/primrose/crates/candelabra-benchmarker/src/indexable.rs
@@ -4,21 +4,21 @@ use log::debug;
 use primrose_library::traits::{Container, Indexable};
 use rand::{distributions::Standard, prelude::Distribution, random};
 
-use crate::{benchmark_op, RunResults, SingleNResults};
+use crate::{benchmark_op, BenchmarkResult, Observation, Results};
 
 /// Benchmark [`primrose_library::traits::Indexable`] operations
 pub trait IndexableExt<E> {
     /// Benchmark at a single `n`.
-    fn benchmark_indexable_at(n: usize) -> SingleNResults;
+    fn benchmark_indexable_at(n: usize) -> Results;
 
     /// Benchmark `first` at a single `n`.
-    fn benchmark_indexable_first(n: usize) -> RunResults;
+    fn benchmark_indexable_first(n: usize) -> Observation;
 
     /// Benchmark `last` at a single `n`.
-    fn benchmark_indexable_last(n: usize) -> RunResults;
+    fn benchmark_indexable_last(n: usize) -> Observation;
 
     /// Benchmark `nth` at a single `n`.
-    fn benchmark_indexable_nth(n: usize) -> RunResults;
+    fn benchmark_indexable_nth(n: usize) -> Observation;
 }
 
 impl<T, E> IndexableExt<E> for T
@@ -26,67 +26,79 @@ where
     T: Container<E> + Indexable<E> + Default,
     Standard: Distribution<E>,
 {
-    fn benchmark_indexable_at(n: usize) -> SingleNResults {
+    fn benchmark_indexable_at(n: usize) -> Results {
         let mut by_op = HashMap::new();
 
         debug!("Benchmarking {} at n = {}", type_name::<T>(), n);
 
         debug!("...first");
-        by_op.insert("first".to_string(), Self::benchmark_indexable_first(n));
+        by_op.insert(
+            "first".to_string(),
+            vec![Self::benchmark_indexable_first(n)],
+        );
         debug!("...last");
-        by_op.insert("last".to_string(), Self::benchmark_indexable_last(n));
+        by_op.insert("last".to_string(), vec![Self::benchmark_indexable_last(n)]);
         debug!("...nth");
-        by_op.insert("nth".to_string(), Self::benchmark_indexable_nth(n));
+        by_op.insert("nth".to_string(), vec![Self::benchmark_indexable_nth(n)]);
         debug!("--- done!");
 
-        SingleNResults { by_op }
+        Results { by_op }
     }
 
-    fn benchmark_indexable_first(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut c = T::default();
-                for _ in 0..n {
-                    c.insert(random());
-                }
-                c
-            },
-            |c| {
-                c.first();
-            },
-            |_| (),
+    fn benchmark_indexable_first(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut c = T::default();
+                    for _ in 0..n {
+                        c.insert(random());
+                    }
+                    c
+                },
+                |c| {
+                    c.first();
+                },
+                |_| (),
+            ),
         )
     }
 
-    fn benchmark_indexable_last(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut c = T::default();
-                for _ in 0..n {
-                    c.insert(random());
-                }
-                c
-            },
-            |c| {
-                c.last();
-            },
-            |_| (),
+    fn benchmark_indexable_last(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut c = T::default();
+                    for _ in 0..n {
+                        c.insert(random());
+                    }
+                    c
+                },
+                |c| {
+                    c.last();
+                },
+                |_| (),
+            ),
         )
     }
 
-    fn benchmark_indexable_nth(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut c = T::default();
-                for _ in 0..n {
-                    c.insert(random());
-                }
-                (c, random::<usize>())
-            },
-            |(c, fetch)| {
-                c.nth(*fetch);
-            },
-            |_| (),
+    fn benchmark_indexable_nth(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut c = T::default();
+                    for _ in 0..n {
+                        c.insert(random());
+                    }
+                    (c, random::<usize>())
+                },
+                |(c, fetch)| {
+                    c.nth(*fetch);
+                },
+                |_| (),
+            ),
         )
     }
 }
diff --git a/primrose/crates/candelabra-benchmarker/src/lib.rs b/primrose/crates/candelabra-benchmarker/src/lib.rs
index daa5706..e6fc69b 100644
--- a/primrose/crates/candelabra-benchmarker/src/lib.rs
+++ b/primrose/crates/candelabra-benchmarker/src/lib.rs
@@ -27,7 +27,7 @@ impl<T, E> Benchmarker<T, E> {
         Self(
             ns,
             Results {
-                by_n: HashMap::new(),
+                by_op: HashMap::new(),
             },
             PhantomData,
         )
@@ -46,7 +46,7 @@ where
     /// Run benchmarks for [`primrose_library::traits::Container`] operations.
     pub fn container(mut self) -> Self {
         for n in self.0 {
-            self.1.add(*n, T::benchmark_container_at(*n));
+            self.1.merge(T::benchmark_container_at(*n));
         }
 
         self
@@ -59,7 +59,7 @@ where
     /// Run benchmarks for [`primrose_library::traits::Indexable`] operations.
     pub fn indexable(mut self) -> Self {
         for n in self.0 {
-            self.1.add(*n, T::benchmark_indexable_at(*n));
+            self.1.merge(T::benchmark_indexable_at(*n));
         }
 
         self
@@ -73,7 +73,7 @@ where
     /// Run benchmarks for [`primrose_library::traits::Stack`] operations.
     pub fn stack(mut self) -> Self {
         for n in self.0 {
-            self.1.add(*n, T::benchmark_stack_at(*n));
+            self.1.merge(T::benchmark_stack_at(*n));
         }
 
         self
diff --git a/primrose/crates/candelabra-benchmarker/src/results.rs b/primrose/crates/candelabra-benchmarker/src/results.rs
index e04ec45..3c0783f 100644
--- a/primrose/crates/candelabra-benchmarker/src/results.rs
+++ b/primrose/crates/candelabra-benchmarker/src/results.rs
@@ -5,23 +5,19 @@ use serde::{Deserialize, Serialize};
 /// Results for a whole suite of benchmarks
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct Results {
-    /// Results for each collection size tested with
-    pub by_n: HashMap<usize, SingleNResults>,
+    /// Results for each collection operation
+    pub by_op: HashMap<OpName, Vec<Observation>>,
 }
 
 /// Name of an operation
 pub type OpName = String;
 
-/// Results for operations all ran at the same container size
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct SingleNResults {
-    /// Results for each operation
-    pub by_op: HashMap<OpName, RunResults>,
-}
+/// The first key in the tuple is the `n` of the container before the benchmark was taken, and the second the results of the benchmark.
+pub type Observation = (usize, BenchmarkResult);
 
 /// Results for a single benchmark
 #[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct RunResults {
+pub struct BenchmarkResult {
     /// Number of times the benchmark was run
     pub times: usize,
 
@@ -36,31 +32,14 @@ pub struct RunResults {
 }
 
 impl Results {
-    pub fn add(&mut self, n: usize, b: SingleNResults) -> &mut Self {
-        self.by_n
-            .entry(n)
-            .and_modify(|res_a| {
-                res_a.merge(b.clone());
-            })
-            .or_insert(b);
-
-        self
-    }
-    pub fn merge(&mut self, b: Self) -> &mut Self {
-        for (n, res_b) in b.by_n {
-            self.add(n, res_b);
-        }
-
-        self
-    }
-}
-
-impl SingleNResults {
     /// Merge results from `b` into these results.
-    /// If `b` contains benchmarks for operations we have a result for, results from `b` are preferred.
+    /// If `b` contains benchmarks for operations we have a result for, the observations are merged.
     pub fn merge(&mut self, b: Self) -> &mut Self {
-        for (name, res_b) in b.by_op {
-            self.by_op.insert(name, res_b);
+        for (name, mut res_b) in b.by_op {
+            self.by_op
+                .entry(name)
+                .and_modify(|res| res.append(&mut res_b))
+                .or_insert(res_b);
         }
 
         self
diff --git a/primrose/crates/candelabra-benchmarker/src/stack.rs b/primrose/crates/candelabra-benchmarker/src/stack.rs
index 2580ce3..7680548 100644
--- a/primrose/crates/candelabra-benchmarker/src/stack.rs
+++ b/primrose/crates/candelabra-benchmarker/src/stack.rs
@@ -4,18 +4,18 @@ use log::debug;
 use primrose_library::traits::Stack;
 use rand::{distributions::Standard, prelude::Distribution, random};
 
-use crate::{benchmark_op, RunResults, SingleNResults};
+use crate::{benchmark_op, Observation, Results};
 
 /// Benchmark [`primrose_library::traits::Stack`] operations
 pub trait StackExt<E> {
     /// Benchmark at a single `n`.
-    fn benchmark_stack_at(n: usize) -> SingleNResults;
+    fn benchmark_stack_at(n: usize) -> Results;
 
     /// Benchmark `push` at a single `n`.
-    fn benchmark_stack_push(n: usize) -> RunResults;
+    fn benchmark_stack_push(n: usize) -> Observation;
 
     /// Benchmark `pop` at a single `n`.
-    fn benchmark_stack_pop(n: usize) -> RunResults;
+    fn benchmark_stack_pop(n: usize) -> Observation;
 }
 
 impl<T, E> StackExt<E> for T
@@ -23,51 +23,57 @@ where
     T: Stack<E> + Default,
     Standard: Distribution<E>,
 {
-    fn benchmark_stack_at(n: usize) -> SingleNResults {
+    fn benchmark_stack_at(n: usize) -> Results {
         let mut by_op = HashMap::new();
 
         debug!("Benchmarking {} at n = {}", type_name::<T>(), n);
 
         debug!("...push");
-        by_op.insert("push".to_string(), Self::benchmark_stack_push(n));
+        by_op.insert("push".to_string(), vec![Self::benchmark_stack_push(n)]);
         debug!("...pop");
-        by_op.insert("pop".to_string(), Self::benchmark_stack_pop(n));
+        by_op.insert("pop".to_string(), vec![Self::benchmark_stack_pop(n)]);
         debug!("--- done!");
 
-        SingleNResults { by_op }
+        Results { by_op }
     }
 
-    fn benchmark_stack_push(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut c = T::default();
-                for _ in 0..n {
-                    c.push(random());
-                }
-                c
-            },
-            |s| s.push(random()),
-            |s| {
-                s.pop();
-            },
+    fn benchmark_stack_push(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut c = T::default();
+                    for _ in 0..n {
+                        c.push(random());
+                    }
+                    c
+                },
+                |s| s.push(random()),
+                |s| {
+                    s.pop();
+                },
+            ),
         )
     }
 
-    fn benchmark_stack_pop(n: usize) -> RunResults {
-        benchmark_op(
-            || {
-                let mut c = T::default();
-                for _ in 0..n {
-                    c.push(random());
-                }
-                c
-            },
-            |s| {
-                s.pop();
-            },
-            |s| {
-                s.push(random());
-            },
+    fn benchmark_stack_pop(n: usize) -> Observation {
+        (
+            n,
+            benchmark_op(
+                || {
+                    let mut c = T::default();
+                    for _ in 0..n {
+                        c.push(random());
+                    }
+                    c
+                },
+                |s| {
+                    s.pop();
+                },
+                |s| {
+                    s.push(random());
+                },
+            ),
         )
     }
 }