diff options
6 files changed, 240 insertions, 219 deletions
diff --git a/primrose/crates/candelabra-benchmarker/src/bench.rs b/primrose/crates/candelabra-benchmarker/src/bench.rs index 971c504..9c4cf74 100644 --- a/primrose/crates/candelabra-benchmarker/src/bench.rs +++ b/primrose/crates/candelabra-benchmarker/src/bench.rs @@ -4,7 +4,7 @@ use std::{ time::{Duration, Instant}, }; -use crate::RunResults; +use crate::BenchmarkResult; /// Benchmark an operation for approx 5 seconds, returning the results. /// @@ -16,7 +16,7 @@ pub fn benchmark_op<T>( mut setup: impl FnMut() -> T, mut op: impl FnMut(&mut T), mut undo: impl FnMut(&mut T), -) -> RunResults { +) -> BenchmarkResult { // let loop_end = Instant::now() + Duration::from_secs(5); let loop_end = Instant::now() + Duration::from_millis(100); @@ -37,7 +37,7 @@ pub fn benchmark_op<T>( times += 1; } - RunResults { + BenchmarkResult { times, min, max, diff --git a/primrose/crates/candelabra-benchmarker/src/container.rs b/primrose/crates/candelabra-benchmarker/src/container.rs index 670e566..39e2288 100644 --- a/primrose/crates/candelabra-benchmarker/src/container.rs +++ b/primrose/crates/candelabra-benchmarker/src/container.rs @@ -4,27 +4,27 @@ use log::debug; use primrose_library::traits::Container; use rand::{distributions::Standard, prelude::Distribution, random, thread_rng, Rng}; -use crate::{bench::benchmark_op, RunResults, SingleNResults}; +use crate::{bench::benchmark_op, Observation, Results}; /// Benchmark [`primrose_library::traits::Container`] operations pub trait ContainerExt<E> { /// Benchmark at a single `n`. - fn benchmark_container_at(n: usize) -> SingleNResults; + fn benchmark_container_at(n: usize) -> Results; /// Benchmark `len` at a single `n`. - fn benchmark_container_len(n: usize) -> RunResults; + fn benchmark_container_len(n: usize) -> Observation; /// Benchmark `contains` at a single `n`. - fn benchmark_container_contains(n: usize) -> RunResults; + fn benchmark_container_contains(n: usize) -> Observation; /// Benchmark `insert` at a single `n`. - fn benchmark_container_insert(n: usize) -> RunResults; + fn benchmark_container_insert(n: usize) -> Observation; /// Benchmark `clear` at a single `n`. - fn benchmark_container_clear(n: usize) -> RunResults; + fn benchmark_container_clear(n: usize) -> Observation; /// Benchmark `remove` at a single `n`. - fn benchmark_container_remove(n: usize) -> RunResults; + fn benchmark_container_remove(n: usize) -> Observation; } impl<T, E> ContainerExt<E> for T @@ -33,77 +33,92 @@ where E: Clone, Standard: Distribution<E>, { - fn benchmark_container_at(n: usize) -> SingleNResults { + fn benchmark_container_at(n: usize) -> Results { let mut by_op = HashMap::new(); debug!("Benchmarking {} at n = {}", type_name::<T>(), n); debug!("...len"); - by_op.insert("len".to_string(), Self::benchmark_container_len(n)); + by_op.insert("len".to_string(), vec![Self::benchmark_container_len(n)]); debug!("...contains"); by_op.insert( "contains".to_string(), - Self::benchmark_container_contains(n), + vec![Self::benchmark_container_contains(n)], ); debug!("...insert"); - by_op.insert("insert".to_string(), Self::benchmark_container_insert(n)); + by_op.insert( + "insert".to_string(), + vec![Self::benchmark_container_insert(n)], + ); debug!("...clear"); - by_op.insert("clear".to_string(), Self::benchmark_container_clear(n)); + by_op.insert( + "clear".to_string(), + vec![Self::benchmark_container_clear(n)], + ); debug!("...remove"); - by_op.insert("remove".to_string(), Self::benchmark_container_remove(n)); + by_op.insert( + "remove".to_string(), + vec![Self::benchmark_container_remove(n)], + ); debug!("--- done!"); - SingleNResults { by_op } + Results { by_op } } - fn benchmark_container_contains(n: usize) -> RunResults { - benchmark_op( - || { - // TODO: maybe we should actually just test the worst case? (at the end) - // we also don't actually test misses yet. - - let mut rng = thread_rng(); - let mut c = T::default(); - - // decide where the element that we will search for will be - let pivot = rng.gen_range(0..n); - - // insert the element at pivot, and keep track of what it is - for _ in 0..pivot { - c.insert(random()); - } - let chosen = rng.gen(); - c.insert(chosen.clone()); - for _ in pivot..n { - c.insert(random()); - } - - (c, chosen) - }, - |(c, search)| { - c.contains(search); - }, - |_| (), + fn benchmark_container_contains(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + // TODO: maybe we should actually just test the worst case? (at the end) + // we also don't actually test misses yet. + + let mut rng = thread_rng(); + let mut c = T::default(); + + // decide where the element that we will search for will be + let pivot = rng.gen_range(0..n); + + // insert the element at pivot, and keep track of what it is + for _ in 0..pivot { + c.insert(random()); + } + let chosen = rng.gen(); + c.insert(chosen.clone()); + for _ in pivot..n { + c.insert(random()); + } + + (c, chosen) + }, + |(c, search)| { + c.contains(search); + }, + |_| (), + ), ) } - fn benchmark_container_len(n: usize) -> RunResults { - benchmark_op( - || { - let mut c = T::default(); - for _ in 0..n { - c.insert(random()); - } - c - }, - |c| { - c.len(); - }, - |_| (), + fn benchmark_container_len(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut c = T::default(); + for _ in 0..n { + c.insert(random()); + } + c + }, + |c| { + c.len(); + }, + |_| (), + ), ) } - fn benchmark_container_insert(n: usize) -> RunResults { + fn benchmark_container_insert(n: usize) -> Observation { let setup_closure = || { let mut c = T::default(); for _ in 0..n { @@ -111,17 +126,20 @@ where } c }; - benchmark_op( - setup_closure, - |c| { - // TODO: rng generation could throw off benchmarks - c.insert(random()); - }, - |c| *c = setup_closure(), + ( + n, + benchmark_op( + setup_closure, + |c| { + // TODO: rng generation could throw off benchmarks + c.insert(random()); + }, + |c| *c = setup_closure(), + ), ) } - fn benchmark_container_clear(n: usize) -> RunResults { + fn benchmark_container_clear(n: usize) -> Observation { let setup_closure = || { let mut c = T::default(); for _ in 0..n { @@ -129,40 +147,46 @@ where } c }; - benchmark_op( - setup_closure, - |c| { - c.clear(); - }, - |c| *c = setup_closure(), + ( + n, + benchmark_op( + setup_closure, + |c| { + c.clear(); + }, + |c| *c = setup_closure(), + ), ) } - fn benchmark_container_remove(n: usize) -> RunResults { - benchmark_op( - || { - let mut rng = thread_rng(); - let mut c = T::default(); - - // decide where the element that we will remove will be - let pivot = rng.gen_range(0..n); - - // insert the element at pivot, and keep track of what it is - for _ in 0..pivot { - c.insert(random()); - } - let chosen = rng.gen(); - c.insert(chosen.clone()); - for _ in pivot..n { - c.insert(random()); - } - - (c, chosen) - }, - |(c, chosen)| { - c.remove(chosen.clone()); - }, - |(c, chosen)| c.insert(chosen.clone()), + fn benchmark_container_remove(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut rng = thread_rng(); + let mut c = T::default(); + + // decide where the element that we will remove will be + let pivot = rng.gen_range(0..n); + + // insert the element at pivot, and keep track of what it is + for _ in 0..pivot { + c.insert(random()); + } + let chosen = rng.gen(); + c.insert(chosen.clone()); + for _ in pivot..n { + c.insert(random()); + } + + (c, chosen) + }, + |(c, chosen)| { + c.remove(chosen.clone()); + }, + |(c, chosen)| c.insert(chosen.clone()), + ), ) } } diff --git a/primrose/crates/candelabra-benchmarker/src/indexable.rs b/primrose/crates/candelabra-benchmarker/src/indexable.rs index eadb397..066ad8a 100644 --- a/primrose/crates/candelabra-benchmarker/src/indexable.rs +++ b/primrose/crates/candelabra-benchmarker/src/indexable.rs @@ -4,21 +4,21 @@ use log::debug; use primrose_library::traits::{Container, Indexable}; use rand::{distributions::Standard, prelude::Distribution, random}; -use crate::{benchmark_op, RunResults, SingleNResults}; +use crate::{benchmark_op, BenchmarkResult, Observation, Results}; /// Benchmark [`primrose_library::traits::Indexable`] operations pub trait IndexableExt<E> { /// Benchmark at a single `n`. - fn benchmark_indexable_at(n: usize) -> SingleNResults; + fn benchmark_indexable_at(n: usize) -> Results; /// Benchmark `first` at a single `n`. - fn benchmark_indexable_first(n: usize) -> RunResults; + fn benchmark_indexable_first(n: usize) -> Observation; /// Benchmark `last` at a single `n`. - fn benchmark_indexable_last(n: usize) -> RunResults; + fn benchmark_indexable_last(n: usize) -> Observation; /// Benchmark `nth` at a single `n`. - fn benchmark_indexable_nth(n: usize) -> RunResults; + fn benchmark_indexable_nth(n: usize) -> Observation; } impl<T, E> IndexableExt<E> for T @@ -26,67 +26,79 @@ where T: Container<E> + Indexable<E> + Default, Standard: Distribution<E>, { - fn benchmark_indexable_at(n: usize) -> SingleNResults { + fn benchmark_indexable_at(n: usize) -> Results { let mut by_op = HashMap::new(); debug!("Benchmarking {} at n = {}", type_name::<T>(), n); debug!("...first"); - by_op.insert("first".to_string(), Self::benchmark_indexable_first(n)); + by_op.insert( + "first".to_string(), + vec![Self::benchmark_indexable_first(n)], + ); debug!("...last"); - by_op.insert("last".to_string(), Self::benchmark_indexable_last(n)); + by_op.insert("last".to_string(), vec![Self::benchmark_indexable_last(n)]); debug!("...nth"); - by_op.insert("nth".to_string(), Self::benchmark_indexable_nth(n)); + by_op.insert("nth".to_string(), vec![Self::benchmark_indexable_nth(n)]); debug!("--- done!"); - SingleNResults { by_op } + Results { by_op } } - fn benchmark_indexable_first(n: usize) -> RunResults { - benchmark_op( - || { - let mut c = T::default(); - for _ in 0..n { - c.insert(random()); - } - c - }, - |c| { - c.first(); - }, - |_| (), + fn benchmark_indexable_first(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut c = T::default(); + for _ in 0..n { + c.insert(random()); + } + c + }, + |c| { + c.first(); + }, + |_| (), + ), ) } - fn benchmark_indexable_last(n: usize) -> RunResults { - benchmark_op( - || { - let mut c = T::default(); - for _ in 0..n { - c.insert(random()); - } - c - }, - |c| { - c.last(); - }, - |_| (), + fn benchmark_indexable_last(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut c = T::default(); + for _ in 0..n { + c.insert(random()); + } + c + }, + |c| { + c.last(); + }, + |_| (), + ), ) } - fn benchmark_indexable_nth(n: usize) -> RunResults { - benchmark_op( - || { - let mut c = T::default(); - for _ in 0..n { - c.insert(random()); - } - (c, random::<usize>()) - }, - |(c, fetch)| { - c.nth(*fetch); - }, - |_| (), + fn benchmark_indexable_nth(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut c = T::default(); + for _ in 0..n { + c.insert(random()); + } + (c, random::<usize>()) + }, + |(c, fetch)| { + c.nth(*fetch); + }, + |_| (), + ), ) } } diff --git a/primrose/crates/candelabra-benchmarker/src/lib.rs b/primrose/crates/candelabra-benchmarker/src/lib.rs index daa5706..e6fc69b 100644 --- a/primrose/crates/candelabra-benchmarker/src/lib.rs +++ b/primrose/crates/candelabra-benchmarker/src/lib.rs @@ -27,7 +27,7 @@ impl<T, E> Benchmarker<T, E> { Self( ns, Results { - by_n: HashMap::new(), + by_op: HashMap::new(), }, PhantomData, ) @@ -46,7 +46,7 @@ where /// Run benchmarks for [`primrose_library::traits::Container`] operations. pub fn container(mut self) -> Self { for n in self.0 { - self.1.add(*n, T::benchmark_container_at(*n)); + self.1.merge(T::benchmark_container_at(*n)); } self @@ -59,7 +59,7 @@ where /// Run benchmarks for [`primrose_library::traits::Indexable`] operations. pub fn indexable(mut self) -> Self { for n in self.0 { - self.1.add(*n, T::benchmark_indexable_at(*n)); + self.1.merge(T::benchmark_indexable_at(*n)); } self @@ -73,7 +73,7 @@ where /// Run benchmarks for [`primrose_library::traits::Stack`] operations. pub fn stack(mut self) -> Self { for n in self.0 { - self.1.add(*n, T::benchmark_stack_at(*n)); + self.1.merge(T::benchmark_stack_at(*n)); } self diff --git a/primrose/crates/candelabra-benchmarker/src/results.rs b/primrose/crates/candelabra-benchmarker/src/results.rs index e04ec45..3c0783f 100644 --- a/primrose/crates/candelabra-benchmarker/src/results.rs +++ b/primrose/crates/candelabra-benchmarker/src/results.rs @@ -5,23 +5,19 @@ use serde::{Deserialize, Serialize}; /// Results for a whole suite of benchmarks #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Results { - /// Results for each collection size tested with - pub by_n: HashMap<usize, SingleNResults>, + /// Results for each collection operation + pub by_op: HashMap<OpName, Vec<Observation>>, } /// Name of an operation pub type OpName = String; -/// Results for operations all ran at the same container size -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct SingleNResults { - /// Results for each operation - pub by_op: HashMap<OpName, RunResults>, -} +/// The first key in the tuple is the `n` of the container before the benchmark was taken, and the second the results of the benchmark. +pub type Observation = (usize, BenchmarkResult); /// Results for a single benchmark #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct RunResults { +pub struct BenchmarkResult { /// Number of times the benchmark was run pub times: usize, @@ -36,31 +32,14 @@ pub struct RunResults { } impl Results { - pub fn add(&mut self, n: usize, b: SingleNResults) -> &mut Self { - self.by_n - .entry(n) - .and_modify(|res_a| { - res_a.merge(b.clone()); - }) - .or_insert(b); - - self - } - pub fn merge(&mut self, b: Self) -> &mut Self { - for (n, res_b) in b.by_n { - self.add(n, res_b); - } - - self - } -} - -impl SingleNResults { /// Merge results from `b` into these results. - /// If `b` contains benchmarks for operations we have a result for, results from `b` are preferred. + /// If `b` contains benchmarks for operations we have a result for, the observations are merged. pub fn merge(&mut self, b: Self) -> &mut Self { - for (name, res_b) in b.by_op { - self.by_op.insert(name, res_b); + for (name, mut res_b) in b.by_op { + self.by_op + .entry(name) + .and_modify(|res| res.append(&mut res_b)) + .or_insert(res_b); } self diff --git a/primrose/crates/candelabra-benchmarker/src/stack.rs b/primrose/crates/candelabra-benchmarker/src/stack.rs index 2580ce3..7680548 100644 --- a/primrose/crates/candelabra-benchmarker/src/stack.rs +++ b/primrose/crates/candelabra-benchmarker/src/stack.rs @@ -4,18 +4,18 @@ use log::debug; use primrose_library::traits::Stack; use rand::{distributions::Standard, prelude::Distribution, random}; -use crate::{benchmark_op, RunResults, SingleNResults}; +use crate::{benchmark_op, Observation, Results}; /// Benchmark [`primrose_library::traits::Stack`] operations pub trait StackExt<E> { /// Benchmark at a single `n`. - fn benchmark_stack_at(n: usize) -> SingleNResults; + fn benchmark_stack_at(n: usize) -> Results; /// Benchmark `push` at a single `n`. - fn benchmark_stack_push(n: usize) -> RunResults; + fn benchmark_stack_push(n: usize) -> Observation; /// Benchmark `pop` at a single `n`. - fn benchmark_stack_pop(n: usize) -> RunResults; + fn benchmark_stack_pop(n: usize) -> Observation; } impl<T, E> StackExt<E> for T @@ -23,51 +23,57 @@ where T: Stack<E> + Default, Standard: Distribution<E>, { - fn benchmark_stack_at(n: usize) -> SingleNResults { + fn benchmark_stack_at(n: usize) -> Results { let mut by_op = HashMap::new(); debug!("Benchmarking {} at n = {}", type_name::<T>(), n); debug!("...push"); - by_op.insert("push".to_string(), Self::benchmark_stack_push(n)); + by_op.insert("push".to_string(), vec![Self::benchmark_stack_push(n)]); debug!("...pop"); - by_op.insert("pop".to_string(), Self::benchmark_stack_pop(n)); + by_op.insert("pop".to_string(), vec![Self::benchmark_stack_pop(n)]); debug!("--- done!"); - SingleNResults { by_op } + Results { by_op } } - fn benchmark_stack_push(n: usize) -> RunResults { - benchmark_op( - || { - let mut c = T::default(); - for _ in 0..n { - c.push(random()); - } - c - }, - |s| s.push(random()), - |s| { - s.pop(); - }, + fn benchmark_stack_push(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut c = T::default(); + for _ in 0..n { + c.push(random()); + } + c + }, + |s| s.push(random()), + |s| { + s.pop(); + }, + ), ) } - fn benchmark_stack_pop(n: usize) -> RunResults { - benchmark_op( - || { - let mut c = T::default(); - for _ in 0..n { - c.push(random()); - } - c - }, - |s| { - s.pop(); - }, - |s| { - s.push(random()); - }, + fn benchmark_stack_pop(n: usize) -> Observation { + ( + n, + benchmark_op( + || { + let mut c = T::default(); + for _ in 0..n { + c.push(random()); + } + c + }, + |s| { + s.pop(); + }, + |s| { + s.push(random()); + }, + ), ) } } |