fix(benchmarker): some cleanup and convenience

3 files changed, 25 insertions, 23 deletions

diff --git a/src/crates/benchmarker/src/container.rs b/src/crates/benchmarker/src/container.rs
index 4e0a111..783d46d 100644
--- a/src/crates/benchmarker/src/container.rs
+++ b/src/crates/benchmarker/src/container.rs
@@ -27,14 +27,6 @@ where
         );
     });
 
-    bench_with_ns::<T, E>(c, ns, "len", |b, container| {
-        b.iter_batched_ref(
-            || black_box(container.clone()),
-            |c| black_box(c.len()),
-            BatchSize::LargeInput,
-        );
-    });
-
     bench_with_ns::<T, E>(c, ns, "insert", |b, container| {
         b.iter_batched_ref(
             || container.clone(),
diff --git a/src/crates/benchmarker/src/lib.rs b/src/crates/benchmarker/src/lib.rs
index bdd5bcd..f3868f1 100644
--- a/src/crates/benchmarker/src/lib.rs
+++ b/src/crates/benchmarker/src/lib.rs
@@ -2,6 +2,8 @@ mod container;
 mod indexable;
 mod stack;
 
+use std::time::Duration;
+
 pub use criterion;
 
 pub use container::*;
@@ -22,6 +24,9 @@ pub fn bench_with_ns<T, E>(
     Standard: Distribution<E>,
 {
     let mut g = c.benchmark_group(name);
+    // HACK: speeding this up makes testing a lot easier. to be seen if this is still as reliable though
+    g.measurement_time(Duration::from_secs(1));
+    g.warm_up_time(Duration::from_millis(500));
     for n in ns {
         let mut container = T::default();
         for _ in 0..*n {
diff --git a/src/crates/candelabra/src/cost/benchmark.rs b/src/crates/candelabra/src/cost/benchmark.rs
index 1de8c07..9c7266d 100644
--- a/src/crates/candelabra/src/cost/benchmark.rs
+++ b/src/crates/candelabra/src/cost/benchmark.rs
@@ -1,5 +1,7 @@
 //! Benchmarking of container types
 
+use std::io::{self, Read};
+use std::process::Stdio;
 use std::str::FromStr;
 use std::{
     collections::HashMap,
@@ -10,7 +12,7 @@ use std::{
 };
 
 use anyhow::{bail, Context, Result};
-use log::{debug, log_enabled, Level};
+use log::debug;
 use primrose::{LibSpec, LibSpecs};
 use serde::{Deserialize, Serialize};
 use tempfile::{tempdir, TempDir};
@@ -21,7 +23,7 @@ use crate::paths::Paths;
 pub const ELEM_TYPE: &str = "usize";
 
 /// String representation of the array of N values we use for benchmarking
-pub const NS: &str = "[8, 256, 1024, 65536]";
+pub const NS: &str = "[8, 256, 1024]";
 
 /// Results for a whole suite of benchmarks
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -61,29 +63,32 @@ pub fn run_benchmarks(name: &str, paths: &Paths, lib_specs: &LibSpecs) -> Result
 
     // Build and run
     debug!("Building and running benchmarks for {}", name);
-    let run_output = Command::new("cargo")
+    let mut child = Command::new("cargo")
         .args(["run", "--release", "--", "--bench"])
         .current_dir(crate_.path())
         .env("CARGO_TARGET_DIR", &paths.target_dir) // Share target directory
-        .output()
+        .stdout(Stdio::piped())
+        .spawn()
         .context("Error running build command")?;
 
-    if !run_output.status.success() {
-        bail!("Error result from benchmark. Output: {:?}", run_output);
+    // tee the output to stdout and a vector
+    let mut stdout = child.stdout.take().unwrap();
+    let mut output = Vec::new();
+    let mut buf = vec![0; 100];
+    while let Ok(None) = child.try_wait() {
+        let n = stdout.read(&mut buf)?;
+        let read = &buf[0..n];
+        io::stdout().write_all(read)?;
+        output.extend(read);
     }
 
-    if log_enabled!(Level::Debug) {
-        if let Ok(stdout) = String::from_utf8(run_output.stdout.clone()) {
-            debug!("stdout: {:?}", stdout);
-        }
-        if let Ok(stderr) = String::from_utf8(run_output.stderr.clone()) {
-            debug!("stderr: {:?}", stderr);
-        }
+    let run_output = child.try_wait().unwrap().unwrap();
+    if !run_output.success() {
+        bail!("Error result from benchmark. Output: {:?}", run_output);
     }
 
     // Deserialise benchmark results
-    let output =
-        String::from_utf8(run_output.stdout).context("Error interpreting output as UTF-8")?;
+    let output = String::from_utf8(output).context("Error interpreting output as UTF-8")?;
     let measurements = output
         .lines()
         .flat_map(|l| {