fix(benchmarker): more precise benchmarking without criterion

2 files changed, 20 insertions, 27 deletions

diff --git a/src/crates/candelabra/src/cost/benchmark.rs b/src/crates/candelabra/src/cost/benchmark.rs
index 2d3470a..bafe9bd 100644
--- a/src/crates/candelabra/src/cost/benchmark.rs
+++ b/src/crates/candelabra/src/cost/benchmark.rs
@@ -8,7 +8,6 @@ use std::{
     fs::{copy, create_dir, File},
     io::Write,
     process::Command,
-    time::Duration,
 };
 
 use anyhow::{bail, Context, Result};
@@ -19,6 +18,8 @@ use tempfile::{tempdir, TempDir};
 
 use crate::paths::Paths;
 
+use super::Cost;
+
 /// The name of the element type we use for benchmarking
 pub const ELEM_TYPE: &str = "usize";
 
@@ -44,14 +45,14 @@ pub type Observation = (usize, BenchmarkResult);
 /// Results for a single benchmark
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct BenchmarkResult {
-    /// The minimum time taken
-    pub min: Duration,
+    /// The minimum cost
+    pub min: Cost,
 
-    /// The maximum time taken
-    pub max: Duration,
+    /// The maximum cost
+    pub max: Cost,
 
-    /// The average (mean) time taken
-    pub avg: Duration,
+    /// The average (mean) cost
+    pub avg: Cost,
 }
 
 /// Run benchmarks for the given container type, returning the results.
@@ -144,17 +145,17 @@ pub(crate) fn parse_criterion_output(
         })
 }
 
-fn parse_time_str(quantity: &str, suffix: &str) -> Option<Duration> {
-    Some(Duration::from_secs_f32(
-        f32::from_str(quantity).ok()?
+fn parse_time_str(quantity: &str, suffix: &str) -> Option<Cost> {
+    Some(
+        f64::from_str(quantity).ok()?
             * match suffix {
-                "ms" => 1e-3,
-                "µs" => 1e-6,
-                "ns" => 1e-9,
-                "ps" => 1e-12,
+                "ms" => 1e6,
+                "µs" => 1e3,
+                "ns" => 1.0,
+                "ps" => 1e-3,
                 _ => todo!(),
             },
-    ))
+    )
 }
 
 fn prepare_crate(name: &str, paths: &Paths, lib_spec: &LibSpec) -> Result<TempDir> {
@@ -197,7 +198,7 @@ primrose-library = {{ path = \"{}\" }}
     let implemented_traits = lib_spec.interface_provide_map.keys();
     for tr in implemented_traits {
         benchmark_statements += &format!(
-            "candelabra_benchmarker::benchmark_{}::<{}<{}>, _>(c, &NS);",
+            "candelabra_benchmarker::benchmark_{}::<{}<{}>, _>(&NS);",
             tr.to_lowercase(),
             name,
             ELEM_TYPE,
@@ -213,16 +214,10 @@ primrose-library = {{ path = \"{}\" }}
         .write_all(
             format!(
                 "
-use candelabra_benchmarker::criterion::{{criterion_group, criterion_main, Criterion}};
-
 const NS: &[usize] = &{};
-
-fn run_benches(c: &mut Criterion) {{
+fn main() {{
     {}
 }}
-
-criterion_group!(benches, run_benches);
-criterion_main!(benches);
 ",
                 NS, benchmark_statements
             )
diff --git a/src/crates/candelabra/src/cost/fit.rs b/src/crates/candelabra/src/cost/fit.rs
index b5f737e..dada377 100644
--- a/src/crates/candelabra/src/cost/fit.rs
+++ b/src/crates/candelabra/src/cost/fit.rs
@@ -89,7 +89,7 @@ impl Estimator {
         let [a, b, c, d] = self.coeffs;
         n = (n + self.transform_x.0) * self.transform_x.1;
         let raw = a + b * n + c * n.powi(2) + d * n.powi(3);
-        (raw / self.transform_y.1) - self.transform_y.0
+        ((raw / self.transform_y.1) - self.transform_y.0).max(0.0) // can't be below 0
     }
 
     /// Convert a list of observations to the format we use internally.
@@ -97,9 +97,7 @@ impl Estimator {
         let xs = results.iter().map(|(n, _)| *n as f64).collect::<Vec<_>>();
         let ys = OVector::<f64, Dyn>::from_iterator(
             results.len(),
-            results
-                .iter()
-                .map(|(_, results)| results.avg.as_nanos() as f64),
+            results.iter().map(|(_, results)| results.avg),
         );
 
         (xs, ys)