analyse some benchmark results

1 files changed, 67 insertions, 5 deletions

diff --git a/analysis/vis.livemd b/analysis/vis.livemd
index 13b1ab4..f3f1aac 100644
--- a/analysis/vis.livemd
+++ b/analysis/vis.livemd
@@ -1,25 +1,35 @@
-# Cost model visualisations
+# Dissertation Visualisations
 
 ```elixir
 Mix.install([
   {:tucan, "~> 0.3.0"},
-  {:kino_vega_lite, "~> 0.1.8"}
+  {:kino_vega_lite, "~> 0.1.8"},
+  {:json, "~> 1.4"},
+  {:explorer, "~> 0.8.0"},
+  {:kino_explorer, "~> 0.1.11"}
 ])
 ```
 
-## Reading / preprocessing data
+## Variables
 
 ```elixir
+require Explorer.DataFrame
+alias Explorer.DataFrame, as: DF
 job_id = "1138"
 job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
 sections_dir = Path.join(job_dir, "sections")
+criterion_dir = Path.join(job_dir, "criterion")
+```
+
+## Read cost model data
 
+```elixir
 {:ok, cost_model_files} = File.ls(sections_dir)
 
 cost_model_files =
   cost_model_files
   |> Enum.filter(fn name -> String.contains?(name, "cost-model") end)
-  # |> Enum.filter(fn name -> String.contains?(name, "vec--Vec") end)
+  |> Enum.filter(fn name -> String.contains?(name, "vec--Vec") end)
   |> Enum.map(fn fname -> Path.join(sections_dir, fname) |> Path.absname() end)
 
 cost_model_files
@@ -72,7 +82,7 @@ end
 cost_models = cost_model_files |> Enum.map(&Parse.cost_model_output/1)
 ```
 
-## Exploratory Plots
+## Cost model exploratory plots
 
 ```elixir
 defmodule PlotCostModel do
@@ -126,3 +136,55 @@ cost_models
 |> Enum.map(&PlotCostModel.plot_all_ops/1)
 |> Tucan.vconcat()
 ```
+
+## Read benchmark data
+
+```elixir
+benchmarks =
+  File.ls!(criterion_dir)
+  |> Enum.map(fn name ->
+    File.ls!(Path.join(criterion_dir, name))
+    |> Enum.map(fn p -> %{bench: name, subbench: p} end)
+  end)
+  |> List.flatten()
+  |> Enum.map(fn %{bench: bench, subbench: subbench} ->
+    File.ls!(Path.join([criterion_dir, bench, subbench]))
+    |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end)
+    |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end)
+    |> Enum.map(fn dir ->
+      raw_results =
+        Path.join(dir, "estimates.json")
+        |> File.read!()
+        |> JSON.decode!()
+
+      %{
+        bench: bench,
+        subbench: subbench,
+        bench_id: bench <> "/" <> subbench,
+        using:
+          Regex.scan(~r/\"(\w*)\", ([\w:]*)/, Path.basename(dir))
+          |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
+        dir: dir,
+        lower_bound: raw_results["mean"]["confidence_interval"]["lower_bound"] / 10 ** 9,
+        mean: raw_results["mean"]["point_estimate"] / 10 ** 9,
+        upper_bound: raw_results["mean"]["confidence_interval"]["upper_bound"] / 10 ** 9
+      }
+    end)
+  end)
+  |> List.flatten()
+  |> DF.new()
+```
+
+```elixir
+# Difference in execution time between worst and best selection
+Tucan.bar(
+  benchmarks
+  |> DF.group_by("bench_id")
+  |> DF.summarise(range: max(mean) - min(mean)),
+  "bench_id",
+  "range",
+  orient: :horizontal,
+  clip: true
+)
+|> Tucan.Scale.set_x_domain(0, 10)
+```