From 68942eb703238c7288dea129cdb96dcf213696b0 Mon Sep 17 00:00:00 2001
From: Aria Shrimpton <me@aria.rip>
Date: Thu, 7 Mar 2024 11:52:41 +0000
Subject: try to do analysis

---
 analysis/vis.livemd | 50 ++++++++++++++++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 20 deletions(-)

(limited to 'analysis/vis.livemd')

diff --git a/analysis/vis.livemd b/analysis/vis.livemd
index b89d841..f4b2ca6 100644
--- a/analysis/vis.livemd
+++ b/analysis/vis.livemd
@@ -17,7 +17,7 @@ require Explorer.DataFrame
 require Explorer.Series
 alias Explorer.DataFrame, as: DF
 alias Explorer.Series, as: SE
-job_id = "1174"
+job_id = "1177"
 job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
 sections_dir = Path.join(job_dir, "sections")
 cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"])
@@ -94,7 +94,7 @@ cost_models
 
 ```elixir
 startn = 0
-endn = 80_000
+endn = 60_000
 resolution = 100
 
 points_for = fn impl, op ->
@@ -146,16 +146,10 @@ Tucan.layers([
 ])
 ```
 
-```elixir
-cost_model_points
-|> DF.group_by(["impl", "n"])
-|> DF.summarise(t: mean(t))
-```
-
 ## Read benchmark data
 
 ```elixir
-benchmarks =
+raw_benchmarks =
   File.ls!(criterion_dir)
   |> Enum.map(fn name ->
     File.ls!(Path.join(criterion_dir, name))
@@ -174,6 +168,7 @@ benchmarks =
 
       %{
         bench_id: bench <> "/" <> subbench,
+        proj: String.split(bench, "-") |> hd,
         using:
           Regex.scan(~r/\"(\w*)\", ([\w:]*)/, Path.basename(dir))
           |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
@@ -186,18 +181,29 @@ benchmarks =
 ```
 
 ```elixir
-# Cost estimates by project, ctn, and implementation
-projs =
-  benchmarks["bench_id"]
-  |> SE.split("-")
-  |> SE.transform(&hd/1)
-  |> SE.split("/")
-  |> SE.transform(&hd/1)
+# Aggregate benchmark results by project, since we can only do assignments by project
+uniq_proj_using =
+  DF.select(raw_benchmarks, ["proj", "using"])
+  |> DF.to_rows()
+  |> Enum.uniq()
+  |> DF.new()
 
-benchmarks = DF.put(benchmarks, "proj", projs)
+uniq_proj_using
+|> DF.mutate(vals: DF.filter(^raw_benchmarks, proj == proj and using == using))
+
+# |> Enum.map(fn %{"proj" => proj, "using" => using} ->
+#   DF.filter(raw_benchmarks, proj == ^proj and using == ^using)
+#   # |> DF.summarise()
+# end)
+# |> DF.concat_rows()
+```
+
+```elixir
+# Cost estimates by project, ctn, and implementation
+projs = SE.distinct(benchmarks["proj"])
 
 cost_estimates =
-  SE.transform(projs |> SE.distinct(), fn proj_name ->
+  SE.transform(projs, fn proj_name ->
     [_, table | _] =
       Path.join(sections_dir, "compare-" <> proj_name)
       |> File.read!()
@@ -247,8 +253,6 @@ cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec
 estimated_costs =
   benchmarks
   |> DF.to_rows_stream()
-  # TODO
-  |> Enum.filter(fn %{"proj" => proj} -> String.contains?(proj, "example") end)
   |> Enum.map(fn %{"bench_id" => bench_id, "proj" => proj, "using" => using} ->
     %{
       bench_id: bench_id,
@@ -297,6 +301,12 @@ position_comparison
 |> DF.collect()
 ```
 
+```elixir
+position_comparison
+|> DF.filter(pos_estimate == 0)
+|> DF.select(["bench_id", "using"])
+```
+
 <!-- livebook:{"reevaluate_automatically":true} -->
 
 ```elixir
-- 
cgit v1.2.3