plot cost model observations also

1 files changed, 59 insertions, 29 deletions

diff --git a/analysis/vis.livemd b/analysis/vis.livemd
index bd33860..b89d841 100644
--- a/analysis/vis.livemd
+++ b/analysis/vis.livemd
@@ -20,18 +20,18 @@ alias Explorer.Series, as: SE
 job_id = "1174"
 job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
 sections_dir = Path.join(job_dir, "sections")
+cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"])
 criterion_dir = Path.join(job_dir, "criterion")
 ```
 
 ## Read cost model data
 
 ```elixir
-{:ok, cost_model_files} = File.ls(sections_dir)
+{:ok, cost_model_files} = File.ls(cm_dir)
 
 cost_model_files =
   cost_model_files
-  |> Enum.filter(fn name -> String.contains?(name, "cost-model") end)
-  |> Enum.map(fn fname -> Path.join(sections_dir, fname) |> Path.absname() end)
+  |> Enum.map(fn fname -> Path.join(cm_dir, fname) |> Path.absname() end)
 
 cost_model_files
 ```
@@ -43,34 +43,46 @@ cost_model_files
 cost_models =
   cost_model_files
   |> Enum.map(fn fname ->
-    {:ok, contents} = File.read(fname)
-    [_, table | _] = String.split(contents, "line\n")
-    [rows | _] = String.split(table, "\n\\end")
-    rows = String.split(rows, "\n")
-
-    [_, impl] = String.split(fname, "cost-model-")
-    impl = String.replace(impl, "-", ":")
-
-    rows
-    |> Enum.map(fn row ->
-      [row | _] = String.split(row, " \\\\")
-      [name, nmrse | coeffs] = String.split(row, " & ")
-      [nmrse | _] = String.split(nmrse)
+    impl = Path.basename(fname) |> String.replace("_", ":")
+    contents = File.read!(fname)
+    contents = JSON.decode!(contents)
 
+    contents["model"]["by_op"]
+    |> Enum.map(fn {op, %{"coeffs" => coeffs}} ->
       %{
-        op: name,
-        coeffs: coeffs |> Enum.map(&String.to_float/1),
-        nmrse: String.to_float(nmrse)
+        op: op,
+        impl: impl,
+        coeffs: coeffs
       }
     end)
-    |> Enum.map(fn d -> d |> Map.put(:impl, impl) end)
     |> DF.new()
   end)
   |> DF.concat_rows()
 ```
 
 ```elixir
-Tucan.histogram(cost_models, "nmrse")
+# Parse cost model information
+cost_model_points =
+  cost_model_files
+  |> Enum.map(fn fname ->
+    impl = Path.basename(fname) |> String.replace("_", ":")
+    contents = File.read!(fname)
+    contents = JSON.decode!(contents)
+
+    contents["results"]["by_op"]
+    |> Enum.flat_map(fn {op, results} ->
+      Enum.map(results, fn [n, cost] ->
+        %{
+          op: op,
+          impl: String.split(impl, "::") |> List.last(),
+          n: n,
+          t: cost
+        }
+      end)
+    end)
+    |> DF.new()
+  end)
+  |> DF.concat_rows()
 ```
 
 ```elixir
@@ -112,14 +124,32 @@ end
 ```elixir
 inspect_op = "insert"
 
-cost_models
-|> DF.filter(op == ^inspect_op)
-|> DF.distinct(["impl"])
-|> DF.to_rows()
-|> Enum.map(fn %{"impl" => impl} -> points_for.(impl, inspect_op) end)
-|> DF.concat_rows()
-|> Tucan.lineplot("n", "t", color_by: "impl", clip: true)
-|> Tucan.Scale.set_y_domain(0, 200)
+Tucan.layers([
+  cost_models
+  |> DF.filter(op == ^inspect_op)
+  |> DF.distinct(["impl"])
+  |> DF.to_rows()
+  |> Enum.map(fn %{"impl" => impl} -> points_for.(impl, inspect_op) end)
+  |> DF.concat_rows()
+  |> Tucan.lineplot("n", "t", color_by: "impl", clip: true)
+  |> Tucan.Scale.set_y_domain(0, 200),
+  Tucan.scatter(
+    cost_model_points
+    |> DF.filter(op == ^inspect_op)
+    |> DF.group_by(["impl", "n"])
+    |> DF.summarise(t: mean(t)),
+    "n",
+    "t",
+    color_by: "impl",
+    clip: true
+  )
+])
+```
+
+```elixir
+cost_model_points
+|> DF.group_by(["impl", "n"])
+|> DF.summarise(t: mean(t))
 ```
 
 ## Read benchmark data