some setup for analysing results and stuff

1 files changed, 128 insertions, 0 deletions

diff --git a/analysis/vis.livemd b/analysis/vis.livemd
new file mode 100644
index 0000000..13b1ab4
--- /dev/null
+++ b/analysis/vis.livemd
@@ -0,0 +1,128 @@
+# Cost model visualisations
+
+```elixir
+Mix.install([
+  {:tucan, "~> 0.3.0"},
+  {:kino_vega_lite, "~> 0.1.8"}
+])
+```
+
+## Reading / preprocessing data
+
+```elixir
+job_id = "1138"
+job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
+sections_dir = Path.join(job_dir, "sections")
+
+{:ok, cost_model_files} = File.ls(sections_dir)
+
+cost_model_files =
+  cost_model_files
+  |> Enum.filter(fn name -> String.contains?(name, "cost-model") end)
+  # |> Enum.filter(fn name -> String.contains?(name, "vec--Vec") end)
+  |> Enum.map(fn fname -> Path.join(sections_dir, fname) |> Path.absname() end)
+
+cost_model_files
+```
+
+```elixir
+defmodule CostModel do
+  defstruct impl: nil, ops: []
+end
+
+defmodule OpCostModel do
+  defstruct op: nil, x0: nil, x1: nil, x2: nil, x3: nil, nmrse: nil
+end
+
+defmodule Parse do
+  def cost_model_row(row) do
+    [name, x0, x1, x2, x3, nmrse] = String.split(row, " & ")
+    [nmrse | _] = String.split(nmrse)
+
+    %OpCostModel{
+      op: name,
+      x0: String.to_float(x0),
+      x1: String.to_float(x1),
+      x2: String.to_float(x2),
+      x3: String.to_float(x3),
+      nmrse: String.to_float(nmrse)
+    }
+  end
+
+  def cost_model_output(fname) do
+    {:ok, contents} = File.read(fname)
+    [_, table | _] = String.split(contents, "line\n")
+    [rows | _] = String.split(table, "\n\\end")
+    rows = String.split(rows, "\n")
+
+    [_, impl] = String.split(fname, "cost-model-")
+    impl = String.replace(impl, "-", ":")
+
+    %CostModel{
+      impl: impl,
+      ops: rows |> Enum.map(&cost_model_row/1)
+    }
+  end
+end
+```
+
+<!-- livebook:{"reevaluate_automatically":true} -->
+
+```elixir
+cost_models = cost_model_files |> Enum.map(&Parse.cost_model_output/1)
+```
+
+## Exploratory Plots
+
+```elixir
+defmodule PlotCostModel do
+  @startn 0
+  @endn 1000
+  @resolution 10
+  def gen_ts(ns, %OpCostModel{x0: x0, x1: x1, x2: x2, x3: x3}) do
+    Enum.map(ns, fn n -> %{n: n, t: x0 + n * x1 + n * n * x2 + n * n * n * x3} end)
+  end
+
+  def points_op(op) do
+    ns = @startn..@endn//@resolution
+
+    gen_ts(ns, op)
+    |> Enum.map(fn data -> Map.put(data, :name, op.op) end)
+  end
+
+  def points_model(model) do
+    model.ops
+    |> Enum.map(fn op ->
+      points_op(op)
+      |> Enum.map(fn point -> Map.put(point, :impl, model.impl) end)
+    end)
+    |> List.flatten()
+  end
+
+  def points(models) do
+    models
+    |> Enum.map(&points_model/1)
+    |> List.flatten()
+  end
+
+  def plot_op(op) do
+    points = points_op(op)
+    Tucan.lineplot(points, "n", "t", title: op.op)
+  end
+
+  def plot_all_ops(%CostModel{impl: impl, ops: ops}) do
+    ops
+    |> Enum.map(&plot_op/1)
+    |> Tucan.concat(columns: 3)
+    |> Tucan.set_title(impl)
+  end
+end
+```
+
+<!-- livebook:{"reevaluate_automatically":true} -->
+
+```elixir
+cost_models
+|> Enum.map(&PlotCostModel.plot_all_ops/1)
+|> Tucan.vconcat()
+```