path: root/analysis/vis.livemd

# Dissertation Visualisations

```elixir
Mix.install([
  {:tucan, "~> 0.3.0"},
  {:kino_vega_lite, "~> 0.1.8"},
  {:json, "~> 1.4"},
  {:explorer, "~> 0.8.0"},
  {:kino_explorer, "~> 0.1.11"}
])
```

## Variables

```elixir
require Explorer.DataFrame
require Explorer.Series
alias Explorer.DataFrame, as: DF
alias Explorer.Series, as: SE
job_id = "1147"
job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
sections_dir = Path.join(job_dir, "sections")
criterion_dir = Path.join(job_dir, "criterion")
```

## Read cost model data

```elixir
{:ok, cost_model_files} = File.ls(sections_dir)

cost_model_files =
  cost_model_files
  |> Enum.filter(fn name -> String.contains?(name, "cost-model") end)
  |> Enum.map(fn fname -> Path.join(sections_dir, fname) |> Path.absname() end)

cost_model_files
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
# Parse cost model information
cost_models =
  cost_model_files
  |> Enum.map(fn fname ->
    {:ok, contents} = File.read(fname)
    [_, table | _] = String.split(contents, "line\n")
    [rows | _] = String.split(table, "\n\\end")
    rows = String.split(rows, "\n")

    [_, impl] = String.split(fname, "cost-model-")
    impl = String.replace(impl, "-", ":")

    rows
    |> Enum.map(fn row ->
      [name, x0, x1, x2, x3, nmrse] = String.split(row, " & ")
      [nmrse | _] = String.split(nmrse)

      %{
        op: name,
        x0: String.to_float(x0),
        x1: String.to_float(x1),
        x2: String.to_float(x2),
        x3: String.to_float(x3),
        nmrse: String.to_float(nmrse)
      }
    end)
    |> Enum.map(fn d -> d |> Map.put(:impl, impl) end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
Tucan.histogram(cost_models, "nmrse")
```

```elixir
cost_models
|> DF.filter(op == "contains")
```

## Cost model exploratory plots

```elixir
startn = 0
endn = 200_000
resolution = 100

points_for = fn impl, op ->
  %{"x0" => [x0], "x1" => [x1], "x2" => [x2], "x3" => [x3]} =
    DF.filter(cost_models, impl == ^impl and op == ^op)
    |> DF.to_columns()

  Enum.map(startn..endn//resolution, fn n ->
    %{
      impl: String.split(impl, "::") |> List.last(),
      op: op,
      n: n,
      t: x0 + n * x1 + n * n * x2 + n * n * n * x3
    }
  end)
  |> DF.new()
end
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
inspect_op = "contains"

cost_models
|> DF.filter(op == ^inspect_op)
|> DF.distinct(["impl"])
|> DF.to_rows()
|> Enum.map(fn %{"impl" => impl} -> points_for.(impl, inspect_op) end)
|> DF.concat_rows()
|> Tucan.lineplot("n", "t", color_by: "impl", clip: true)
|> Tucan.Scale.set_y_domain(0, 200)
```

## Read benchmark data

```elixir
benchmarks =
  File.ls!(criterion_dir)
  |> Enum.map(fn name ->
    File.ls!(Path.join(criterion_dir, name))
    |> Enum.map(fn p -> %{bench: name, subbench: p} end)
  end)
  |> List.flatten()
  |> Enum.map(fn %{bench: bench, subbench: subbench} ->
    File.ls!(Path.join([criterion_dir, bench, subbench]))
    |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end)
    |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end)
    |> Enum.map(fn dir ->
      raw_results =
        Path.join(dir, "estimates.json")
        |> File.read!()
        |> JSON.decode!()

      %{
        bench_id: bench <> "/" <> subbench,
        using:
          Regex.scan(~r/\"(\w*)\", ([\w:]*)/, Path.basename(dir))
          |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
        mean: raw_results["mean"]["point_estimate"] / 10 ** 9
      }
    end)
  end)
  |> List.flatten()
  |> DF.new()
```

```elixir
# Cost estimates by project, ctn, and implementation
projs =
  benchmarks["bench_id"]
  |> SE.split("-")
  |> SE.transform(&hd/1)
  |> SE.split("/")
  |> SE.transform(&hd/1)

benchmarks = DF.put(benchmarks, "proj", projs)

cost_estimates =
  SE.transform(projs |> SE.distinct(), fn proj_name ->
    [_, table | _] =
      Path.join(sections_dir, "compare-" <> proj_name)
      |> File.read!()
      |> String.split("& file \\\\\n\\hline\n")

    table
    |> String.split("\n\\end{tabular}")
    |> hd
    |> String.split("\n")
    |> Enum.map(fn x -> String.split(x, " & ") end)
    |> Enum.map(fn [ctn, impl, cost | _] ->
      %{
        proj: proj_name,
        ctn: ctn,
        impl:
          impl
          |> String.replace("\\_", "_"),
        cost:
          if String.contains?(cost, ".") do
            String.to_float(cost)
          else
            String.to_integer(cost)
          end
      }
    end)
  end)
  |> SE.to_list()
  |> List.flatten()
  |> DF.new()
```

```elixir
# Get cost of assignment from cost estimates
cost_of_assignment = fn proj, assignment ->
  assignment
  |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} ->
    DF.filter(cost_estimates, proj == ^proj and ctn == ^ctn and impl == ^impl)["cost"][0]
  end)
  |> Enum.sum()
end

cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec::Vec"}])
```

```elixir
# Estimate cost for each benchmarked assignment
estimated_costs =
  benchmarks
  |> DF.to_rows_stream()
  # TODO
  |> Enum.filter(fn %{"proj" => proj} -> String.contains?(proj, "example") end)
  |> Enum.map(fn %{"bench_id" => bench_id, "proj" => proj, "using" => using} ->
    %{
      bench_id: bench_id,
      using: using,
      estimated_cost: cost_of_assignment.(proj, using)
    }
  end)
  |> DF.new()
```

```elixir
# Compare each assignments position in the estimates to its position in the results
sorted_estimates =
  estimated_costs
  |> DF.group_by(["bench_id"])
  |> DF.sort_by(estimated_cost)

sorted_results =
  benchmarks
  |> DF.group_by(["bench_id"])
  |> DF.sort_by(mean)

position_comparison =
  sorted_estimates
  |> DF.to_rows_stream()
  |> Enum.map(fn %{"bench_id" => bench_id, "using" => using} ->
    %{
      bench_id: bench_id,
      using: using,
      pos_estimate:
        DF.filter(sorted_estimates, bench_id == ^bench_id)["using"]
        |> SE.to_list()
        |> Enum.find_index(fn u -> u == using end),
      pos_results:
        DF.filter(sorted_results, bench_id == ^bench_id)["using"]
        |> SE.to_list()
        |> Enum.find_index(fn u -> u == using end)
    }
  end)
  |> DF.new()
```

```elixir
position_comparison
|> DF.filter(pos_estimate != pos_results)
|> DF.collect()
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
# Difference in execution time between worst and best selection
Tucan.bar(
  benchmarks
  |> DF.group_by("bench_id")
  |> DF.summarise(range: max(mean) - min(mean)),
  "bench_id",
  "range",
  orient: :horizontal,
  clip: true
)
|> Tucan.Scale.set_x_domain(0, 5)
```