path: root/analysis/vis.livemd

# Dissertation Visualisations

```elixir
Mix.install([
  {:tucan, "~> 0.3.0"},
  {:kino_vega_lite, "~> 0.1.8"},
  {:json, "~> 1.4"},
  {:explorer, "~> 0.8.0"},
  {:kino_explorer, "~> 0.1.11"},
  {:math, "~> 0.7.0"}
])
```

## Variables

```elixir
require Explorer.DataFrame
require Explorer.Series
alias Explorer.DataFrame, as: DF
alias Explorer.Series, as: SE
job_id = "current"
job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
sections_dir = Path.join(job_dir, "sections")
cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"])
criterion_dir = Path.join(job_dir, "criterion")
```

## Read cost model data

```elixir
{:ok, cost_model_files} = File.ls(cm_dir)

cost_model_files =
  cost_model_files
  |> Enum.map(fn fname -> Path.join(cm_dir, fname) |> Path.absname() end)

cost_model_files
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
# Parse cost model information
cost_models =
  cost_model_files
  |> Enum.map(fn fname ->
    impl = Path.basename(fname) |> String.replace("_", ":")
    contents = File.read!(fname)
    contents = JSON.decode!(contents)

    contents["model"]["by_op"]
    |> Enum.map(fn {op, %{"coeffs" => coeffs}} ->
      %{
        op: op,
        impl: impl,
        coeffs: coeffs
      }
    end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
# Parse cost model information
cost_model_points =
  cost_model_files
  |> Enum.map(fn fname ->
    impl = Path.basename(fname) |> String.replace("_", ":")
    contents = File.read!(fname)
    contents = JSON.decode!(contents)

    contents["results"]["by_op"]
    |> Enum.flat_map(fn {op, results} ->
      Enum.map(results, fn [n, cost] ->
        %{
          op: op,
          impl: String.split(impl, "::") |> List.last(),
          n: n,
          t: cost
        }
      end)
    end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
cost_models
|> DF.filter(op == "contains")
```

## Cost model exploratory plots

```elixir
startn = 100
endn = 5000
resolution = 50

points_for = fn impl, op ->
  %{"coeffs" => [coeffs]} =
    DF.filter(cost_models, impl == ^impl and op == ^op)
    |> DF.to_columns()

  Enum.map(startn..endn//resolution, fn n ->
    t =
      (coeffs
       |> Enum.take(3)
       |> Enum.with_index()
       |> Enum.map(fn {coeff, idx} -> coeff * n ** idx end)
       |> Enum.sum()) + Enum.at(coeffs, 3) * Math.log2(n)

    %{
      impl: String.split(impl, "::") |> List.last(),
      op: op,
      n: n,
      t: t
    }
  end)
  |> DF.new()
end
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
# inspect_op = "insert"
# impls = ["BTreeSet", "EagerSortedVec", "HashSet"]

inspect_op = "pop"
impls = ["Vec", "LinkedList"]

Tucan.layers([
  cost_models
  |> DF.filter(op == ^inspect_op)
  |> DF.distinct(["impl"])
  |> DF.to_rows()
  |> Enum.map(fn %{"impl" => impl} -> points_for.(impl, inspect_op) end)
  |> DF.concat_rows()
  |> DF.filter(impl in ^impls)
  |> Tucan.lineplot("n", "t", color_by: "impl", clip: true),
  # |> Tucan.Scale.set_y_domain(0, 200)
  Tucan.scatter(
    cost_model_points
    |> DF.filter(op == ^inspect_op and impl in ^impls)
    |> DF.group_by(["impl", "n"]),
    # |> DF.summarise(t: mean(t)),
    "n",
    "t",
    color_by: "impl",
    clip: true
  )
])
|> Tucan.Scale.set_x_domain(startn, endn)
|> Tucan.Scale.set_y_domain(0, 200)
|> Tucan.set_size(500, 500)
```

## Read benchmark data

```elixir
raw_benchmarks =
  File.ls!(criterion_dir)
  |> Enum.map(fn name ->
    File.ls!(Path.join(criterion_dir, name))
    |> Enum.map(fn p -> %{bench: name, subbench: p} end)
  end)
  |> List.flatten()
  |> Enum.map(fn %{bench: bench, subbench: subbench} ->
    File.ls!(Path.join([criterion_dir, bench, subbench]))
    |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end)
    |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end)
    |> Enum.map(fn dir ->
      raw_results =
        Path.join(dir, "estimates.json")
        |> File.read!()
        |> JSON.decode!()

      %{
        bench_id: bench <> "/" <> subbench,
        proj: String.split(bench, "-") |> hd,
        using:
          Regex.scan(~r/\"(\w*)\", ([^)]*)/, Path.basename(dir))
          |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
        mean: raw_results["mean"]["point_estimate"] / 10 ** 9
      }
    end)
  end)
  |> List.flatten()
  |> DF.new()
```

```elixir
# Aggregate benchmark results by project, since we can only do assignments by project

# Can't group by lists, so we need to do this weird shit

# Build list of using values to index into
usings =
  raw_benchmarks["using"]
  |> SE.to_list()
  |> Enum.uniq()

benchmarks =
  raw_benchmarks
  # Make a column corresponding to using that isn't a list
  |> DF.put(
    "using_idx",
    raw_benchmarks["using"]
    |> SE.to_list()
    |> Enum.map(fn using -> Enum.find_index(usings, &(&1 == using)) end)
  )
  # Get the total benchmark time for each project and assignment
  |> DF.group_by(["proj", "using_idx"])
  |> DF.summarise(time: sum(mean))
  # Convert using_idx back to original using values
  |> DF.to_rows()
  |> Enum.map(fn row = %{"using_idx" => using_idx} ->
    Map.put(row, "using", Enum.at(usings, using_idx))
  end)
  |> DF.new()
  |> DF.select(["proj", "time", "using"])
```

```elixir
# Cost estimates by project, ctn, and implementation
projs = SE.distinct(benchmarks["proj"])

cost_estimates =
  SE.transform(projs, fn proj_name ->
    [_, table | _] =
      Path.join(sections_dir, "compare-" <> proj_name)
      |> File.read!()
      |> String.split("& file \\\\\n\\hline\n")

    table
    |> String.split("\n\\end{tabular}")
    |> hd
    |> String.split("\n")
    |> Enum.map(fn x -> String.split(x, " & ") end)
    |> Enum.map(fn [ctn, impl, cost | _] ->
      %{
        proj: proj_name,
        ctn: ctn,
        impl:
          impl
          |> String.replace("\\_", "_"),
        cost:
          if String.contains?(cost, ".") do
            String.to_float(cost)
          else
            String.to_integer(cost)
          end
      }
    end)
  end)
  |> SE.to_list()
  |> List.flatten()
  |> DF.new()
```

```elixir
# Get cost of assignment from cost estimates
cost_of_assignment = fn proj, assignment ->
  assignment
  |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} ->
    DF.filter(cost_estimates, proj == ^proj and ctn == ^ctn and impl == ^impl)["cost"][0]
  end)
  |> Enum.sum()
end

cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec::Vec"}])
```

```elixir
# Estimate cost for each benchmarked assignment
estimated_costs =
  benchmarks
  |> DF.to_rows_stream()
  |> Enum.map(fn %{"proj" => proj, "using" => using} ->
    %{
      proj: proj,
      using: using,
      estimated_cost: cost_of_assignment.(proj, using)
    }
  end)
  |> DF.new()
```

```elixir
# Compare each assignments position in the estimates to its position in the results
sorted_estimates =
  estimated_costs
  |> DF.group_by(["proj"])
  |> DF.sort_by(estimated_cost)

sorted_results =
  benchmarks
  |> DF.group_by(["proj"])
  |> DF.sort_by(time)

position_comparison =
  sorted_estimates
  |> DF.to_rows_stream()
  |> Enum.map(fn %{"proj" => proj, "using" => using} ->
    %{
      proj: proj,
      using: using,
      pos_estimate:
        DF.filter(sorted_estimates, proj == ^proj)["using"]
        |> SE.to_list()
        |> Enum.find_index(fn u -> u == using end),
      pos_results:
        DF.filter(sorted_results, proj == ^proj)["using"]
        |> SE.to_list()
        |> Enum.find_index(fn u -> u == using end)
    }
  end)
  |> DF.new()
```

```elixir
position_comparison
|> DF.filter(pos_estimate != pos_results)
|> DF.collect()
```

```elixir
position_comparison
|> DF.filter(pos_estimate == 0)
|> DF.select(["proj", "using"])
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
# Difference in execution time between worst and best selection
Tucan.bar(
  benchmarks
  |> DF.group_by("bench_id")
  |> DF.summarise(range: max(mean) - min(mean)),
  "bench_id",
  "range",
  orient: :horizontal,
  clip: true
)
|> Tucan.Scale.set_x_domain(0, 5)
```