path: root/analysis/vis.livemd

<!-- livebook:{"app_settings":{"slug":"asdf"}} -->

# Dissertation Visualisations

```elixir
Mix.install([
  {:tucan, "~> 0.3.0"},
  {:kino_vega_lite, "~> 0.1.8"},
  {:json, "~> 1.4"},
  {:explorer, "~> 0.8.0"},
  {:kino_explorer, "~> 0.1.11"},
  {:math, "~> 0.7.0"}
])
```

## Setup

```elixir
# Some common variables
require Explorer.DataFrame
require Explorer.Series
require VegaLite
alias Explorer.DataFrame, as: DF
alias Explorer.Series, as: SE
job_id = "current"
job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
sections_dir = Path.join(job_dir, "sections")
cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"])
criterion_dir = Path.join(job_dir, "criterion")
```

<!-- livebook:{"branch_parent_index":0} -->

## Cost models

We read in the cost models from the JSON output.

```elixir
{:ok, cost_model_files} = File.ls(cm_dir)

cost_model_files =
  cost_model_files
  |> Enum.map(fn fname -> Path.join(cm_dir, fname) |> Path.absname() end)

# Should be one for each library implementation
cost_model_files
```

```elixir
# Find the coefficients, ie the actual cost models
cost_models =
  cost_model_files
  |> Enum.map(fn fname ->
    impl = Path.basename(fname) |> String.replace("_", ":")
    contents = File.read!(fname)
    contents = JSON.decode!(contents)

    contents["model"]["by_op"]
    |> Enum.map(fn {op, %{"coeffs" => coeffs}} ->
      %{
        op: op,
        impl: impl,
        coeffs: coeffs
      }
    end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
# Get the raw data points
cost_model_points =
  cost_model_files
  |> Enum.map(fn fname ->
    impl = Path.basename(fname) |> String.replace("_", ":")
    contents = File.read!(fname)
    contents = JSON.decode!(contents)

    contents["results"]["by_op"]
    |> Enum.flat_map(fn {op, results} ->
      Enum.map(results, fn [n, cost] ->
        %{
          op: op,
          impl: String.split(impl, "::") |> List.last(),
          n: n,
          t: cost
        }
      end)
    end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
# Discard points outside one standard deviation, as we do when fitting
cost_model_points =
  cost_model_points
  |> DF.group_by(["impl", "op", "n"])
  |> DF.mutate(avg: mean(t), dev: standard_deviation(t))
  |> DF.filter(abs(t - avg) < dev)
  |> DF.discard(["avg", "dev"])
  |> DF.mutate(t: cast(t, {:duration, :nanosecond}))
```

We can now plot our graphs. The below module provides most of the code, with cells below it specifying our actual graphs.

```elixir
defmodule CostModel do
  @defaults %{y_domain: nil, ns: 1..60_000//100, draw_points: true}
  @all_impls Enum.sort([
               "SortedVec",
               "SortedVecSet",
               "SortedVecMap",
               "Vec",
               "VecSet",
               "VecMap",
               "BTreeSet",
               "BTreeMap",
               "HashSet",
               "HashMap",
               "LinkedList"
             ])

  #  Make the names in the legends shorter and more readable
  def friendly_impl_name(impl) do
    String.split(impl, "::") |> List.last()
  end

  # Get a dataframe of points lying on the cost model, one point for each of `ns`.
  def points_for(cost_models, ns, impl, op) do
    # Get coefficients
    %{"coeffs" => [coeffs]} =
      DF.filter(cost_models, impl == ^impl and op == ^op)
      |> DF.to_columns()

    Enum.map(ns, fn n ->
      t =
        (coeffs
         |> Enum.take(3)
         |> Enum.with_index()
         |> Enum.map(fn {coeff, idx} -> coeff * n ** idx end)
         |> Enum.sum()) + Enum.at(coeffs, 3) * Math.log2(n)

      %{
        impl: friendly_impl_name(impl),
        op: op,
        n: n,
        t: max(t, 0)
      }
    end)
    |> DF.new()
  end

  # Plot the specified cost model, optionally specifying the x/y domains and omitting points
  def plot(cost_models, cost_model_points, impls, op, opts \\ []) do
    %{y_domain: y_domain, ns: ns, draw_points: draw_points} = Enum.into(opts, @defaults)

    plot =
      Tucan.layers(
        # The actual cost model function
        [
          cost_models
          |> DF.filter(op == ^op)
          |> DF.distinct(["impl"])
          |> DF.to_rows()
          |> Enum.map(fn %{"impl" => impl} -> points_for(cost_models, ns, impl, op) end)
          |> DF.concat_rows()
          |> DF.filter(impl in ^impls)
          |> DF.sort_by(impl)
          |> Tucan.lineplot("n", "t", color_by: "impl", clip: true)
        ] ++
          if(draw_points,
            # The raw points, if necessary
            do: [
              cost_model_points
              |> DF.filter(op == ^op and impl in ^impls)
              |> DF.group_by(["impl", "n"])
              |> DF.sort_by(impl)
              |> DF.mutate(t: cast(t, :f32))
              |> Tucan.scatter(
                "n",
                "t",
                color_by: "impl",
                clip: true
              )
            ],
            else: []
          )
      )

    # Adjust x/y domain and set title, etc
    plot =
      plot
      |> Tucan.Axes.set_y_title("Estimated cost")
      |> Tucan.Axes.set_x_title("Size of container (n)")
      |> Tucan.set_size(500, 250)
      |> Tucan.Legend.set_title(:color, "Implementation")
      |> Tucan.Scale.set_x_domain(ns.first, ns.last)

    case y_domain do
      [lo, hi] -> Tucan.Scale.set_y_domain(plot, lo, hi)
      _ -> plot
    end
  end

  # Plot the cost models for `op` across all implementations, grouped by the 2D array `impl_splits`
  def split_plot(cost_models, cost_model_points, impl_splits, op) do
    Enum.map(impl_splits, &plot(cost_models, cost_model_points, &1, op))
    |> Tucan.vconcat()
    # Ensures we don't share a legend for them all
    |> VegaLite.resolve(:scale, color: :independent)
  end
end
```

Below are our actual graphs, which are displayed and exported to JSON files in the thesis directory.

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
graph =
  CostModel.split_plot(
    cost_models,
    cost_model_points,
    [
      ["Vec", "LinkedList"],
      ["SortedVec", "SortedVecSet", "SortedVecMap", "VecSet", "VecMap"],
      ["BTreeSet", "BTreeMap", "HashSet", "HashMap"]
    ],
    "insert"
  )

VegaLite.Export.save!(graph, "../thesis/assets/insert.json")

graph
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
graph =
  CostModel.plot(
    cost_models,
    cost_model_points,
    ["VecSet", "SortedVecSet", "HashSet", "BTreeSet"],
    "insert",
    ns: 1..3000//10,
    y_domain: [0, 200],
    draw_points: false
  )

VegaLite.Export.save!(graph, "../thesis/assets/insert_small_n.json")

graph
```

```elixir
graph =
  CostModel.split_plot(
    cost_models,
    cost_model_points,
    [
      ["SortedVec", "SortedVecSet", "SortedVecMap"],
      [
        "Vec",
        "LinkedList",
        "VecMap",
        "VecSet"
      ],
      ["BTreeSet", "BTreeMap", "HashSet", "HashMap"]
    ],
    "contains"
  )

VegaLite.Export.save!(graph, "../thesis/assets/contains.json")

graph
```

The below block can be used to inspect the cost models of certain operations and implementations

```elixir
impls = ["SortedVec", "SortedVecSet", "SortedVecMap"]
op = "insert"

CostModel.plot(
  cost_models,
  cost_model_points,
  impls,
  op
)
```

<!-- livebook:{"branch_parent_index":0} -->

## Benchmarks

We read in benchmark data from criterion's JSON output.

```elixir
# Read in the results of every individual criterion benchmark
raw_benchmarks =
  File.ls!(criterion_dir)
  |> Enum.map(fn name ->
    File.ls!(Path.join(criterion_dir, name))
    |> Enum.map(fn p -> %{bench: name, subbench: p} end)
  end)
  |> List.flatten()
  |> Enum.map(fn %{bench: bench, subbench: subbench} ->
    File.ls!(Path.join([criterion_dir, bench, subbench]))
    |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end)
    |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end)
    |> Enum.map(fn dir ->
      raw_results =
        Path.join(dir, "estimates.json")
        |> File.read!()
        |> JSON.decode!()

      %{
        bench_id: bench <> "/" <> subbench,
        proj: String.split(bench, "-") |> hd,
        using:
          Regex.scan(~r/\"(\w*)\", ([^)]*)/, Path.basename(dir))
          |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
        mean: raw_results["mean"]["point_estimate"],
        stderr: raw_results["mean"]["standard_error"]
      }
    end)
  end)
  |> List.flatten()
  |> DF.new()
  |> DF.mutate(
    mean: cast(mean, {:duration, :nanosecond}),
    stderr: cast(stderr, {:duration, :nanosecond})
  )
```

```elixir
# Helper function for making the `using` field look nicer
display_using = fn using ->
  using
  |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} -> ctn <> "=" <> impl end)
  |> Enum.join(", ")
end
```

```elixir
# Aggregate benchmark results by project, since we can only do assignments by project
# Unfortunately we can't group by lists, so we need to do some weird shit.
# This is basically equivalent to:
# benchmarks = raw_benchmarks
# |> DF.group_by(["proj", "using"])
# |> DF.summarise(time: sum(mean))

# Build list of using values to index into
usings =
  raw_benchmarks["using"]
  |> SE.to_list()
  |> Enum.uniq()

benchmarks =
  raw_benchmarks
  # Make a column corresponding to using that isn't a list
  |> DF.put(
    "using_idx",
    raw_benchmarks["using"]
    |> SE.to_list()
    |> Enum.map(fn using -> Enum.find_index(usings, &(&1 == using)) end)
  )
  # Get the total benchmark time for each project and assignment
  |> DF.group_by(["proj", "using_idx"])
  |> DF.summarise(time: sum(cast(mean, :f32)))
  # Convert using_idx back to original using values
  |> DF.to_rows()
  |> Enum.map(fn row = %{"using_idx" => using_idx} ->
    Map.put(row, "using", Enum.at(usings, using_idx))
  end)
  |> DF.new()
  |> DF.select(["proj", "time", "using"])
```

We read our cost estimates from the log output.

```elixir
# Cost estimates by project, ctn, and implementation
projs = SE.distinct(benchmarks["proj"])

cost_estimates =
  SE.transform(projs, fn proj_name ->
    [_, table | _] =
      Path.join(sections_dir, "compare-" <> proj_name)
      |> File.read!()
      |> String.split("& file \\\\\n\\hline\n")

    table
    |> String.split("\n\\end{tabular}")
    |> hd
    |> String.split("\n")
    |> Enum.map(fn x -> String.split(x, " & ") end)
    |> Enum.map(fn [ctn, impl, cost | _] ->
      %{
        proj: proj_name,
        ctn: ctn,
        impl:
          impl
          |> String.replace("\\_", "_"),
        cost:
          if String.contains?(cost, ".") do
            String.to_float(cost)
          else
            String.to_integer(cost)
          end
      }
    end)
  end)
  |> SE.to_list()
  |> List.flatten()
  |> DF.new()
```

```elixir
# Double-check that we have all of the cost estimates for everything mentioned in the assignments
estimate_impls = SE.distinct(cost_estimates["impl"])

true =
  (raw_benchmarks
   |> DF.explode("using")
   |> DF.unnest("using"))["impl"]
  |> SE.distinct()
  |> SE.to_list()
  |> Enum.all?(&SE.any?(SE.equal(estimate_impls, &1)))
```

We then find the estimated cost of every assignment that we benchmarked

```elixir
# Gets the cost of assignment from cost estimates
cost_of_assignment = fn proj, assignment ->
  assignment
  |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} ->
    DF.filter(cost_estimates, proj == ^proj and ctn == ^ctn and impl == ^impl)["cost"][0]
  end)
  |> Enum.sum()
end

cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec::Vec"}])
```

```elixir
# For each benchmarked assignment, estimate the cost.
estimated_costs =
  benchmarks
  |> DF.to_rows_stream()
  |> Enum.map(fn %{"proj" => proj, "using" => using} ->
    %{
      proj: proj,
      using: using,
      estimated_cost: cost_of_assignment.(proj, using)
    }
  end)
  |> DF.new()
```

Now we can compare our benchmark results to our estimated costs.

We first filter out adaptive containers, to later consider them separately.

```elixir
# Don't worry about adaptive containers for now
singular_estimated_costs =
  estimated_costs
  |> DF.to_rows_stream()
  |> Enum.filter(fn %{"using" => using} ->
    Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end)
  end)
  |> DF.new()

singular_benchmarks =
  benchmarks
  |> DF.to_rows_stream()
  |> Enum.filter(fn %{"using" => using} ->
    Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end)
  end)
  |> DF.new()

DF.n_rows(singular_benchmarks)
```

```elixir
# Tools for printing out latex
defmodule Latex do
  def escape_latex(val) do
    if is_number(val) do
      "$" <> to_string(val) <> "$"
    else
      String.replace(to_string(val), ~r/(\\|{|}|_|\^|#|&|\$|%|~)/, "\\\\\\1")
    end
  end

  def table(df) do
    cols = DF.names(df)

    "\\begin{tabular}{|" <>
      String.duplicate("c|", length(cols)) <>
      "}\n" <>
      Enum.join(Enum.map(cols, &escape_latex/1), " & ") <>
      " \\\\\n\\hline\n" <>
      (DF.to_rows(df)
       |> Enum.map(fn row ->
         cols
         |> Enum.map(&escape_latex(row[&1]))
         |> Enum.join(" & ")
       end)
       |> Enum.join(" \\\\\n")) <>
      " \\\\\n\\end{tabular}"
  end
end
```

Compare the fastest and slowest assignments for each project

```elixir
singular_benchmarks
|> DF.group_by("proj")
|> DF.summarise(max: max(time), min: min(time))
|> DF.mutate(spread: round((max - min) * ^(10 ** -6), 2), slowdown: round(max / min - 1, 1))
|> DF.discard(["max", "min"])
|> DF.sort_by(proj)
|> DF.rename(%{
  "proj" => "Project",
  "spread" => "Maximum slowdown (ms)",
  "slowdown" => "Maximum relative slowdown"
})

# |> Latex.table()
# |> IO.puts()
```

Compare the predicted and actual best implementation for each container type

```elixir
selection_comparison =
  singular_benchmarks
  |> DF.explode("using")
  |> DF.unnest("using")
  |> DF.group_by(["proj"])
  |> DF.filter(time == min(time))
  |> DF.join(
    cost_estimates
    |> DF.filter(not contains(impl, "until"))
    |> DF.group_by(["proj", "ctn"])
    |> DF.filter(cost == min(cost))
    |> DF.rename(%{"impl" => "predicted_impl"})
  )
  |> DF.select(["proj", "ctn", "impl", "predicted_impl"])
  |> DF.rename(%{"impl" => "best_impl"})
```

```elixir
selection_comparison
|> DF.put(
  "best_impl",
  SE.transform(selection_comparison["best_impl"], &CostModel.friendly_impl_name/1)
)
|> DF.put(
  "predicted_impl",
  SE.transform(selection_comparison["predicted_impl"], &CostModel.friendly_impl_name/1)
)
|> DF.put(
  "mark",
  SE.not_equal(selection_comparison["best_impl"], selection_comparison["predicted_impl"])
  |> SE.transform(&if &1, do: "*", else: "")
)
|> DF.ungroup()
|> DF.sort_by(proj)
|> DF.rename(%{
  "mark" => " ",
  "proj" => "Project",
  "ctn" => "Container Type",
  "best_impl" => "Best implementation",
  "predicted_impl" => "Predicted best"
})

# |> Latex.table()
# |> IO.puts()
```

We now look at adaptive containers, starting by seeing when they get suggested

```elixir
# Container types where an adaptive container was suggested
adaptive_suggestions =
  estimated_costs
  |> DF.explode("using")
  |> DF.unnest("using")
  |> DF.filter(contains(impl, "until"))
  |> DF.distinct(["proj", "ctn", "impl"])

adaptive_suggestions
# Hacky way to make things look nicer
|> DF.mutate(impl: replace(impl, "std::collections::", ""))
|> DF.mutate(impl: replace(impl, "std::vec::", ""))
|> DF.mutate(impl: replace(impl, "primrose_library::", ""))
|> DF.sort_by(asc: proj, asc: ctn)
|> DF.rename(%{
  "proj" => "Project",
  "ctn" => "Container Type",
  "impl" => "Suggestion"
})

# |> Latex.table()
# |> IO.puts()
```

Get benchmarks for projects we suggested an adaptive container for, and find the benchmark 'size' as a new column

```elixir
adaptive_projs = DF.distinct(adaptive_suggestions, ["proj"])["proj"]
adaptive_estimated_costs = estimated_costs |> DF.filter(proj in ^adaptive_projs)

adaptive_raw_benchmarks =
  raw_benchmarks
  |> DF.filter(proj in ^adaptive_projs)

adaptive_raw_benchmarks =
  adaptive_raw_benchmarks
  |> DF.put(
    "n",
    adaptive_raw_benchmarks["bench_id"]
    |> SE.split("/")
    |> SE.transform(&Enum.at(&1, 1))
  )
  |> DF.put(
    "using",
    adaptive_raw_benchmarks["using"]
    |> SE.transform(display_using)
  )
```

We then summarise the results for each benchmark size, for assignments that either involve an adaptive container or are the best possible assignment

```elixir
format_dur = fn dur ->
  String.split(to_string(dur), " ") |> hd
end

best_usings =
  adaptive_raw_benchmarks
  # get best set of assignments for each project
  |> DF.group_by(["proj", "using"])
  |> DF.filter(not contains(using, "until"))
  |> DF.summarise(total: sum(cast(mean, :f32)))
  |> DF.group_by(["proj"])
  |> DF.filter(total == min(total))
  |> DF.discard("total")
  |> DF.rename(%{"using" => "best_using"})
  # select adaptive container and the best assignment for each project
  |> DF.join(adaptive_raw_benchmarks)
  |> DF.filter(using == best_using or contains(using, "until"))

# summary data point
best_usings =
  best_usings
  |> DF.put("mean", SE.transform(best_usings["mean"], format_dur))
  |> DF.put("stderr", SE.transform(best_usings["stderr"], format_dur))
  |> DF.mutate(value: mean <> " +/- " <> stderr)
  |> DF.select(["proj", "using", "n", "value"])
```

Finally, we print them out per-project for clarity

```elixir
for proj <- SE.distinct(best_usings["proj"]) |> SE.to_enum() do
  best_usings
  |> DF.filter(proj == ^proj)
  |> DF.select(["proj", "using", "n", "value"])
  |> DF.pivot_wider("n", "value")
  |> Latex.table()
  |> IO.puts()
end
```