path: root/analysis/vis.livemd

<!-- livebook:{"app_settings":{"slug":"asdf"}} -->

# Dissertation Visualisations

```elixir
Mix.install([
  {:tucan, "~> 0.3.0"},
  {:kino_vega_lite, "~> 0.1.8"},
  {:json, "~> 1.4"},
  {:explorer, "~> 0.8.0"},
  {:kino_explorer, "~> 0.1.11"},
  {:math, "~> 0.7.0"}
])
```

## Variables

```elixir
require Explorer.DataFrame
require Explorer.Series
alias Explorer.DataFrame, as: DF
alias Explorer.Series, as: SE
job_id = "current"
job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname()
sections_dir = Path.join(job_dir, "sections")
cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"])
criterion_dir = Path.join(job_dir, "criterion")
```

## Read cost model data

```elixir
{:ok, cost_model_files} = File.ls(cm_dir)

cost_model_files =
  cost_model_files
  |> Enum.map(fn fname -> Path.join(cm_dir, fname) |> Path.absname() end)

cost_model_files
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
# Parse cost model information
cost_models =
  cost_model_files
  |> Enum.map(fn fname ->
    impl = Path.basename(fname) |> String.replace("_", ":")
    contents = File.read!(fname)
    contents = JSON.decode!(contents)

    contents["model"]["by_op"]
    |> Enum.map(fn {op, %{"coeffs" => coeffs}} ->
      %{
        op: op,
        impl: impl,
        coeffs: coeffs
      }
    end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
# Parse cost model information
cost_model_points =
  cost_model_files
  |> Enum.map(fn fname ->
    impl = Path.basename(fname) |> String.replace("_", ":")
    contents = File.read!(fname)
    contents = JSON.decode!(contents)

    contents["results"]["by_op"]
    |> Enum.flat_map(fn {op, results} ->
      Enum.map(results, fn [n, cost] ->
        %{
          op: op,
          impl: String.split(impl, "::") |> List.last(),
          n: n,
          t: cost
        }
      end)
    end)
    |> DF.new()
  end)
  |> DF.concat_rows()
```

```elixir
cost_models
|> DF.filter(op == "contains")
```

## Cost model exploratory plots

```elixir
startn = 1
endn = 5000
resolution = 1

points_for = fn impl, op ->
  %{"coeffs" => [coeffs]} =
    DF.filter(cost_models, impl == ^impl and op == ^op)
    |> DF.to_columns()

  Enum.map(startn..endn//resolution, fn n ->
    t =
      (coeffs
       |> Enum.take(3)
       |> Enum.with_index()
       |> Enum.map(fn {coeff, idx} -> coeff * n ** idx end)
       |> Enum.sum()) + Enum.at(coeffs, 3) * Math.log2(n)

    %{
      impl: String.split(impl, "::") |> List.last(),
      op: op,
      n: n,
      t: max(t, 0)
    }
  end)
  |> DF.new()
end
```

<!-- livebook:{"reevaluate_automatically":true} -->

```elixir
set_impls = ["BTreeSet", "HashSet", "VecSet", "SortedVecSet"]
mapping_impls = ["HashMap", "BTreeMap", "VecMap", "SortedVecMap"]
other_impls = ["Vec", "LinkedList", "SortedVec"]

inspect_op = "remove"
impls = ["Vec", "LinkedList"]

Tucan.layers([
  cost_models
  |> DF.filter(op == ^inspect_op)
  |> DF.distinct(["impl"])
  |> DF.to_rows()
  |> Enum.map(fn %{"impl" => impl} -> points_for.(impl, inspect_op) end)
  |> DF.concat_rows()
  |> DF.filter(impl in ^impls)
  |> Tucan.lineplot("n", "t", color_by: "impl", clip: true)
  # Tucan.scatter(
  #   cost_model_points
  #   |> DF.filter(op == ^inspect_op and impl in ^impls)
  #   |> DF.group_by(["impl", "n"]),
  #   # |> DF.summarise(t: mean(t))
  #   "n",
  #   "t",
  #   color_by: "impl",
  #   clip: true
  # )
])
|> Tucan.Axes.set_y_title("Estimated cost")
|> Tucan.Axes.set_x_title("Size of container (n)")
|> Tucan.Scale.set_x_domain(startn, endn)
# |> Tucan.Scale.set_y_domain(0, 10_000)
|> Tucan.set_size(500, 250)
|> Tucan.Legend.set_title(:color, "Implementation")
|> Tucan.Legend.set_orientation(:color, "bottom")
```

```elixir
cost_model_points
|> DF.filter(impl == "LinkedList" and op == "remove")
|> DF.group_by("n")
|> DF.summarise(t: mean(t))
```

## Read benchmark data

```elixir
# Read in the results of every individual criterion benchmark
raw_benchmarks =
  File.ls!(criterion_dir)
  |> Enum.map(fn name ->
    File.ls!(Path.join(criterion_dir, name))
    |> Enum.map(fn p -> %{bench: name, subbench: p} end)
  end)
  |> List.flatten()
  |> Enum.map(fn %{bench: bench, subbench: subbench} ->
    File.ls!(Path.join([criterion_dir, bench, subbench]))
    |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end)
    |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end)
    |> Enum.map(fn dir ->
      raw_results =
        Path.join(dir, "estimates.json")
        |> File.read!()
        |> JSON.decode!()

      %{
        bench_id: bench <> "/" <> subbench,
        proj: String.split(bench, "-") |> hd,
        using:
          Regex.scan(~r/\"(\w*)\", ([^)]*)/, Path.basename(dir))
          |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end),
        mean: raw_results["mean"]["point_estimate"] / 10 ** 9
      }
    end)
  end)
  |> List.flatten()
  |> DF.new()
```

```elixir
# Aggregate benchmark results by project, since we can only do assignments by project
# Unfortunately we can't group by lists, so we need to do some weird shit.
# This is basically equivalent to:
# benchmarks = raw_benchmarks
# |> DF.group_by(["proj", "using"])
# |> DF.summarise(time: sum(mean))

# Build list of using values to index into
usings =
  raw_benchmarks["using"]
  |> SE.to_list()
  |> Enum.uniq()

benchmarks =
  raw_benchmarks
  # Make a column corresponding to using that isn't a list
  |> DF.put(
    "using_idx",
    raw_benchmarks["using"]
    |> SE.to_list()
    |> Enum.map(fn using -> Enum.find_index(usings, &(&1 == using)) end)
  )
  # Get the total benchmark time for each project and assignment
  |> DF.group_by(["proj", "using_idx"])
  |> DF.summarise(time: sum(mean))
  # Convert using_idx back to original using values
  |> DF.to_rows()
  |> Enum.map(fn row = %{"using_idx" => using_idx} ->
    Map.put(row, "using", Enum.at(usings, using_idx))
  end)
  |> DF.new()
  |> DF.select(["proj", "time", "using"])
```

## Read cost estimate data

```elixir
# Cost estimates by project, ctn, and implementation
projs = SE.distinct(benchmarks["proj"])

cost_estimates =
  SE.transform(projs, fn proj_name ->
    [_, table | _] =
      Path.join(sections_dir, "compare-" <> proj_name)
      |> File.read!()
      |> String.split("& file \\\\\n\\hline\n")

    table
    |> String.split("\n\\end{tabular}")
    |> hd
    |> String.split("\n")
    |> Enum.map(fn x -> String.split(x, " & ") end)
    |> Enum.map(fn [ctn, impl, cost | _] ->
      %{
        proj: proj_name,
        ctn: ctn,
        impl:
          impl
          |> String.replace("\\_", "_"),
        cost:
          if String.contains?(cost, ".") do
            String.to_float(cost)
          else
            String.to_integer(cost)
          end
      }
    end)
  end)
  |> SE.to_list()
  |> List.flatten()
  |> DF.new()
```

```elixir
# Double-check that we have all of the cost estimates for everything mentioned in the assignments
estimate_impls = SE.distinct(cost_estimates["impl"])

true =
  (benchmarks
   |> DF.explode("using")
   |> DF.unnest("using"))["impl"]
  |> SE.distinct()
  |> SE.to_list()
  |> Enum.all?(fn impl -> SE.equal(estimate_impls, impl) |> SE.any?() end)
```

```elixir
# Gets the cost of assignment from cost estimates
cost_of_assignment = fn proj, assignment ->
  assignment
  |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} ->
    DF.filter(cost_estimates, proj == ^proj and ctn == ^ctn and impl == ^impl)["cost"][0]
  end)
  |> Enum.sum()
end

cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec::Vec"}])
```

```elixir
# For each benchmarked assignment, estimate the cost.
estimated_costs =
  benchmarks
  |> DF.to_rows_stream()
  |> Enum.map(fn %{"proj" => proj, "using" => using} ->
    %{
      proj: proj,
      using: using,
      estimated_cost: cost_of_assignment.(proj, using)
    }
  end)
  |> DF.new()
```

## Estimates vs results (ignoring adaptive containers)

```elixir
# Don't worry about adaptive containers for now
singular_estimated_costs =
  estimated_costs
  |> DF.to_rows_stream()
  |> Enum.filter(fn %{"using" => using} ->
    Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end)
  end)
  |> DF.new()

singular_benchmarks =
  benchmarks
  |> DF.to_rows_stream()
  |> Enum.filter(fn %{"using" => using} ->
    Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end)
  end)
  |> DF.new()

DF.n_rows(singular_benchmarks)
```

```elixir
# Compare each assignments position in the estimates to its position in the results
sorted_singular_estimates =
  singular_estimated_costs
  |> DF.group_by(["proj"])
  |> DF.sort_by(estimated_cost)

sorted_singular_results =
  singular_benchmarks
  |> DF.group_by(["proj"])
  |> DF.sort_by(time)

singular_position_comparison =
  sorted_singular_estimates
  |> DF.to_rows_stream()
  |> Enum.map(fn %{"proj" => proj, "using" => using} ->
    %{
      proj: proj,
      using: using,
      pos_estimate:
        DF.filter(sorted_singular_estimates, proj == ^proj)["using"]
        |> SE.to_list()
        |> Enum.find_index(fn u -> u == using end),
      pos_results:
        DF.filter(sorted_singular_results, proj == ^proj)["using"]
        |> SE.to_list()
        |> Enum.find_index(fn u -> u == using end)
    }
  end)
  |> DF.new()
```

```elixir
# Everywhere we predicted wrong.
singular_position_comparison
|> DF.filter(pos_estimate == 0 and pos_estimate != pos_results)
|> DF.collect()
```

```elixir
singular_estimated_costs
|> DF.filter(proj == "prime_sieve")
|> DF.sort_by(estimated_cost)
```

```elixir
singular_benchmarks
|> DF.filter(proj == "prime_sieve")
|> DF.sort_by(time)
```