app_settings":{"slug":"asdf"}} --> # Dissertation Visualisations ```elixir Mix.install([ {:tucan, "~> 0.3.0"}, {:kino_vega_lite, "~> 0.1.8"}, {:json, "~> 1.4"}, {:explorer, "~> 0.8.0"}, {:kino_explorer, "~> 0.1.11"}, {:math, "~> 0.7.0"} ]) ``` ## Setup ```elixir # Some common variables require Explorer.DataFrame require Explorer.Series require VegaLite alias Explorer.DataFrame, as: DF alias Explorer.Series, as: SE job_id = "current" job_dir = Path.expand(~c"./" ++ job_id) |> Path.absname() sections_dir = Path.join(job_dir, "sections") cm_dir = Path.join([job_dir, "candelabra", "benchmark_results"]) criterion_dir = Path.join(job_dir, "criterion") ``` ## Cost models We read in the cost models from the JSON output. ```elixir {:ok, cost_model_files} = File.ls(cm_dir) cost_model_files = cost_model_files |> Enum.map(fn fname -> Path.join(cm_dir, fname) |> Path.absname() end) # Should be one for each library implementation cost_model_files ``` ```elixir # Find the coefficients, ie the actual cost models cost_models = cost_model_files |> Enum.map(fn fname -> impl = Path.basename(fname) |> String.replace("_", ":") contents = File.read!(fname) contents = JSON.decode!(contents) contents["model"]["by_op"] |> Enum.map(fn {op, %{"coeffs" => coeffs}} -> %{ op: op, impl: impl, coeffs: coeffs } end) |> DF.new() end) |> DF.concat_rows() ``` ```elixir # Get the raw data points cost_model_points = cost_model_files |> Enum.map(fn fname -> impl = Path.basename(fname) |> String.replace("_", ":") contents = File.read!(fname) contents = JSON.decode!(contents) contents["results"]["by_op"] |> Enum.flat_map(fn {op, results} -> Enum.map(results, fn [n, cost] -> %{ op: op, impl: String.split(impl, "::") |> List.last(), n: n, t: cost } end) end) |> DF.new() end) |> DF.concat_rows() ``` ```elixir # Discard points outside one standard deviation, as we do when fitting cost_model_points = cost_model_points |> DF.group_by(["impl", "op", "n"]) |> DF.mutate(avg: mean(t), dev: standard_deviation(t)) |> DF.filter(abs(t - avg) < dev) |> DF.discard(["avg", "dev"]) |> DF.mutate(t: cast(t, {:duration, :nanosecond})) ``` We can now plot our graphs. The below module provides most of the code, with cells below it specifying our actual graphs. ```elixir defmodule CostModel do @defaults %{y_domain: nil, ns: 1..60_000//100, draw_points: true} @all_impls Enum.sort([ "SortedVec", "SortedVecSet", "SortedVecMap", "Vec", "VecSet", "VecMap", "BTreeSet", "BTreeMap", "HashSet", "HashMap", "LinkedList" ]) # Make the names in the legends shorter and more readable def friendly_impl_name(impl) do String.split(impl, "::") |> List.last() end # Get a dataframe of points lying on the cost model, one point for each of `ns`. def points_for(cost_models, ns, impl, op) do # Get coefficients %{"coeffs" => [coeffs]} = DF.filter(cost_models, impl == ^impl and op == ^op) |> DF.to_columns() Enum.map(ns, fn n -> t = (coeffs |> Enum.take(3) |> Enum.with_index() |> Enum.map(fn {coeff, idx} -> coeff * n ** idx end) |> Enum.sum()) + Enum.at(coeffs, 3) * Math.log2(n) %{ impl: friendly_impl_name(impl), op: op, n: n, t: max(t, 0) } end) |> DF.new() end # Plot the specified cost model, optionally specifying the x/y domains and omitting points def plot(cost_models, cost_model_points, impls, op, opts \\ []) do %{y_domain: y_domain, ns: ns, draw_points: draw_points} = Enum.into(opts, @defaults) plot = Tucan.layers( # The actual cost model function [ cost_models |> DF.filter(op == ^op) |> DF.distinct(["impl"]) |> DF.to_rows() |> Enum.map(fn %{"impl" => impl} -> points_for(cost_models, ns, impl, op) end) |> DF.concat_rows() |> DF.filter(impl in ^impls) |> DF.sort_by(impl) |> Tucan.lineplot("n", "t", color_by: "impl", clip: true) ] ++ if(draw_points, # The raw points, if necessary do: [ cost_model_points |> DF.filter(op == ^op and impl in ^impls) |> DF.group_by(["impl", "n"]) |> DF.sort_by(impl) |> DF.mutate(t: cast(t, :f32)) |> Tucan.scatter( "n", "t", color_by: "impl", clip: true ) ], else: [] ) ) # Adjust x/y domain and set title, etc plot = plot |> Tucan.Axes.set_y_title("Estimated cost") |> Tucan.Axes.set_x_title("Size of container (n)") |> Tucan.set_size(500, 250) |> Tucan.Legend.set_title(:color, "Implementation") |> Tucan.Scale.set_x_domain(ns.first, ns.last) case y_domain do [lo, hi] -> Tucan.Scale.set_y_domain(plot, lo, hi) _ -> plot end end # Plot the cost models for `op` across all implementations, grouped by the 2D array `impl_splits` def split_plot(cost_models, cost_model_points, impl_splits, op) do Enum.map(impl_splits, &plot(cost_models, cost_model_points, &1, op)) |> Tucan.vconcat() # Ensures we don't share a legend for them all |> VegaLite.resolve(:scale, color: :independent) end end ``` Below are our actual graphs, which are displayed and exported to JSON files in the thesis directory. ```elixir graph = CostModel.split_plot( cost_models, cost_model_points, [ ["Vec", "LinkedList"], ["SortedVec", "SortedVecSet", "SortedVecMap", "VecSet", "VecMap"], ["BTreeSet", "BTreeMap", "HashSet", "HashMap"] ], "insert" ) VegaLite.Export.save!(graph, "../thesis/assets/insert.json") graph ``` ```elixir graph = CostModel.plot( cost_models, cost_model_points, ["VecSet", "SortedVecSet", "HashSet", "BTreeSet"], "insert", ns: 1..3000//10, y_domain: [0, 200], draw_points: false ) VegaLite.Export.save!(graph, "../thesis/assets/insert_small_n.json") graph ``` ```elixir graph = CostModel.split_plot( cost_models, cost_model_points, [ ["SortedVec", "SortedVecSet", "SortedVecMap"], [ "Vec", "LinkedList", "VecMap", "VecSet" ], ["BTreeSet", "BTreeMap", "HashSet", "HashMap"] ], "contains" ) VegaLite.Export.save!(graph, "../thesis/assets/contains.json") graph ``` The below block can be used to inspect the cost models of certain operations and implementations ```elixir impls = ["SortedVec", "SortedVecSet", "SortedVecMap"] op = "insert" CostModel.plot( cost_models, cost_model_points, impls, op ) ``` ## Benchmarks We read in benchmark data from criterion's JSON output. ```elixir # Read in the results of every individual criterion benchmark raw_benchmarks = File.ls!(criterion_dir) |> Enum.map(fn name -> File.ls!(Path.join(criterion_dir, name)) |> Enum.map(fn p -> %{bench: name, subbench: p} end) end) |> List.flatten() |> Enum.map(fn %{bench: bench, subbench: subbench} -> File.ls!(Path.join([criterion_dir, bench, subbench])) |> Enum.filter(fn x -> String.contains?(x, "Mapping2D") end) |> Enum.map(fn x -> Path.join([criterion_dir, bench, subbench, x]) end) |> Enum.map(fn dir -> raw_results = Path.join(dir, "estimates.json") |> File.read!() |> JSON.decode!() %{ bench_id: bench <> "/" <> subbench, proj: String.split(bench, "-") |> hd, using: Regex.scan(~r/\"(\w*)\", ([^)]*)/, Path.basename(dir)) |> Enum.map(fn [_, ctn, impl] -> %{ctn: ctn, impl: impl} end), mean: raw_results["mean"]["point_estimate"], stderr: raw_results["mean"]["standard_error"] } end) end) |> List.flatten() |> DF.new() |> DF.mutate( mean: cast(mean, {:duration, :nanosecond}), stderr: cast(stderr, {:duration, :nanosecond}) ) ``` ```elixir # Helper function for making the `using` field look nicer display_using = fn using -> using |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} -> ctn <> "=" <> impl end) |> Enum.join(", ") end ``` ```elixir # Aggregate benchmark results by project, since we can only do assignments by project # Unfortunately we can't group by lists, so we need to do some weird shit. # This is basically equivalent to: # benchmarks = raw_benchmarks # |> DF.group_by(["proj", "using"]) # |> DF.summarise(time: sum(mean)) # Build list of using values to index into usings = raw_benchmarks["using"] |> SE.to_list() |> Enum.uniq() benchmarks = raw_benchmarks # Make a column corresponding to using that isn't a list |> DF.put( "using_idx", raw_benchmarks["using"] |> SE.to_list() |> Enum.map(fn using -> Enum.find_index(usings, &(&1 == using)) end) ) # Get the total benchmark time for each project and assignment |> DF.group_by(["proj", "using_idx"]) |> DF.summarise(time: sum(cast(mean, :f32))) # Convert using_idx back to original using values |> DF.to_rows() |> Enum.map(fn row = %{"using_idx" => using_idx} -> Map.put(row, "using", Enum.at(usings, using_idx)) end) |> DF.new() |> DF.select(["proj", "time", "using"]) ``` We read our cost estimates from the log output. ```elixir # Cost estimates by project, ctn, and implementation projs = SE.distinct(benchmarks["proj"]) cost_estimates = SE.transform(projs, fn proj_name -> [_, table | _] = Path.join(sections_dir, "compare-" <> proj_name) |> File.read!() |> String.split("& file \\\\\n\\hline\n") table |> String.split("\n\\end{tabular}") |> hd |> String.split("\n") |> Enum.map(fn x -> String.split(x, " & ") end) |> Enum.map(fn [ctn, impl, cost | _] -> %{ proj: proj_name, ctn: ctn, impl: impl |> String.replace("\\_", "_"), cost: if String.contains?(cost, ".") do String.to_float(cost) else String.to_integer(cost) end } end) end) |> SE.to_list() |> List.flatten() |> DF.new() ``` ```elixir # Double-check that we have all of the cost estimates for everything mentioned in the assignments estimate_impls = SE.distinct(cost_estimates["impl"]) true = (raw_benchmarks |> DF.explode("using") |> DF.unnest("using"))["impl"] |> SE.distinct() |> SE.to_list() |> Enum.all?(&SE.any?(SE.equal(estimate_impls, &1))) ``` We then find the estimated cost of every assignment that we benchmarked ```elixir # Gets the cost of assignment from cost estimates cost_of_assignment = fn proj, assignment -> assignment |> Enum.map(fn %{"ctn" => ctn, "impl" => impl} -> DF.filter(cost_estimates, proj == ^proj and ctn == ^ctn and impl == ^impl)["cost"][0] end) |> Enum.sum() end cost_of_assignment.("example_stack", [%{"ctn" => "StackCon", "impl" => "std::vec::Vec"}]) ``` ```elixir # For each benchmarked assignment, estimate the cost. estimated_costs = benchmarks |> DF.to_rows_stream() |> Enum.map(fn %{"proj" => proj, "using" => using} -> %{ proj: proj, using: using, estimated_cost: cost_of_assignment.(proj, using) } end) |> DF.new() ``` Now we can compare our benchmark results to our estimated costs. We first filter out adaptive containers, to later consider them separately. ```elixir # Don't worry about adaptive containers for now singular_estimated_costs = estimated_costs |> DF.to_rows_stream() |> Enum.filter(fn %{"using" => using} -> Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end) end) |> DF.new() singular_benchmarks = benchmarks |> DF.to_rows_stream() |> Enum.filter(fn %{"using" => using} -> Enum.all?(using, fn %{"impl" => impl} -> !String.contains?(impl, "until") end) end) |> DF.new() DF.n_rows(singular_benchmarks) ``` ```elixir # Tools for printing out latex defmodule Latex do def escape_latex(val) do if is_number(val) do "$" <> to_string(val) <> "$" else String.replace(to_string(val), ~r/(\\|{|}|_|\^|#|&|\$|%|~)/, "\\\\\\1") end end def table(df) do cols = DF.names(df) "\\begin{tabular}{|" <> String.duplicate("c|", length(cols)) <> "}\n" <> Enum.join(Enum.map(cols, &escape_latex/1), " & ") <> " \\\\\n\\hline\n" <> (DF.to_rows(df) |> Enum.map(fn row -> cols |> Enum.map(&escape_latex(row[&1])) |> Enum.join(" & ") end) |> Enum.join(" \\\\\n")) <> " \\\\\n\\end{tabular}" end end ``` Compare the fastest and slowest assignments for each project ```elixir singular_benchmarks |> DF.group_by("proj") |> DF.summarise(max: max(time), min: min(time)) |> DF.mutate(spread: round((max - min) * ^(10 ** -6), 2), slowdown: round(max / min - 1, 1)) |> DF.discard(["max", "min"]) |> DF.sort_by(proj) |> DF.rename(%{ "proj" => "Project", "spread" => "Maximum slowdown (ms)", "slowdown" => "Maximum relative slowdown" }) # |> Latex.table() # |> IO.puts() ``` Compare the predicted and actual best implementation for each container type ```elixir selection_comparison = singular_benchmarks |> DF.explode("using") |> DF.unnest("using") |> DF.group_by(["proj"]) |> DF.filter(time == min(time)) |> DF.join( cost_estimates |> DF.filter(not contains(impl, "until")) |> DF.group_by(["proj", "ctn"]) |> DF.filter(cost == min(cost)) |> DF.rename(%{"impl" => "predicted_impl"}) ) |> DF.select(["proj", "ctn", "impl", "predicted_impl"]) |> DF.rename(%{"impl" => "best_impl"}) ``` ```elixir selection_comparison |> DF.put( "best_impl", SE.transform(selection_comparison["best_impl"], &CostModel.friendly_impl_name/1) ) |> DF.put( "predicted_impl", SE.transform(selection_comparison["predicted_impl"], &CostModel.friendly_impl_name/1) ) |> DF.put( "mark", SE.not_equal(selection_comparison["best_impl"], selection_comparison["predicted_impl"]) |> SE.transform(&if &1, do: "*", else: "") ) |> DF.ungroup() |> DF.sort_by(proj) |> DF.rename(%{ "mark" => " ", "proj" => "Project", "ctn" => "Container Type", "best_impl" => "Best implementation", "predicted_impl" => "Predicted best" }) # |> Latex.table() # |> IO.puts() ``` We now look at adaptive containers, starting by seeing when they get suggested ```elixir # Container types where an adaptive container was suggested adaptive_suggestions = estimated_costs |> DF.explode("using") |> DF.unnest("using") |> DF.filter(contains(impl, "until")) |> DF.distinct(["proj", "ctn", "impl"]) adaptive_suggestions # Hacky way to make things look nicer |> DF.mutate(impl: replace(impl, "std::collections::", "")) |> DF.mutate(impl: replace(impl, "std::vec::", "")) |> DF.mutate(impl: replace(impl, "primrose_library::", "")) |> DF.sort_by(asc: proj, asc: ctn) |> DF.rename(%{ "proj" => "Project", "ctn" => "Container Type", "impl" => "Suggestion" }) # |> Latex.table() # |> IO.puts() ``` Get benchmarks for projects we suggested an adaptive container for, and find the benchmark 'size' as a new column ```elixir adaptive_projs = DF.distinct(adaptive_suggestions, ["proj"])["proj"] adaptive_estimated_costs = estimated_costs |> DF.filter(proj in ^adaptive_projs) adaptive_raw_benchmarks = raw_benchmarks |> DF.filter(proj in ^adaptive_projs) adaptive_raw_benchmarks = adaptive_raw_benchmarks |> DF.put( "n", adaptive_raw_benchmarks["bench_id"] |> SE.split("/") |> SE.transform(&Enum.at(&1, 1)) ) |> DF.put( "using", adaptive_raw_benchmarks["using"] |> SE.transform(display_using) ) ``` We then summarise the results for each benchmark size, for assignments that either involve an adaptive container or are the best possible assignment ```elixir format_dur = fn dur -> String.split(to_string(dur), " ") |> hd end best_usings = adaptive_raw_benchmarks # get best set of assignments for each project |> DF.group_by(["proj", "using"]) |> DF.filter(not contains(using, "until")) |> DF.summarise(total: sum(cast(mean, :f32))) |> DF.group_by(["proj"]) |> DF.filter(total == min(total)) |> DF.discard("total") |> DF.rename(%{"using" => "best_using"}) # select adaptive container and the best assignment for each project |> DF.join(adaptive_raw_benchmarks) |> DF.filter(using == best_using or contains(using, "until")) # summary data point best_usings = best_usings |> DF.put("mean", SE.transform(best_usings["mean"], format_dur)) |> DF.put("stderr", SE.transform(best_usings["stderr"], format_dur)) |> DF.mutate(value: mean <> " +/- " <> stderr) |> DF.select(["proj", "using", "n", "value"]) ``` Finally, we print them out per-project for clarity ```elixir for proj <- SE.distinct(best_usings["proj"]) |> SE.to_enum() do best_usings |> DF.filter(proj == ^proj) |> DF.select(["proj", "using", "n", "value"]) |> DF.pivot_wider("n", "value") |> Latex.table() |> IO.puts() end ```