Add benchmarking scripts

These scripts were originally written by horrors, and have since been
hacked up a lot by jade. We are putting them up as a CL since it is
better to have checked in benchmarking scripts than to not have
benchmarking scripts.

cc: https://git.lix.systems/lix-project/lix/issues/23

Co-authored-by: eldritch horrors <pennae@lix.systems>
Change-Id: I95c2f9d24753ac468944c5781deec9508fd5cb8c

5 files changed, 503 insertions, 0 deletions

diff --git a/bench/.gitignore b/bench/.gitignore
new file mode 100644
index 000000000..8115aa6f2
--- /dev/null
+++ b/bench/.gitignore
@@ -0,0 +1,3 @@
+bench-*.json
+bench-*.md
+nixpkgs
diff --git a/bench/README.md b/bench/README.md
new file mode 100644
index 000000000..4aefcd7a8
--- /dev/null
+++ b/bench/README.md
@@ -0,0 +1,91 @@
+# Benchmarking scripts for Lix
+
+These are very much WIP, and have a few clumsy assumptions that we would
+somewhat rather be fixed, but we have committed them to let others be able to
+do benchmarking in the mean time.
+
+## Benchmarking procedure
+
+Build some Lixes you want to compare, by whichever means you wish.
+
+Get a computer that is not busy and *strongly preferably* is bare-metal or at
+least not a cloud VM (e.g. go make coffee when running benchmarks).
+
+From the root of a Lix checkout, run `./bench/bench.sh resultlink-one
+resultlink-two`, where `resultlink-one` and `resultlink-two` are the result
+links from the builds you want to test (they can be any directory with bin/nix
+in it, however).
+
+To get the summary again, run `./bench/summarize.jq bench/bench-*.json`.
+
+## Example results
+
+(vim tip: `:r !bench/summarize.jq bench/bench-*.json` to dump it directly into
+your editor)
+
+```
+result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6
+c6fe19ccb59cac54b5b3f25e160870 hello
+  mean:     15.993s ± 0.081s
+            user: 13.321s | system: 1.865s
+  median:   15.994s
+  range:    15.829s ... 16.096s
+  relative: 1
+result/bin/nix --extra-experimental-features 'nix-command flakes' search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19cc
+b59cac54b5b3f25e160870 hello
+  mean:     15.897s ± 0.075s
+            user: 13.248s | system: 1.843s
+  median:   15.88s
+  range:    15.807s ... 16.047s
+  relative: 0.994
+
+---
+
+result/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix
+  mean:     0.4s ± 0.024s
+            user: 0.335s | system: 0.046s
+  median:   0.386s
+  range:    0.379s ... 0.43s
+  relative: 1
+
+result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix
+  mean:     0.404s ± 0.024s
+            user: 0.338s | system: 0.046s
+  median:   0.386s
+  range:    0.384s ... 0.436s
+  relative: 1.008
+
+---
+
+result-asserts/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
+  mean:     5.838s ± 0.023s
+            user: 5.083s | system: 0.464s
+  median:   5.845s
+  range:    5.799s ... 5.867s
+  relative: 1
+
+result/bin/nix --extra-experimental-features 'nix-command flakes' eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
+  mean:     5.788s ± 0.044s
+            user: 5.056s | system: 0.439s
+  median:   5.79s
+  range:    5.715s ... 5.876s
+  relative: 0.991
+
+---
+
+GC_INITIAL_HEAP_SIZE=10g result-asserts/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
+  mean:     4.147s ± 0.021s
+            user: 3.457s | system: 0.487s
+  median:   4.147s
+  range:    4.123s ... 4.195s
+  relative: 1
+
+GC_INITIAL_HEAP_SIZE=10g result/bin/nix eval --extra-experimental-features 'nix-command flakes' --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'
+  mean:     4.149s ± 0.027s
+            user: 3.483s | system: 0.456s
+  median:   4.142s
+  range:    4.126s ... 4.215s
+  relative: 1
+
+---
+```
diff --git a/bench/bench.sh b/bench/bench.sh
new file mode 100755
index 000000000..70acd4640
--- /dev/null
+++ b/bench/bench.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+shopt -s inherit_errexit
+
+scriptdir=$(cd "$(dirname -- "$0")" ; pwd -P)
+cd "$scriptdir/.."
+
+if [[ $# -lt 2 ]]; then
+    # FIXME(jade): it is a reasonable use case to want to run a benchmark run
+    # on just one build. However, since we are using hyperfine in comparison
+    # mode, we would have to combine the JSON ourselves to support that, which
+    # would probably be better done by writing a benchmarking script in
+    # not-bash.
+    echo "Fewer than two result dirs given, nothing to compare!" >&2
+    echo "Pass some directories (with names indicating which alternative they are) with bin/nix in them" >&2
+    echo "Usage: ./bench/bench.sh result-1 result-2 [result-3...]" >&2
+    exit 1
+fi
+
+_exit=""
+trap "$_exit" EXIT
+
+# XXX: yes this is very silly. flakes~!!
+nix build --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs
+
+export NIX_REMOTE="$(mktemp -d)"
+_exit='rm -rfv "$NIX_REMOTE"; $_exit'
+export NIX_PATH="nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix"
+
+builds=("$@")
+
+flake_args="--extra-experimental-features 'nix-command flakes'"
+
+hyperfineArgs=(
+    --parameter-list BUILD "$(IFS=,; echo "${builds[*]}")"
+    --warmup 2 --runs 10
+)
+
+declare -A cases
+cases=(
+    [search]="{BUILD}/bin/nix $flake_args search --no-eval-cache github:nixos/nixpkgs/e1fa12d4f6c6fe19ccb59cac54b5b3f25e160870 hello"
+    [rebuild]="{BUILD}/bin/nix $flake_args eval --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'"
+    [rebuild-lh]="GC_INITIAL_HEAP_SIZE=10g {BUILD}/bin/nix eval $flake_args --raw --impure --expr 'with import <nixpkgs/nixos> {}; system'"
+    [parse]="{BUILD}/bin/nix $flake_args eval -f bench/nixpkgs/pkgs/development/haskell-modules/hackage-packages.nix"
+)
+
+benches=(
+    rebuild
+    rebuild-lh
+    search
+    parse
+)
+
+for k in "${benches[@]}"; do
+    taskset -c 2,3 \
+        chrt -f 50 \
+        hyperfine "${hyperfineArgs[@]}"  --export-json="bench/bench-${k}.json" --export-markdown="bench/bench-${k}.md" "${cases[$k]}"
+done
+
+echo "Benchmarks summary (from ./bench/summarize.jq bench/bench-*.json)"
+bench/summarize.jq bench/*.json
diff --git a/bench/configuration.nix b/bench/configuration.nix
new file mode 100644
index 000000000..54782a1d3
--- /dev/null
+++ b/bench/configuration.nix
@@ -0,0 +1,325 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+{
+  boot = {
+    initrd = {
+      availableKernelModules = [
+        "xhci_pci"
+        "ahci"
+      ];
+      kernelModules = [ "dm-snapshot" ];
+      luks.devices = {
+        croot = {
+          device = "/dev/sdb";
+          allowDiscards = true;
+        };
+      };
+    };
+    kernelModules = [ "kvm-intel" ];
+    kernelPackages = pkgs.linuxPackages_latest;
+
+    loader = {
+      systemd-boot.enable = true;
+      efi.canTouchEfiVariables = true;
+    };
+  };
+
+  hardware = {
+    enableRedistributableFirmware = true;
+    cpu.intel.updateMicrocode = true;
+    opengl.driSupport32Bit = true;
+    opengl.extraPackages = with pkgs; [
+      vaapiIntel
+      intel-media-driver
+      intel-compute-runtime
+    ];
+  };
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/sda2";
+      fsType = "xfs";
+      options = [ "noatime" ];
+    };
+
+    "/boot" = {
+      device = "/dev/sda1";
+      fsType = "vfat";
+    };
+
+    "/nas" = {
+      device = "nas:/";
+      fsType = "nfs4";
+      options = [
+        "ro"
+        "x-systemd.automount"
+      ];
+    };
+  };
+  swapDevices = [ { device = "/dev/swap"; } ];
+
+  networking = {
+    useDHCP = false;
+    hostName = "host";
+    wireless = {
+      enable = true;
+      interfaces = [ "eth1" ];
+    };
+    interfaces = {
+      eth0.useDHCP = true;
+      eth1.useDHCP = true;
+    };
+    wg-quick.interfaces = {
+      wg0 = {
+        address = [ "2001:db8::1" ];
+        privateKeyFile = "/etc/secrets/wg0.key";
+        peers = [
+          {
+            publicKey = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
+            endpoint = "[2001:db8::2]:61021";
+            allowedIPs = [ "2001::db8:1::/64" ];
+          }
+        ];
+      };
+    };
+
+    firewall.allowedUDPPorts = [ 4567 ];
+  };
+
+  i18n = {
+    defaultLocale = "en_US.UTF-8";
+    inputMethod.enabled = "ibus";
+  };
+
+  services = {
+    xserver = {
+      enable = true;
+      layout = "us";
+      xkbVariant = "altgr-intl";
+      xkbOptions = "ctrl:nocaps";
+      libinput.enable = true;
+      wacom.enable = true;
+      videoDrivers = [ "modesetting" ];
+      modules = [ pkgs.xf86_input_wacom ];
+
+      displayManager.sx.enable = true;
+      windowManager.i3.enable = true;
+    };
+
+    udev.extraHwdb = ''
+      # not like this mattered at all
+      # we're not running udev from here
+    '';
+
+    udev.extraRules = ''
+      # ACTION=="add", SUBSYSTEM=="input", ...
+    '';
+  };
+
+  sound.enable = true;
+  hardware.pulseaudio = {
+    enable = true;
+    package = pkgs.pulseaudioFull;
+    daemon.config = {
+      lock-memory = "yes";
+      realtime-scheduling = "yes";
+      rlimit-rtprio = "-1";
+    };
+  };
+
+  programs = {
+    light.enable = true;
+    wireshark = {
+      enable = true;
+      package = pkgs.wireshark-qt;
+    };
+    gnupg.agent = {
+      enable = true;
+    };
+  };
+
+  fonts.packages = with pkgs; [
+    font-awesome
+    noto-fonts
+    noto-fonts-cjk
+    noto-fonts-emoji
+    noto-fonts-extra
+    dejavu_fonts
+    powerline-fonts
+    source-code-pro
+    cantarell-fonts
+  ];
+
+  users = {
+    mutableUsers = false;
+
+    users = {
+      user = {
+        isNormalUser = true;
+        group = "user";
+        extraGroups = [
+          "wheel"
+          "video"
+          "audio"
+          "dialout"
+          "users"
+          "kvm"
+          "wireshark"
+        ];
+        password = "unimportant";
+      };
+    };
+
+    groups = {
+      user = { };
+    };
+  };
+
+  security = {
+    pam.loginLimits = [
+      {
+        domain = "@audio";
+        item = "memlock";
+        type = "-";
+        value = "unlimited";
+      }
+      {
+        domain = "@audio";
+        item = "rtprio";
+        type = "-";
+        value = "99";
+      }
+      {
+        domain = "@audio";
+        item = "nofile";
+        type = "soft";
+        value = "99999";
+      }
+      {
+        domain = "@audio";
+        item = "nofile";
+        type = "hard";
+        value = "99999";
+      }
+    ];
+
+    sudo.extraRules = [
+      {
+        users = [ "user" ];
+        commands = [
+          {
+            command = "${pkgs.linuxPackages.cpupower}/bin/cpupower";
+            options = [ "NOPASSWD" ];
+          }
+        ];
+      }
+    ];
+  };
+
+  environment.systemPackages = with pkgs; [
+    a2jmidid
+    age
+    ardour
+    bemenu
+    blender
+    breeze-icons
+    breeze-qt5
+    bubblewrap
+    calf
+    claws-mail
+    darktable
+    duperemove
+    emacs
+    feh
+    file
+    firefox
+    fluidsynth
+    gnome3.adwaita-icon-theme
+    gnuplot
+    graphviz
+    helm
+    i3status-rust
+    inkscape
+    jack2
+    jq
+    krita
+    ldns
+    libqalculate
+    libreoffice
+    man-pages
+    nheko
+    nix-diff
+    nix-index
+    nix-output-monitor
+    open-music-kontrollers.patchmatrix
+    pamixer
+    pavucontrol
+    pciutils
+    picom
+    pwgen
+    redshift
+    ripgrep
+    rlwrap
+    silver-searcher
+    soundfont-fluid
+    whois
+    wol
+    xclip
+    xdot
+    xdotool
+    xorg.xkbcomp
+    yt-dlp
+    zathura
+    borgbackup
+    linuxPackages.cpupower
+    mtr
+    kitty
+    xf86_input_wacom
+  ];
+
+  environment.pathsToLink = [ "/share/soundfonts" ];
+
+  systemd.user.services.run-python = {
+    after = [ "network-online.target" ];
+    script = ''
+      exec ${pkgs.python3}/bin/python
+    '';
+    serviceConfig = {
+      CapabilityBoundingSet = [ "" ];
+      KeyringMode = "private";
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateTmp = true;
+      PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      RestrictAddressFamilies = "AF_INET AF_INET6";
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+        "@system-service"
+        "~ @resources @privileged"
+      ];
+      UMask = "077";
+    };
+  };
+
+  system.stateVersion = "23.11";
+}
diff --git a/bench/summarize.jq b/bench/summarize.jq
new file mode 100755
index 000000000..5d1449108
--- /dev/null
+++ b/bench/summarize.jq
@@ -0,0 +1,22 @@
+#!/usr/bin/env -S jq -Mrf
+
+def round3:
+    . * 1000 | round | . / 1000
+    ;
+
+def stats($first):
+    [
+        "  mean:     \(.mean | round3)s ± \(.stddev | round3)s",
+        "            user: \(.user | round3)s | system: \(.system | round3)s",
+        "  median:   \(.median | round3)s",
+        "  range:    \(.min | round3)s ... \(.max | round3)s",
+        "  relative: \(.mean / $first.mean | round3)"
+    ]
+    | join("\n")
+    ;
+
+def fmt($first):
+    "\(.command)\n" + (. | stats($first))
+    ;
+
+[.results | .[0] as $first | .[] | fmt($first)] | join("\n\n") | (. + "\n\n---\n")