Merge changes from topic "releng" into main

* changes:
  releng: add prod environment, ready for release
  releng: automatically figure out if we should tag latest for docker
  releng: support multiarch docker images
  manual: rewrite the docker guide now that we have images
  Rewrite docker to be sensible and smaller
  Implement docker upload in the releng tools

13 files changed, 900 insertions, 170 deletions

diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md
index 2d7701024..3a5e6ef77 100644
--- a/doc/manual/src/installation/installing-docker.md
+++ b/doc/manual/src/installation/installing-docker.md
@@ -1,64 +1,62 @@
 # Using Lix within Docker
 
-Currently the Lix project doesn't ship docker images. However, we have the infrastructure to do it, it's just not yet been done. See https://git.lix.systems/lix-project/lix/issues/252
-
-<!--
+Lix is available on the following two container registries:
+- [ghcr.io/lix-project/lix](https://ghcr.io/lix-project/lix)
+- [git.lix.systems/lix-project/lix](https://git.lix.systems/lix-project/-/packages/container/lix)
 
 To run the latest stable release of Lix with Docker run the following command:
 
 ```console
-$ docker run -ti nixos/nix
-Unable to find image 'nixos/nix:latest' locally
-latest: Pulling from nixos/nix
-5843afab3874: Pull complete
-b52bf13f109c: Pull complete
-1e2415612aa3: Pull complete
-Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff
-Status: Downloaded newer image for nixos/nix:latest
-35ca4ada6e96:/# nix --version
-nix (Nix) 2.3.12
-35ca4ada6e96:/# exit
+~ » sudo podman run -it ghcr.io/lix-project/lix:latest
+Trying to pull ghcr.io/lix-project/lix:latest...
+
+bash-5.2# nix --version
+nix (Lix, like Nix) 2.90.0
 ```
 
 # What is included in Lix's Docker image?
 
-The official Docker image is created using `pkgs.dockerTools.buildLayeredImage`
+The official Docker image is created using [nix2container]
 (and not with `Dockerfile` as it is usual with Docker images). You can still
 base your custom Docker image on it as you would do with any other Docker
 image.
 
-The Docker image is also not based on any other image and includes minimal set
-of runtime dependencies that are required to use Lix:
-
- - pkgs.nix
- - pkgs.bashInteractive
- - pkgs.coreutils-full
- - pkgs.gnutar
- - pkgs.gzip
- - pkgs.gnugrep
- - pkgs.which
- - pkgs.curl
- - pkgs.less
- - pkgs.wget
- - pkgs.man
- - pkgs.cacert.out
- - pkgs.findutils
+[nix2container]: https://github.com/nlewo/nix2container
+
+The Docker image is also not based on any other image and includes the nixpkgs
+that Lix was built with along with a minimal set of tools in the system profile:
+
+- bashInteractive
+- cacert.out
+- coreutils-full
+- curl
+- findutils
+- gitMinimal
+- gnugrep
+- gnutar
+- gzip
+- iana-etc
+- less
+- libxml2
+- lix
+- man
+- openssh
+- sqlite
+- wget
+- which
 
 # Docker image with the latest development version of Lix
 
-To get the latest image that was built by [Hydra](https://hydra.nixos.org) run
-the following command:
+FIXME: There are not currently images of development versions of Lix. Tracking issue: https://git.lix.systems/lix-project/lix/issues/381
 
-```console
-$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load
-$ docker run -ti nix:2.5pre20211105
-```
+You can build a Docker image from source yourself and copy it to either:
+
+Podman: `nix run '.#dockerImage.copyTo' containers-storage:lix`
+
+Docker: `nix run '.#dockerImage.copyToDockerDaemon'`
 
-You can also build a Docker image from source yourself:
+Then:
 
 ```console
-$ nix build ./\#hydraJobs.dockerImage.x86_64-linux
-$ docker load -i ./result/image.tar.gz
-$ docker run -ti nix:2.5pre20211105
+$ docker run -ti lix
 ```
--->
diff --git a/docker.nix b/docker.nix
index cec3a2950..b23a9637f 100644
--- a/docker.nix
+++ b/docker.nix
@@ -1,5 +1,8 @@
 {
   pkgs ? import <nixpkgs> { },
+  # Git commit ID, if available
+  lixRevision ? null,
+  nix2container,
   lib ? pkgs.lib,
   name ? "lix",
   tag ? "latest",
@@ -12,26 +15,51 @@
   flake-registry ? null,
 }:
 let
+  layerContents = with pkgs; [
+    # pulls in glibc and openssl, about 60MB
+    { contents = [ coreutils-full ]; }
+    # some stuff that is low in the closure graph and small ish, mostly to make
+    # incremental lix updates cheaper
+    {
+      contents = [
+        curl
+        libxml2
+        sqlite
+      ];
+    }
+    # 50MB of git
+    { contents = [ gitMinimal ]; }
+    # 144MB of nixpkgs
+    {
+      contents = [ channel ];
+      inProfile = false;
+    }
+  ];
+
+  # These packages are left to be auto layered by nix2container, since it is
+  # less critical that they get layered sensibly and they tend to not be deps
+  # of anything in particular
+  autoLayered = with pkgs; [
+    bashInteractive
+    gnutar
+    gzip
+    gnugrep
+    which
+    less
+    wget
+    man
+    cacert.out
+    findutils
+    iana-etc
+    openssh
+    nix
+  ];
+
   defaultPkgs =
-    with pkgs;
-    [
-      nix
-      bashInteractive
-      coreutils-full
-      gnutar
-      gzip
-      gnugrep
-      which
-      curl
-      less
-      wget
-      man
-      cacert.out
-      findutils
-      iana-etc
-      git
-      openssh
-    ]
+    lib.lists.flatten (
+      map (x: if !(x ? inProfile) || x.inProfile then x.contents else [ ]) layerContents
+    )
+    ++ autoLayered
     ++ extraPkgs;
 
   users =
@@ -139,16 +167,17 @@ let
     ))
     + "\n";
 
+  nixpkgs = pkgs.path;
+  channel = pkgs.runCommand "channel-nixpkgs" { } ''
+    mkdir $out
+    ${lib.optionalString bundleNixpkgs ''
+      ln -s ${nixpkgs} $out/nixpkgs
+      echo "[]" > $out/manifest.nix
+    ''}
+  '';
+
   baseSystem =
     let
-      nixpkgs = pkgs.path;
-      channel = pkgs.runCommand "channel-nixos" { inherit bundleNixpkgs; } ''
-        mkdir $out
-        if [ "$bundleNixpkgs" ]; then
-          ln -s ${nixpkgs} $out/nixpkgs
-          echo "[]" > $out/manifest.nix
-        fi
-      '';
       rootEnv = pkgs.buildPackages.buildEnv {
         name = "root-profile-env";
         paths = defaultPkgs;
@@ -187,7 +216,7 @@ let
       profile = pkgs.buildPackages.runCommand "user-environment" { } ''
         mkdir $out
         cp -a ${rootEnv}/* $out/
-        ln -s ${manifest} $out/manifest.nix
+        ln -sf ${manifest} $out/manifest.nix
       '';
       flake-registry-path =
         if (flake-registry == null) then
@@ -236,6 +265,7 @@ let
           ln -s /nix/var/nix/profiles/share $out/usr/
 
           mkdir -p $out/nix/var/nix/gcroots
+          ln -s /nix/var/nix/profiles $out/nix/var/nix/gcroots/profiles
 
           mkdir $out/tmp
 
@@ -248,14 +278,14 @@ let
           mkdir -p $out/nix/var/nix/profiles/per-user/root
 
           ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
-          ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+          ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
           ln -s /nix/var/nix/profiles/default $out/root/.nix-profile
 
           ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link
-          ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
+          ln -s /nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels
 
           mkdir -p $out/root/.nix-defexpr
-          ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
+          ln -s /nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels
           echo "${channelURL} ${channelName}" > $out/root/.nix-channels
 
           mkdir -p $out/bin $out/usr/bin
@@ -273,43 +303,99 @@ let
           ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName
         '')
       );
-in
-pkgs.dockerTools.buildLayeredImageWithNixDb {
 
-  inherit name tag maxLayers;
+  layers = builtins.foldl' (
+    layersList: el:
+    let
+      layer = nix2container.buildLayer {
+        deps = el.contents;
+        layers = layersList;
+      };
+    in
+    layersList ++ [ layer ]
+  ) [ ] layerContents;
 
-  contents = [ baseSystem ];
+  image = nix2container.buildImage {
 
-  extraCommands = ''
-    rm -rf nix-support
-    ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles
-  '';
-  fakeRootCommands = ''
-    chmod 1777 tmp
-    chmod 1777 var/tmp
-  '';
+    inherit name tag maxLayers;
 
-  config = {
-    Cmd = [ "/root/.nix-profile/bin/bash" ];
-    Env = [
-      "USER=root"
-      "PATH=${
-        lib.concatStringsSep ":" [
-          "/root/.nix-profile/bin"
-          "/nix/var/nix/profiles/default/bin"
-          "/nix/var/nix/profiles/default/sbin"
-        ]
-      }"
-      "MANPATH=${
-        lib.concatStringsSep ":" [
-          "/root/.nix-profile/share/man"
-          "/nix/var/nix/profiles/default/share/man"
-        ]
-      }"
-      "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-      "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-      "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
-      "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
+    inherit layers;
+
+    copyToRoot = [ baseSystem ];
+
+    initializeNixDatabase = true;
+
+    perms = [
+      {
+        path = baseSystem;
+        regex = "(/var)?/tmp";
+        mode = "1777";
+      }
     ];
+
+    config = {
+      Cmd = [ "/root/.nix-profile/bin/bash" ];
+      Env = [
+        "USER=root"
+        "PATH=${
+          lib.concatStringsSep ":" [
+            "/root/.nix-profile/bin"
+            "/nix/var/nix/profiles/default/bin"
+            "/nix/var/nix/profiles/default/sbin"
+          ]
+        }"
+        "MANPATH=${
+          lib.concatStringsSep ":" [
+            "/root/.nix-profile/share/man"
+            "/nix/var/nix/profiles/default/share/man"
+          ]
+        }"
+        "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+        "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+        "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
+        "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels"
+      ];
+
+      Labels = {
+        "org.opencontainers.image.title" = "Lix";
+        "org.opencontainers.image.source" = "https://git.lix.systems/lix-project/lix";
+        "org.opencontainers.image.vendor" = "Lix project";
+        "org.opencontainers.image.version" = pkgs.nix.version;
+        "org.opencontainers.image.description" = "Minimal Lix container image, with some batteries included.";
+      } // lib.optionalAttrs (lixRevision != null) { "org.opencontainers.image.revision" = lixRevision; };
+    };
+
+    meta = {
+      description = "Docker image for Lix. This is built with nix2container; see that project's README for details";
+      longDescription = ''
+        Docker image for Lix, built with nix2container.
+        To copy it to your docker daemon, nix run .#dockerImage.copyToDockerDaemon
+        To copy it to podman, nix run .#dockerImage.copyTo containers-storage:lix
+      '';
+    };
   };
+in
+image
+// {
+  # We don't ship the tarball as the default output because it is a strange thing to want imo
+  tarball =
+    pkgs.buildPackages.runCommand "docker-image-tarball-${pkgs.nix.version}"
+      {
+        nativeBuildInputs = [ pkgs.buildPackages.bubblewrap ];
+        meta.description = "Docker image tarball with Lix for ${pkgs.system}";
+      }
+      ''
+        mkdir -p $out/nix-support
+        image=$out/image.tar
+        # bwrap for foolish temp dir selection code that forces /var/tmp:
+        # https://github.com/containers/skopeo.git/blob/60ee543f7f7c242f46cc3a7541d9ac8ab1c89168/vendor/github.com/containers/image/v5/internal/tmpdir/tmpdir.go#L15-L18
+        mkdir -p $TMPDIR/fake-var/tmp
+        args=(--unshare-user --bind "$TMPDIR/fake-var" /var)
+        for dir in /*; do
+          args+=(--dev-bind "/$dir" "/$dir")
+        done
+        bwrap ''${args[@]} -- ${lib.getExe image.copyTo} docker-archive:$image
+        gzip $image
+        echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
+      '';
 }
diff --git a/flake.lock b/flake.lock
index 8e59b2942..436cbcd03 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "nix2container": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1712990762,
+        "narHash": "sha256-hO9W3w7NcnYeX8u8cleHiSpK2YJo7ecarFTUlbybl7k=",
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "20aad300c925639d5d6cbe30013c8357ce9f2a2e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1715123187,
@@ -67,6 +83,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "nix2container": "nix2container",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression",
         "pre-commit-hooks": "pre-commit-hooks"
diff --git a/flake.nix b/flake.nix
index 8460201b0..6c7abaeec 100644
--- a/flake.nix
+++ b/flake.nix
@@ -8,6 +8,10 @@
       url = "github:cachix/git-hooks.nix";
       flake = false;
     };
+    nix2container = {
+      url = "github:nlewo/nix2container";
+      flake = false;
+    };
     flake-compat = {
       url = "github:edolstra/flake-compat";
       flake = false;
@@ -20,6 +24,7 @@
       nixpkgs,
       nixpkgs-regression,
       pre-commit-hooks,
+      nix2container,
       flake-compat,
     }:
 
@@ -330,19 +335,13 @@
             dockerImage =
               let
                 pkgs = nixpkgsFor.${system}.native;
-                image = import ./docker.nix {
-                  inherit pkgs;
-                  tag = pkgs.nix.version;
-                };
+                nix2container' = import nix2container { inherit pkgs system; };
               in
-              pkgs.runCommand "docker-image-tarball-${pkgs.nix.version}"
-                { meta.description = "Docker image with Lix for ${system}"; }
-                ''
-                  mkdir -p $out/nix-support
-                  image=$out/image.tar.gz
-                  ln -s ${image} $image
-                  echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
-                '';
+              import ./docker.nix {
+                inherit pkgs;
+                nix2container = nix2container'.nix2container;
+                tag = pkgs.nix.version;
+              };
           }
           // builtins.listToAttrs (
             map (crossSystem: {
diff --git a/package.nix b/package.nix
index f298dfb0b..c9e8ece3f 100644
--- a/package.nix
+++ b/package.nix
@@ -39,6 +39,7 @@
   pkg-config,
   python3,
   rapidcheck,
+  skopeo,
   sqlite,
   toml11,
   util-linuxMinimal ? utillinuxMinimal,
@@ -416,6 +417,8 @@ stdenv.mkDerivation (finalAttrs: {
           p: [
             p.yapf
             p.python-frontmatter
+            p.requests
+            p.xdg-base-dirs
             (p.toPythonModule xonsh-unwrapped)
           ]
         );
@@ -452,6 +455,8 @@ stdenv.mkDerivation (finalAttrs: {
             lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm
             ++ [
               pythonEnv
+              # docker image tool
+              skopeo
               just
               nixfmt
               # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above.
diff --git a/releng/__init__.py b/releng/__init__.py
index 401b8e322..59d1709b9 100644
--- a/releng/__init__.py
+++ b/releng/__init__.py
@@ -2,11 +2,30 @@ from xonsh.main import setup
 setup()
 del setup
 
-from releng import environment
-from releng import create_release
-from releng import keys
-from releng import version
-from releng import cli
+import logging
+
+from . import environment
+from . import create_release
+from . import keys
+from . import version
+from . import cli
+from . import docker
+from . import docker_assemble
+from . import gitutils
+
+rootLogger = logging.getLogger()
+rootLogger.setLevel(logging.DEBUG)
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+
+fmt = logging.Formatter('{asctime} {levelname} {name}: {message}',
+                        datefmt='%b %d %H:%M:%S',
+                        style='{')
+
+if not any(isinstance(h, logging.StreamHandler) for h in rootLogger.handlers):
+    hand = logging.StreamHandler()
+    hand.setFormatter(fmt)
+    rootLogger.addHandler(hand)
 
 def reload():
     import importlib
@@ -15,3 +34,6 @@ def reload():
     importlib.reload(keys)
     importlib.reload(version)
     importlib.reload(cli)
+    importlib.reload(docker)
+    importlib.reload(docker_assemble)
+    importlib.reload(gitutils)
diff --git a/releng/cli.py b/releng/cli.py
index bba50f534..89391e0a7 100644
--- a/releng/cli.py
+++ b/releng/cli.py
@@ -1,4 +1,8 @@
 from . import create_release
+from . import docker
+from .environment import RelengEnvironment
+from . import environment
+import functools
 import argparse
 import sys
 
@@ -18,13 +22,16 @@ def do_tag(args):
                                 no_check_git=args.no_check_git)
 
 
-def do_upload(args):
-    create_release.setup_creds()
+def do_upload(env: RelengEnvironment, args):
+    create_release.setup_creds(env)
     if args.target == 'all':
-        create_release.upload_artifacts(force_push_tag=args.force_push_tag,
-                                        noconfirm=args.noconfirm)
+        docker.check_all_logins(env)
+        create_release.upload_artifacts(env,
+                                        force_push_tag=args.force_push_tag,
+                                        noconfirm=args.noconfirm,
+                                        no_check_git=args.no_check_git)
     elif args.target == 'manual':
-        create_release.upload_manual()
+        create_release.upload_manual(env)
     else:
         raise ValueError('invalid target, unreachable')
 
@@ -77,6 +84,10 @@ def main():
 
     upload = sps.add_parser(
         'upload', help='Upload artifacts to cache and releases bucket')
+    upload.add_argument(
+        '--no-check-git',
+        action='store_true',
+        help="Don't check git state before uploading. For testing.")
     upload.add_argument('--force-push-tag',
                         action='store_true',
                         help='Force push the tag. For testing.')
@@ -90,7 +101,12 @@ def main():
         '--noconfirm',
         action='store_true',
         help="Don't ask for confirmation. For testing/automation.")
-    upload.set_defaults(cmd=do_upload)
+    upload.add_argument('--environment',
+                        choices=list(environment.ENVIRONMENTS.keys()),
+                        default='staging',
+                        help='Environment to release to')
+    upload.set_defaults(cmd=lambda args: do_upload(
+        environment.ENVIRONMENTS[args.environment], args))
 
     args = ap.parse_args()
     args.cmd(args)
diff --git a/releng/create_release.xsh b/releng/create_release.xsh
index c57a92b2f..94c78a83f 100644
--- a/releng/create_release.xsh
+++ b/releng/create_release.xsh
@@ -7,19 +7,15 @@ import tempfile
 import hashlib
 import datetime
 from . import environment
+from .environment import RelengEnvironment
 from . import keys
+from . import docker
 from .version import VERSION, RELEASE_NAME, MAJOR
+from .gitutils import verify_are_on_tag, git_preconditions
 
 $RAISE_SUBPROC_ERROR = True
 $XONSH_SHOW_TRACEBACK = True
 
-RELENG_ENV = environment.STAGING
-
-RELEASES_BUCKET = RELENG_ENV.releases_bucket
-DOCS_BUCKET = RELENG_ENV.docs_bucket
-CACHE_STORE = RELENG_ENV.cache_store_uri()
-REPO = RELENG_ENV.git_repo
-
 GCROOTS_DIR = Path('./release/gcroots')
 BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
 DRVS_TXT = Path('./release/drvs.txt')
@@ -35,23 +31,14 @@ MAX_JOBS = 2
 RELEASE_SYSTEMS = ["x86_64-linux"]
 
 
-def setup_creds():
-    key = keys.get_ephemeral_key(RELENG_ENV)
+def setup_creds(env: RelengEnvironment):
+    key = keys.get_ephemeral_key(env)
     $AWS_SECRET_ACCESS_KEY = key.secret_key
     $AWS_ACCESS_KEY_ID = key.id
     $AWS_DEFAULT_REGION = 'garage'
     $AWS_ENDPOINT_URL = environment.S3_ENDPOINT
 
 
-def git_preconditions():
-    # verify there is nothing in index ready to stage
-    proc = !(git diff-index --quiet --cached HEAD --)
-    assert proc.rtn == 0
-    # verify there is nothing *stageable* and tracked
-    proc = !(git diff-files --quiet)
-    assert proc.rtn == 0
-
-
 def official_release_commit_tag(force_tag=False):
     print('[+] Setting officialRelease in flake.nix and tagging')
     prev_branch = $(git symbolic-ref --short HEAD).strip()
@@ -102,13 +89,13 @@ def eval_jobs():
     ]
 
 
-def upload_drv_paths_and_outputs(paths: list[str]):
+def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
     proc = subprocess.Popen([
             'nix',
             'copy',
             '-v',
             '--to',
-            CACHE_STORE,
+            env.cache_store_uri(),
             '--stdin',
         ],
         stdin=subprocess.PIPE,
@@ -245,33 +232,38 @@ def prepare_release_notes():
     git commit -m @(commit_msg)
 
 
-def verify_are_on_tag():
-    current_tag = $(git describe --tag).strip()
-    assert current_tag == VERSION
-
-
-def upload_artifacts(noconfirm=False, force_push_tag=False):
+def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False):
+    if not no_check_git:
+        verify_are_on_tag()
+        git_preconditions()
     assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env
 
     tree @(ARTIFACTS)
 
+    env_part = f'environment {env.name}'
     not noconfirm and confirm(
-        f'Would you like to release {ARTIFACTS} as {VERSION}? Type "I want to release this" to confirm\n',
-        'I want to release this'
+        f'Would you like to release {ARTIFACTS} as {VERSION} in {env.colour(env_part)}? Type "I want to release this to {env.name}" to confirm\n',
+        f'I want to release this to {env.name}'
     )
 
+    docker_images = list((ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz'))
+    assert docker_images
+
     print('[+] Upload to cache')
     with open(DRVS_TXT) as fh:
-        upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x])
+        upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x])
 
+    print('[+] Upload docker images')
+    for target in env.docker_targets:
+        docker.upload_docker_images(target, docker_images)
 
     print('[+] Upload to release bucket')
-    aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/
+    aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/
     print('[+] Upload manual')
-    upload_manual()
+    upload_manual(env)
 
     print('[+] git push tag')
-    git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}'
+    git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
 
 
 def do_tag_merge(force_tag=False, no_check_git=False):
@@ -290,7 +282,7 @@ def build_manual(eval_result):
     cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL)
 
 
-def upload_manual():
+def upload_manual(env: RelengEnvironment):
     stable = json.loads($(nix eval --json '.#nix.officialRelease'))
     if stable:
         version = MAJOR
@@ -298,9 +290,9 @@ def upload_manual():
         version = 'nightly'
 
     print('[+] aws s3 sync manual')
-    aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/
+    aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
     if stable:
-        aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/
+        aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
 
 
 def build_artifacts(no_check_git=False):
@@ -318,7 +310,8 @@ def build_artifacts(no_check_git=False):
     build_manual(eval_result)
 
     with open(DRVS_TXT, 'w') as fh:
-        fh.write('\n'.join(drv_paths))
+        # don't bother putting the release tarballs themselves because they are duplicate and huge
+        fh.write('\n'.join(x['drvPath'] for x in eval_result if x['attr'] != 'lix-release-tarballs'))
 
     make_artifacts_dir(eval_result, ARTIFACTS)
     print(f'[+] Done! See {ARTIFACTS}')
diff --git a/releng/docker.xsh b/releng/docker.xsh
new file mode 100644
index 000000000..20fb30cd3
--- /dev/null
+++ b/releng/docker.xsh
@@ -0,0 +1,74 @@
+import json
+import logging
+from pathlib import Path
+import tempfile
+
+import requests
+
+from .environment import DockerTarget, RelengEnvironment
+from .version import VERSION, MAJOR
+from . import gitutils
+from .docker_assemble import Registry, OCIIndex, OCIIndexItem
+from . import docker_assemble
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+def check_all_logins(env: RelengEnvironment):
+    for target in env.docker_targets:
+        check_login(target)
+
+def check_login(target: DockerTarget):
+    skopeo login @(target.registry_name())
+
+def upload_docker_images(target: DockerTarget, paths: list[Path]):
+    if not paths: return
+
+    sess = requests.Session()
+    sess.headers['User-Agent'] = 'lix-releng'
+
+    tag_names = [DockerTarget.resolve(tag, version=VERSION, major=MAJOR) for tag in target.tags]
+
+    # latest only gets tagged for the current release branch of Lix
+    if not gitutils.is_maintenance_branch('HEAD'):
+        tag_names.append('latest')
+
+    meta = {}
+
+    reg = docker_assemble.Registry(sess)
+    manifests = []
+
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp = Path(tmp)
+
+        for path in paths:
+            digest_file = tmp / (path.name + '.digest')
+            inspection = json.loads($(skopeo inspect docker-archive:@(path)))
+
+            docker_arch = inspection['Architecture']
+            docker_os = inspection['Os']
+            meta = inspection['Labels']
+
+            log.info('Pushing image %s for %s to %s', path, docker_arch, target.registry_path)
+
+            # insecure-policy: we don't have any signature policy, we are just uploading an image
+            # We upload to a junk tag, because otherwise it will upload to `latest`, which is undesirable
+            skopeo --insecure-policy copy --format oci --digestfile @(digest_file) docker-archive:@(path) docker://@(target.registry_path):temp
+
+            digest = digest_file.read_text().strip()
+
+            # skopeo doesn't give us the manifest size directly, so we just ask the registry
+            metadata = reg.image_info(target.registry_path, digest)
+
+            manifests.append(OCIIndexItem(metadata=metadata, architecture=docker_arch, os=docker_os))
+    # delete the temp tag, which we only have to create because of skopeo
+    # limitations anyhow (it seems to not have a way to say "don't tag it, find
+    # your checksum and put it there")
+    # FIXME: this is not possible because GitHub only has a proprietary API for it. amazing. 11/10.
+    # reg.delete_tag(target.registry_path, 'temp')
+
+    log.info('Pushed images to %r, building a bigger and more menacing manifest from %r with metadata %r', target, manifests, meta)
+    # send the multiarch manifest to each tag
+    index = OCIIndex(manifests=manifests, annotations=meta)
+    for tag in tag_names:
+        reg.upload_index(target.registry_path, tag, index)
diff --git a/releng/docker_assemble.py b/releng/docker_assemble.py
new file mode 100644
index 000000000..ef1d8c4e6
--- /dev/null
+++ b/releng/docker_assemble.py
@@ -0,0 +1,399 @@
+from typing import Any, Literal, Optional
+import re
+from pathlib import Path
+import json
+import dataclasses
+import time
+from urllib.parse import unquote
+import urllib.request
+import logging
+
+import requests.auth
+import requests
+import xdg_base_dirs
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+DEBUG_REQUESTS = False
+if DEBUG_REQUESTS:
+    urllib3_logger = logging.getLogger('requests.packages.urllib3')
+    urllib3_logger.setLevel(logging.DEBUG)
+    urllib3_logger.propagate = True
+
+# So, there is a bunch of confusing stuff happening in this file. The gist of why it's Like This is:
+#
+# nix2container does not concern itself with tags (reasonably enough):
+# https://github.com/nlewo/nix2container/issues/59
+#
+# This is fine. But then we noticed: docker images don't play nice if you have
+# multiple architectures you want to abstract over if you don't do special
+# things. Those special things are images with manifests containing multiple
+# images.
+#
+# Docker has a data model vaguely analogous to git: you have higher level
+# objects referring to a bunch of content-addressed blobs.
+#
+# A multiarch image is more or less just a manifest that refers to more
+# manifests; in OCI it is an Index.
+#
+# See the API spec here: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#definitions
+# And the Index spec here: https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md
+#
+# skopeo doesn't *know* how to make multiarch *manifests*:
+# https://github.com/containers/skopeo/issues/1136
+#
+# There is a tool called manifest-tool that is supposed to do this
+# (https://github.com/estesp/manifest-tool) but it doesn't support putting in
+# annotations on the outer image, and I *really* didn't want to write golang to
+# fix that. Thus, a little bit of homebrew containers code.
+#
+# Essentially what we are doing in here is splatting a bunch of images into the
+# registry without tagging them (except as "temp", due to podman issues), then
+# simply sending a new composite manifest ourselves.
+
+DockerArchitecture = Literal['amd64'] | Literal['arm64']
+MANIFEST_MIME = 'application/vnd.oci.image.manifest.v1+json'
+INDEX_MIME = 'application/vnd.oci.image.index.v1+json'
+
+
+@dataclasses.dataclass(frozen=True, order=True)
+class ImageMetadata:
+    size: int
+    digest: str
+    """sha256:SOMEHEX"""
+
+
+@dataclasses.dataclass(frozen=True, order=True)
+class OCIIndexItem:
+    """Information about an untagged uploaded image."""
+
+    metadata: ImageMetadata
+
+    architecture: DockerArchitecture
+
+    os: str = 'linux'
+
+    def serialize(self):
+        return {
+            'mediaType': MANIFEST_MIME,
+            'size': self.metadata.size,
+            'digest': self.metadata.digest,
+            'platform': {
+                'architecture': self.architecture,
+                'os': self.os,
+            }
+        }
+
+
+@dataclasses.dataclass(frozen=True)
+class OCIIndex:
+    manifests: list[OCIIndexItem]
+
+    annotations: dict[str, str]
+
+    def serialize(self):
+        return {
+            'schemaVersion': 2,
+            'manifests': [item.serialize() for item in sorted(self.manifests)],
+            'annotations': self.annotations
+        }
+
+
+def docker_architecture_from_nix_system(system: str) -> DockerArchitecture:
+    MAP = {
+        'x86_64-linux': 'amd64',
+        'aarch64-linux': 'arm64',
+    }
+    return MAP[system]  # type: ignore
+
+
+@dataclasses.dataclass
+class TaggingOperation:
+    manifest: OCIIndex
+    tags: list[str]
+    """Tags this image is uploaded under"""
+
+
+runtime_dir = xdg_base_dirs.xdg_runtime_dir()
+config_dir = xdg_base_dirs.xdg_config_home()
+
+AUTH_FILES = ([runtime_dir / 'containers/auth.json'] if runtime_dir else []) + \
+    [config_dir / 'containers/auth.json', Path.home() / '.docker/config.json']
+
+
+# Copied from Werkzeug https://github.com/pallets/werkzeug/blob/62e3ea45846d06576199a2f8470be7fe44c867c1/src/werkzeug/http.py#L300-L325
+def parse_list_header(value: str) -> list[str]:
+    """Parse a header value that consists of a list of comma separated items according
+    to `RFC 9110 <https://httpwg.org/specs/rfc9110.html#abnf.extension>`__.
+
+    This extends :func:`urllib.request.parse_http_list` to remove surrounding quotes
+    from values.
+
+    .. code-block:: python
+
+        parse_list_header('token, "quoted value"')
+        ['token', 'quoted value']
+
+    This is the reverse of :func:`dump_header`.
+
+    :param value: The header value to parse.
+    """
+    result = []
+
+    for item in urllib.request.parse_http_list(value):
+        if len(item) >= 2 and item[0] == item[-1] == '"':
+            item = item[1:-1]
+
+        result.append(item)
+
+    return result
+
+
+# https://www.rfc-editor.org/rfc/rfc2231#section-4
+_charset_value_re = re.compile(
+    r"""
+    ([\w!#$%&*+\-.^`|~]*)'  # charset part, could be empty
+    [\w!#$%&*+\-.^`|~]*'  # don't care about language part, usually empty
+    ([\w!#$%&'*+\-.^`|~]+)  # one or more token chars with percent encoding
+    """,
+    re.ASCII | re.VERBOSE,
+)
+
+
+# Copied from: https://github.com/pallets/werkzeug/blob/62e3ea45846d06576199a2f8470be7fe44c867c1/src/werkzeug/http.py#L327-L394
+def parse_dict_header(value: str) -> dict[str, str | None]:
+    """Parse a list header using :func:`parse_list_header`, then parse each item as a
+    ``key=value`` pair.
+
+    .. code-block:: python
+
+        parse_dict_header('a=b, c="d, e", f')
+        {"a": "b", "c": "d, e", "f": None}
+
+    This is the reverse of :func:`dump_header`.
+
+    If a key does not have a value, it is ``None``.
+
+    This handles charsets for values as described in
+    `RFC 2231 <https://www.rfc-editor.org/rfc/rfc2231#section-3>`__. Only ASCII, UTF-8,
+    and ISO-8859-1 charsets are accepted, otherwise the value remains quoted.
+
+    :param value: The header value to parse.
+
+    .. versionchanged:: 3.0
+        Passing bytes is not supported.
+
+    .. versionchanged:: 3.0
+        The ``cls`` argument is removed.
+
+    .. versionchanged:: 2.3
+        Added support for ``key*=charset''value`` encoded items.
+
+    .. versionchanged:: 0.9
+       The ``cls`` argument was added.
+    """
+    result: dict[str, str | None] = {}
+
+    for item in parse_list_header(value):
+        key, has_value, value = item.partition("=")
+        key = key.strip()
+
+        if not has_value:
+            result[key] = None
+            continue
+
+        value = value.strip()
+        encoding: str | None = None
+
+        if key[-1] == "*":
+            # key*=charset''value becomes key=value, where value is percent encoded
+            # adapted from parse_options_header, without the continuation handling
+            key = key[:-1]
+            match = _charset_value_re.match(value)
+
+            if match:
+                # If there is a charset marker in the value, split it off.
+                encoding, value = match.groups()
+                assert encoding
+                encoding = encoding.lower()
+
+            # A safe list of encodings. Modern clients should only send ASCII or UTF-8.
+            # This list will not be extended further. An invalid encoding will leave the
+            # value quoted.
+            if encoding in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}:
+                # invalid bytes are replaced during unquoting
+                value = unquote(value, encoding=encoding)
+
+        if len(value) >= 2 and value[0] == value[-1] == '"':
+            value = value[1:-1]
+
+        result[key] = value
+
+    return result
+
+
+def parse_www_authenticate(www_authenticate):
+    scheme, _, rest = www_authenticate.partition(' ')
+    scheme = scheme.lower()
+    rest = rest.strip()
+
+    parsed = parse_dict_header(rest.rstrip('='))
+    return parsed
+
+
+class AuthState:
+
+    def __init__(self, auth_files: list[Path] = AUTH_FILES):
+        self.auth_map: dict[str, str] = {}
+        for f in auth_files:
+            self.auth_map.update(AuthState.load_auth_file(f))
+        self.token_cache: dict[str, str] = {}
+
+    @staticmethod
+    def load_auth_file(path: Path) -> dict[str, str]:
+        if path.exists():
+            with path.open() as fh:
+                try:
+                    json_obj = json.load(fh)
+                    return {k: v['auth'] for k, v in json_obj['auths'].items()}
+                except (json.JSONDecodeError, KeyError) as e:
+                    log.exception('JSON decode error in %s', path, exc_info=e)
+        return {}
+
+    def get_token(self, hostname: str) -> Optional[str]:
+        return self.token_cache.get(hostname)
+
+    def obtain_token(self, session: requests.Session, token_endpoint: str,
+                     scope: str, service: str, image_path: str) -> str:
+        authority, _, _ = image_path.partition('/')
+        if tok := self.get_token(authority):
+            return tok
+
+        creds = self.find_credential_for(image_path)
+        if not creds:
+            raise ValueError('No credentials available for ' + image_path)
+
+        resp = session.get(token_endpoint,
+                           params={
+                               'client_id': 'lix-releng',
+                               'scope': scope,
+                               'service': service,
+                           },
+                           headers={
+                               'Authorization': 'Basic ' + creds
+                           }).json()
+        token = resp['token']
+        self.token_cache[service] = token
+        return token
+
+    def find_credential_for(self, image_path: str):
+        trails = image_path.split('/')
+        for i in range(len(trails)):
+            prefix = '/'.join(trails[:len(trails) - i])
+            if prefix in self.auth_map:
+                return self.auth_map[prefix]
+
+        return None
+
+
+class RegistryAuthenticator(requests.auth.AuthBase):
+    """Authenticates to an OCI compliant registry"""
+
+    def __init__(self, auth_state: AuthState, session: requests.Session,
+                 image: str):
+        self.auth_map: dict[str, str] = {}
+        self.image = image
+        self.session = session
+        self.auth_state = auth_state
+
+    def response_hook(self, r: requests.Response,
+                      **kwargs: Any) -> requests.Response:
+        if r.status_code == 401:
+            www_authenticate = r.headers.get('www-authenticate', '').lower()
+            parsed = parse_www_authenticate(www_authenticate)
+            assert parsed
+
+            tok = self.auth_state.obtain_token(
+                self.session,
+                parsed['realm'],  # type: ignore
+                parsed['scope'],  # type: ignore
+                parsed['service'],  # type: ignore
+                self.image)
+
+            new_req = r.request.copy()
+            new_req.headers['Authorization'] = 'Bearer ' + tok
+
+            return self.session.send(new_req)
+        else:
+            return r
+
+    def __call__(self,
+                 r: requests.PreparedRequest) -> requests.PreparedRequest:
+        authority, _, _ = self.image.partition('/')
+        auth_may = self.auth_state.get_token(authority)
+
+        if auth_may:
+            r.headers['Authorization'] = 'Bearer ' + auth_may
+
+        r.register_hook('response', self.response_hook)
+        return r
+
+
+class Registry:
+
+    def __init__(self, session: requests.Session):
+        self.auth_state = AuthState()
+        self.session = session
+
+    def image_info(self, image_path: str, manifest_id: str) -> ImageMetadata:
+        authority, _, path = image_path.partition('/')
+        resp = self.session.head(
+            f'https://{authority}/v2/{path}/manifests/{manifest_id}',
+            headers={'Accept': MANIFEST_MIME},
+            auth=RegistryAuthenticator(self.auth_state, self.session,
+                                       image_path))
+        resp.raise_for_status()
+        return ImageMetadata(int(resp.headers['content-length']),
+                             resp.headers['docker-content-digest'])
+
+    def delete_tag(self, image_path: str, tag: str):
+        authority, _, path = image_path.partition('/')
+        resp = self.session.delete(
+            f'https://{authority}/v2/{path}/manifests/{tag}',
+            headers={'Content-Type': INDEX_MIME},
+            auth=RegistryAuthenticator(self.auth_state, self.session,
+                                       image_path))
+        resp.raise_for_status()
+
+    def _upload_index(self, image_path: str, tag: str, index: OCIIndex):
+        authority, _, path = image_path.partition('/')
+        body = json.dumps(index.serialize(),
+                          separators=(',', ':'),
+                          sort_keys=True)
+
+        resp = self.session.put(
+            f'https://{authority}/v2/{path}/manifests/{tag}',
+            data=body,
+            headers={'Content-Type': INDEX_MIME},
+            auth=RegistryAuthenticator(self.auth_state, self.session,
+                                       image_path))
+        resp.raise_for_status()
+
+        return resp.headers['Location']
+
+    def upload_index(self,
+                     image_path: str,
+                     tag: str,
+                     index: OCIIndex,
+                     retries=20,
+                     retry_delay=1):
+        # eventual consistency lmao
+        for _ in range(retries):
+            try:
+                return self._upload_index(image_path, tag, index)
+            except requests.HTTPError as e:
+                if e.response.status_code != 404:
+                    raise
+
+            time.sleep(retry_delay)
diff --git a/releng/environment.py b/releng/environment.py
index 58633d548..ca8194fe5 100644
--- a/releng/environment.py
+++ b/releng/environment.py
@@ -1,5 +1,9 @@
-import dataclasses
+from typing import Callable
 import urllib.parse
+import re
+import functools
+import subprocess
+import dataclasses
 
 S3_HOST = 's3.lix.systems'
 S3_ENDPOINT = 'https://s3.lix.systems'
@@ -17,8 +21,31 @@ DEFAULT_STORE_URI_BITS = {
 
 
 @dataclasses.dataclass
+class DockerTarget:
+    registry_path: str
+    """Registry path without the tag, e.g. ghcr.io/lix-project/lix"""
+
+    tags: list[str]
+    """List of tags this image should take. There must be at least one."""
+
+    @staticmethod
+    def resolve(item: str, version: str, major: str) -> str:
+        """
+        Applies templates:
+        - version: the Lix version e.g. 2.90.0
+        - major: the major Lix version e.g. 2.90
+        """
+        return item.format(version=version, major=major)
+
+    def registry_name(self) -> str:
+        [a, _, _] = self.registry_path.partition('/')
+        return a
+
+
+@dataclasses.dataclass
 class RelengEnvironment:
     name: str
+    colour: Callable[[str], str]
 
     cache_store_overlay: dict[str, str]
     cache_bucket: str
@@ -26,22 +53,79 @@ class RelengEnvironment:
     docs_bucket: str
     git_repo: str
 
+    docker_targets: list[DockerTarget]
+
     def cache_store_uri(self):
         qs = DEFAULT_STORE_URI_BITS.copy()
         qs.update(self.cache_store_overlay)
         return self.cache_bucket + "?" + urllib.parse.urlencode(qs)
 
+
+SGR = '\x1b['
+RED = '31;1m'
+GREEN = '32;1m'
+RESET = '0m'
+
+
+def sgr(colour: str, text: str) -> str:
+    return f'{SGR}{colour}{text}{SGR}{RESET}'
+
+
 STAGING = RelengEnvironment(
     name='staging',
+    colour=functools.partial(sgr, GREEN),
     docs_bucket='s3://staging-docs',
     cache_bucket='s3://staging-cache',
-    cache_store_overlay={
-        'secret-key': 'staging.key'
-    },
+    cache_store_overlay={'secret-key': 'staging.key'},
     releases_bucket='s3://staging-releases',
     git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging',
+    docker_targets=[
+        # latest will be auto tagged if appropriate
+        DockerTarget('git.lix.systems/lix-project/lix-releng-staging',
+                     tags=['{version}', '{major}']),
+        DockerTarget('ghcr.io/lix-project/lix-releng-staging',
+                     tags=['{version}', '{major}']),
+    ],
 )
 
+GERRIT_REMOTE_RE = re.compile(r'^ssh://(\w+@)?gerrit.lix.systems:2022/lix$')
+
+
+def guess_gerrit_remote():
+    """
+    Deals with people having unknown gerrit username.
+    """
+    out = [
+        x.split()[1] for x in subprocess.check_output(
+            ['git', 'remote', '-v']).decode().splitlines()
+    ]
+    return next(x for x in out if GERRIT_REMOTE_RE.match(x))
+
+
+PROD = RelengEnvironment(
+    name='production',
+    colour=functools.partial(sgr, RED),
+    docs_bucket='s3://docs',
+    cache_bucket='s3://cache',
+    # FIXME: we should decrypt this with age into a tempdir in the future, but
+    # the issue is how to deal with the recipients file. For now, we should
+    # just delete it after doing a release.
+    cache_store_overlay={'secret-key': 'prod.key'},
+    releases_bucket='s3://releases',
+    git_repo=guess_gerrit_remote(),
+    docker_targets=[
+        # latest will be auto tagged if appropriate
+        DockerTarget('git.lix.systems/lix-project/lix',
+                     tags=['{version}', '{major}']),
+        DockerTarget('ghcr.io/lix-project/lix', tags=['{version}', '{major}']),
+    ],
+)
+
+ENVIRONMENTS = {
+    'staging': STAGING,
+    'production': PROD,
+}
+
 
 @dataclasses.dataclass
 class S3Credentials:
diff --git a/releng/gitutils.xsh b/releng/gitutils.xsh
new file mode 100644
index 000000000..3352a6b21
--- /dev/null
+++ b/releng/gitutils.xsh
@@ -0,0 +1,37 @@
+import subprocess
+import json
+
+
+def version_compare(v1: str, v2: str):
+    return json.loads($(nix-instantiate --eval --json --argstr v1 @(v1) --argstr v2 @(v2) --expr '{v1, v2}: builtins.compareVersions v1 v2'))
+
+
+def latest_tag_on_branch(branch: str) -> str:
+    return $(git describe --abbrev=0 @(branch) e>/dev/null).strip()
+
+
+def is_maintenance_branch(branch: str) -> bool:
+    try:
+        main_tag = latest_tag_on_branch('main')
+        current_tag = latest_tag_on_branch(branch)
+
+        return version_compare(current_tag, main_tag) < 0
+    except subprocess.CalledProcessError:
+        # This is the case before Lix releases 2.90, since main *has* no
+        # release tag on it.
+        # FIXME: delete this case after 2.91
+        return False
+
+
+def verify_are_on_tag():
+    current_tag = $(git describe --tag).strip()
+    assert current_tag == VERSION
+
+
+def git_preconditions():
+    # verify there is nothing in index ready to stage
+    proc = !(git diff-index --quiet --cached HEAD --)
+    assert proc.rtn == 0
+    # verify there is nothing *stageable* and tracked
+    proc = !(git diff-files --quiet)
+    assert proc.rtn == 0
diff --git a/releng/release-jobs.nix b/releng/release-jobs.nix
index e693a3a81..4db0baed3 100644
--- a/releng/release-jobs.nix
+++ b/releng/release-jobs.nix
@@ -33,7 +33,7 @@ let
       targetName = "*.tar.xz";
     }) systems
     ++ builtins.map (system: {
-      target = hydraJobs.dockerImage.${system};
+      target = hydraJobs.dockerImage.${system}.tarball;
       targetName = "image.tar.gz";
       rename = "lix-${lix.version}-docker-image-${system}.tar.gz";
     }) dockerSystems;