Merge changes from topic "releng" into main

* changes:
  releng: add prod environment, ready for release
  releng: automatically figure out if we should tag latest for docker
  releng: support multiarch docker images
  manual: rewrite the docker guide now that we have images
  Rewrite docker to be sensible and smaller
  Implement docker upload in the releng tools

1 files changed, 29 insertions, 36 deletions

diff --git a/releng/create_release.xsh b/releng/create_release.xsh
index c57a92b2f..94c78a83f 100644
--- a/releng/create_release.xsh
+++ b/releng/create_release.xsh
@@ -7,19 +7,15 @@ import tempfile
 import hashlib
 import datetime
 from . import environment
+from .environment import RelengEnvironment
 from . import keys
+from . import docker
 from .version import VERSION, RELEASE_NAME, MAJOR
+from .gitutils import verify_are_on_tag, git_preconditions
 
 $RAISE_SUBPROC_ERROR = True
 $XONSH_SHOW_TRACEBACK = True
 
-RELENG_ENV = environment.STAGING
-
-RELEASES_BUCKET = RELENG_ENV.releases_bucket
-DOCS_BUCKET = RELENG_ENV.docs_bucket
-CACHE_STORE = RELENG_ENV.cache_store_uri()
-REPO = RELENG_ENV.git_repo
-
 GCROOTS_DIR = Path('./release/gcroots')
 BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
 DRVS_TXT = Path('./release/drvs.txt')
@@ -35,23 +31,14 @@ MAX_JOBS = 2
 RELEASE_SYSTEMS = ["x86_64-linux"]
 
 
-def setup_creds():
-    key = keys.get_ephemeral_key(RELENG_ENV)
+def setup_creds(env: RelengEnvironment):
+    key = keys.get_ephemeral_key(env)
     $AWS_SECRET_ACCESS_KEY = key.secret_key
     $AWS_ACCESS_KEY_ID = key.id
     $AWS_DEFAULT_REGION = 'garage'
     $AWS_ENDPOINT_URL = environment.S3_ENDPOINT
 
 
-def git_preconditions():
-    # verify there is nothing in index ready to stage
-    proc = !(git diff-index --quiet --cached HEAD --)
-    assert proc.rtn == 0
-    # verify there is nothing *stageable* and tracked
-    proc = !(git diff-files --quiet)
-    assert proc.rtn == 0
-
-
 def official_release_commit_tag(force_tag=False):
     print('[+] Setting officialRelease in flake.nix and tagging')
     prev_branch = $(git symbolic-ref --short HEAD).strip()
@@ -102,13 +89,13 @@ def eval_jobs():
     ]
 
 
-def upload_drv_paths_and_outputs(paths: list[str]):
+def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]):
     proc = subprocess.Popen([
             'nix',
             'copy',
             '-v',
             '--to',
-            CACHE_STORE,
+            env.cache_store_uri(),
             '--stdin',
         ],
         stdin=subprocess.PIPE,
@@ -245,33 +232,38 @@ def prepare_release_notes():
     git commit -m @(commit_msg)
 
 
-def verify_are_on_tag():
-    current_tag = $(git describe --tag).strip()
-    assert current_tag == VERSION
-
-
-def upload_artifacts(noconfirm=False, force_push_tag=False):
+def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False):
+    if not no_check_git:
+        verify_are_on_tag()
+        git_preconditions()
     assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env
 
     tree @(ARTIFACTS)
 
+    env_part = f'environment {env.name}'
     not noconfirm and confirm(
-        f'Would you like to release {ARTIFACTS} as {VERSION}? Type "I want to release this" to confirm\n',
-        'I want to release this'
+        f'Would you like to release {ARTIFACTS} as {VERSION} in {env.colour(env_part)}? Type "I want to release this to {env.name}" to confirm\n',
+        f'I want to release this to {env.name}'
     )
 
+    docker_images = list((ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz'))
+    assert docker_images
+
     print('[+] Upload to cache')
     with open(DRVS_TXT) as fh:
-        upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x])
+        upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x])
 
+    print('[+] Upload docker images')
+    for target in env.docker_targets:
+        docker.upload_docker_images(target, docker_images)
 
     print('[+] Upload to release bucket')
-    aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/
+    aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/
     print('[+] Upload manual')
-    upload_manual()
+    upload_manual(env)
 
     print('[+] git push tag')
-    git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}'
+    git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}'
 
 
 def do_tag_merge(force_tag=False, no_check_git=False):
@@ -290,7 +282,7 @@ def build_manual(eval_result):
     cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL)
 
 
-def upload_manual():
+def upload_manual(env: RelengEnvironment):
     stable = json.loads($(nix eval --json '.#nix.officialRelease'))
     if stable:
         version = MAJOR
@@ -298,9 +290,9 @@ def upload_manual():
         version = 'nightly'
 
     print('[+] aws s3 sync manual')
-    aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/
+    aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/
     if stable:
-        aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/
+        aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/
 
 
 def build_artifacts(no_check_git=False):
@@ -318,7 +310,8 @@ def build_artifacts(no_check_git=False):
     build_manual(eval_result)
 
     with open(DRVS_TXT, 'w') as fh:
-        fh.write('\n'.join(drv_paths))
+        # don't bother putting the release tarballs themselves because they are duplicate and huge
+        fh.write('\n'.join(x['drvPath'] for x in eval_result if x['attr'] != 'lix-release-tarballs'))
 
     make_artifacts_dir(eval_result, ARTIFACTS)
     print(f'[+] Done! See {ARTIFACTS}')