aboutsummaryrefslogtreecommitdiff
path: root/releng/create_release.xsh
diff options
context:
space:
mode:
Diffstat (limited to 'releng/create_release.xsh')
-rw-r--r--releng/create_release.xsh324
1 files changed, 324 insertions, 0 deletions
diff --git a/releng/create_release.xsh b/releng/create_release.xsh
new file mode 100644
index 000000000..c57a92b2f
--- /dev/null
+++ b/releng/create_release.xsh
@@ -0,0 +1,324 @@
+import json
+import subprocess
+import itertools
+import textwrap
+from pathlib import Path
+import tempfile
+import hashlib
+import datetime
+from . import environment
+from . import keys
+from .version import VERSION, RELEASE_NAME, MAJOR
+
+$RAISE_SUBPROC_ERROR = True
+$XONSH_SHOW_TRACEBACK = True
+
+RELENG_ENV = environment.STAGING
+
+RELEASES_BUCKET = RELENG_ENV.releases_bucket
+DOCS_BUCKET = RELENG_ENV.docs_bucket
+CACHE_STORE = RELENG_ENV.cache_store_uri()
+REPO = RELENG_ENV.git_repo
+
+GCROOTS_DIR = Path('./release/gcroots')
+BUILT_GCROOTS_DIR = Path('./release/gcroots-build')
+DRVS_TXT = Path('./release/drvs.txt')
+ARTIFACTS = Path('./release/artifacts')
+MANUAL = Path('./release/manual')
+
+RELENG_MSG = "Release created with releng/create_release.xsh"
+
+BUILD_CORES = 16
+MAX_JOBS = 2
+
+# TODO
+RELEASE_SYSTEMS = ["x86_64-linux"]
+
+
+def setup_creds():
+ key = keys.get_ephemeral_key(RELENG_ENV)
+ $AWS_SECRET_ACCESS_KEY = key.secret_key
+ $AWS_ACCESS_KEY_ID = key.id
+ $AWS_DEFAULT_REGION = 'garage'
+ $AWS_ENDPOINT_URL = environment.S3_ENDPOINT
+
+
+def git_preconditions():
+ # verify there is nothing in index ready to stage
+ proc = !(git diff-index --quiet --cached HEAD --)
+ assert proc.rtn == 0
+ # verify there is nothing *stageable* and tracked
+ proc = !(git diff-files --quiet)
+ assert proc.rtn == 0
+
+
+def official_release_commit_tag(force_tag=False):
+ print('[+] Setting officialRelease in flake.nix and tagging')
+ prev_branch = $(git symbolic-ref --short HEAD).strip()
+
+ git switch --detach
+ sed -i 's/officialRelease = false/officialRelease = true/' flake.nix
+ git add flake.nix
+ message = f'release: {VERSION} "{RELEASE_NAME}"\n\nRelease produced with releng/create_release.xsh'
+ git commit -m @(message)
+ git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION)
+
+ return prev_branch
+
+
+def merge_to_release(prev_branch):
+ git switch @(prev_branch)
+ # Create a merge back into the release branch so that git tools understand
+ # that the release branch contains the tag, without the release commit
+ # actually influencing the tree.
+ merge_msg = textwrap.dedent("""\
+ release: merge release {VERSION} back to mainline
+
+ This merge commit returns to the previous state prior to the release but leaves the tag in the branch history.
+ {RELENG_MSG}
+ """).format(VERSION=VERSION, RELENG_MSG=RELENG_MSG)
+ git merge -m @(merge_msg) -s ours @(VERSION)
+
+
+def realise(paths: list[str]):
+ args = [
+ '--realise',
+ '--max-jobs',
+ MAX_JOBS,
+ '--cores',
+ BUILD_CORES,
+ '--log-format',
+ 'bar-with-logs',
+ '--add-root',
+ BUILT_GCROOTS_DIR
+ ]
+ nix-store @(args) @(paths)
+
+
+def eval_jobs():
+ nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake '.#release-jobs')
+ return [x for x in (json.loads(s) for s in nej_output.strip().split('\n'))
+ if x['system'] in RELEASE_SYSTEMS
+ ]
+
+
+def upload_drv_paths_and_outputs(paths: list[str]):
+ proc = subprocess.Popen([
+ 'nix',
+ 'copy',
+ '-v',
+ '--to',
+ CACHE_STORE,
+ '--stdin',
+ ],
+ stdin=subprocess.PIPE,
+ env=__xonsh__.env.detype(),
+ )
+
+ proc.stdin.write('\n'.join(itertools.chain(paths, x + '^*' for x in paths)).encode())
+ proc.stdin.close()
+ rv = proc.wait()
+ if rv != 0:
+ raise subprocess.CalledProcessError(rv, proc.args)
+
+
+def make_manifest(eval_result):
+ manifest = {vs['system']: vs['outputs']['out'] for vs in eval_result}
+ def manifest_line(system, out):
+ return f' {system} = "{out}";'
+
+ manifest_text = textwrap.dedent("""\
+ # This file was generated by releng/create_release.xsh in Lix
+ {{
+ {lines}
+ }}
+ """).format(lines='\n'.join(manifest_line(s, p) for (s, p) in manifest.items()))
+
+ return manifest_text
+
+
+def make_git_tarball(to: Path):
+ git archive --verbose --prefix=lix-@(VERSION)/ --format=tar.gz -o @(to) @(VERSION)
+
+
+def confirm(prompt, expected):
+ resp = input(prompt)
+
+ if resp != expected:
+ raise ValueError('Unconfirmed')
+
+
+def sha256_file(f: Path):
+ hasher = hashlib.sha256()
+
+ with open(f, 'rb') as h:
+ while data := h.read(1024 * 1024):
+ hasher.update(data)
+
+ return hasher.hexdigest()
+
+
+def make_artifacts_dir(eval_result, d: Path):
+ d.mkdir(exist_ok=True, parents=True)
+ version_dir = d / 'lix' / f'lix-{VERSION}'
+ version_dir.mkdir(exist_ok=True, parents=True)
+
+ tarballs_drv = next(p for p in eval_result if p['attr'] == 'tarballs')
+ cp --no-preserve=mode -r @(tarballs_drv['outputs']['out'])/* @(version_dir)
+
+ # FIXME: upgrade-nix searches for manifest.nix at root, which is rather annoying
+ with open(d / 'manifest.nix', 'w') as h:
+ h.write(make_manifest(eval_result))
+
+ with open(version_dir / 'manifest.nix', 'w') as h:
+ h.write(make_manifest(eval_result))
+
+ print('[+] Make sources tarball')
+
+ filename = f'lix-{VERSION}.tar.gz'
+ git_tarball = version_dir / filename
+ make_git_tarball(git_tarball)
+
+ file_hash = sha256_file(git_tarball)
+
+ print(f'Hash: {file_hash}')
+ with open(version_dir / f'{filename}.sha256', 'w') as h:
+ h.write(file_hash)
+
+
+def prepare_release_notes():
+ print('[+] Preparing release notes')
+ RELEASE_NOTES_PATH = Path('doc/manual/rl-next')
+
+ if RELEASE_NOTES_PATH.isdir():
+ notes_body = subprocess.check_output(['build-release-notes', '--change-authors', 'doc/manual/change-authors.yml', 'doc/manual/rl-next']).decode()
+ else:
+ # I guess nobody put release notes on their changes?
+ print('[-] Warning: seemingly missing any release notes, not worrying about it')
+ notes_body = ''
+
+ rl_path = Path(f'doc/manual/src/release-notes/rl-{MAJOR}.md')
+
+ existing_rl = ''
+ try:
+ with open(rl_path, 'r') as fh:
+ existing_rl = fh.read()
+ except FileNotFoundError:
+ pass
+
+ date = datetime.datetime.now().strftime('%Y-%m-%d')
+
+ minor_header = f'# Lix {VERSION} ({date})'
+
+ header = f'# Lix {MAJOR} "{RELEASE_NAME}"'
+ if existing_rl.startswith(header):
+ # strip the header off for minor releases
+ lines = existing_rl.splitlines()
+ header = lines[0]
+ existing_rl = '\n'.join(lines[1:])
+ else:
+ header += f' ({date})\n\n'
+
+ header += '\n' + minor_header + '\n'
+
+ notes = header
+ notes += notes_body
+ notes += "\n\n"
+ notes += existing_rl
+
+ # make pre-commit happy about one newline
+ notes = notes.rstrip()
+ notes += "\n"
+
+ with open(rl_path, 'w') as fh:
+ fh.write(notes)
+
+ commit_msg = textwrap.dedent("""\
+ release: release notes for {VERSION}
+
+ {RELENG_MSG}
+ """).format(VERSION=VERSION, RELENG_MSG=RELENG_MSG)
+
+ git add @(rl_path)
+ git rm doc/manual/rl-next/*.md
+
+ git commit -m @(commit_msg)
+
+
+def verify_are_on_tag():
+ current_tag = $(git describe --tag).strip()
+ assert current_tag == VERSION
+
+
+def upload_artifacts(noconfirm=False, force_push_tag=False):
+ assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env
+
+ tree @(ARTIFACTS)
+
+ not noconfirm and confirm(
+ f'Would you like to release {ARTIFACTS} as {VERSION}? Type "I want to release this" to confirm\n',
+ 'I want to release this'
+ )
+
+ print('[+] Upload to cache')
+ with open(DRVS_TXT) as fh:
+ upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x])
+
+
+ print('[+] Upload to release bucket')
+ aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/
+ print('[+] Upload manual')
+ upload_manual()
+
+ print('[+] git push tag')
+ git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}'
+
+
+def do_tag_merge(force_tag=False, no_check_git=False):
+ if not no_check_git:
+ git_preconditions()
+ prev_branch = official_release_commit_tag(force_tag=force_tag)
+ merge_to_release(prev_branch)
+ git switch --detach @(VERSION)
+
+
+def build_manual(eval_result):
+ manual = next(x['outputs']['doc'] for x in eval_result if x['attr'] == 'build.x86_64-linux')
+ print('[+] Building manual')
+ realise([manual])
+
+ cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL)
+
+
+def upload_manual():
+ stable = json.loads($(nix eval --json '.#nix.officialRelease'))
+ if stable:
+ version = MAJOR
+ else:
+ version = 'nightly'
+
+ print('[+] aws s3 sync manual')
+ aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/
+ if stable:
+ aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/
+
+
+def build_artifacts(no_check_git=False):
+ rm -rf release/
+ if not no_check_git:
+ verify_are_on_tag()
+ git_preconditions()
+
+ print('[+] Evaluating')
+ eval_result = eval_jobs()
+ drv_paths = [x['drvPath'] for x in eval_result]
+
+ print('[+] Building')
+ realise(drv_paths)
+ build_manual(eval_result)
+
+ with open(DRVS_TXT, 'w') as fh:
+ fh.write('\n'.join(drv_paths))
+
+ make_artifacts_dir(eval_result, ARTIFACTS)
+ print(f'[+] Done! See {ARTIFACTS}')
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 23:00:33 +0000