diff options
Diffstat (limited to 'releng/create_release.xsh')
| -rw-r--r-- | releng/create_release.xsh | 297 |
1 files changed, 297 insertions, 0 deletions
diff --git a/releng/create_release.xsh b/releng/create_release.xsh new file mode 100644 index 000000000..c9725c44c --- /dev/null +++ b/releng/create_release.xsh @@ -0,0 +1,297 @@ +import json +import subprocess +import itertools +import textwrap +from pathlib import Path +import tempfile +import hashlib +import datetime +from . import environment +from . import keys +from .version import VERSION, RELEASE_NAME, MAJOR + +$RAISE_SUBPROC_ERROR = True +$XONSH_SHOW_TRACEBACK = True + +RELENG_ENV = environment.STAGING + +RELEASES_BUCKET = RELENG_ENV.releases_bucket +CACHE_STORE = RELENG_ENV.cache_store_uri() +REPO = RELENG_ENV.git_repo + +GCROOTS_DIR = Path('./release/gcroots') +BUILT_GCROOTS_DIR = Path('./release/gcroots-build') +DRVS_TXT = Path('./release/drvs.txt') +ARTIFACTS = Path('./release/artifacts') + +RELENG_MSG = "Release created with releng/create_release.xsh" + +BUILD_CORES = 16 +MAX_JOBS = 2 + +# TODO +RELEASE_SYSTEMS = ["x86_64-linux"] + + +def setup_creds(): + key = keys.get_ephemeral_key(RELENG_ENV) + $AWS_SECRET_ACCESS_KEY = key.secret_key + $AWS_ACCESS_KEY_ID = key.id + $AWS_DEFAULT_REGION = 'garage' + $AWS_ENDPOINT_URL = environment.S3_ENDPOINT + + +def git_preconditions(): + # verify there is nothing in index ready to stage + proc = !(git diff-index --quiet --cached HEAD --) + assert proc.rtn == 0 + # verify there is nothing *stageable* and tracked + proc = !(git diff-files --quiet) + assert proc.rtn == 0 + + +def official_release_commit_tag(force_tag=False): + print('[+] Setting officialRelease in flake.nix and tagging') + prev_branch = $(git symbolic-ref --short HEAD).strip() + + git switch --detach + sed -i 's/officialRelease = false/officialRelease = true/' flake.nix + git add flake.nix + message = f'release: {VERSION} "{RELEASE_NAME}"\n\nRelease produced with releng/create_release.xsh' + git commit -m @(message) + git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION) + + return prev_branch + + +def merge_to_release(prev_branch): + git switch @(prev_branch) + # Create a merge back into the release branch so that git tools understand + # that the release branch contains the tag, without the release commit + # actually influencing the tree. + merge_msg = textwrap.dedent("""\ + release: merge release {VERSION} back to mainline + + This merge commit returns to the previous state prior to the release but leaves the tag in the branch history. + {RELENG_MSG} + """).format(VERSION=VERSION, RELENG_MSG=RELENG_MSG) + git merge -m @(merge_msg) -s ours @(VERSION) + + +def realise(paths: list[str]): + args = [ + '--realise', + '--max-jobs', + MAX_JOBS, + '--cores', + BUILD_CORES, + '--log-format', + 'bar-with-logs', + '--add-root', + BUILT_GCROOTS_DIR + ] + nix-store @(args) @(paths) + + +def eval_jobs(): + nej_output = $(nix-eval-jobs --workers 4 --gc-roots-dir @(GCROOTS_DIR) --force-recurse --flake '.#release-jobs') + return [x for x in (json.loads(s) for s in nej_output.strip().split('\n')) + if x['system'] in RELEASE_SYSTEMS + ] + + +def upload_drv_paths_and_outputs(paths: list[str]): + proc = subprocess.Popen([ + 'nix', + 'copy', + '-v', + '--to', + CACHE_STORE, + '--stdin', + ], + stdin=subprocess.PIPE, + env=__xonsh__.env.detype(), + ) + + proc.stdin.write('\n'.join(itertools.chain(paths, x + '^*' for x in paths)).encode()) + proc.stdin.close() + rv = proc.wait() + if rv != 0: + raise subprocess.CalledProcessError(rv, proc.args) + + +def make_manifest(eval_result): + manifest = {vs['system']: vs['outputs']['out'] for vs in eval_result} + def manifest_line(system, out): + return f' {system} = "{out}";' + + manifest_text = textwrap.dedent("""\ + # This file was generated by releng/create_release.xsh in Lix + {{ + {lines} + }} + """).format(lines='\n'.join(manifest_line(s, p) for (s, p) in manifest.items())) + + return manifest_text + + +def make_git_tarball(to: Path): + git archive --verbose --prefix=lix-@(VERSION)/ --format=tar.gz -o @(to) @(VERSION) + + +def confirm(prompt, expected): + resp = input(prompt) + + if resp != expected: + raise ValueError('Unconfirmed') + + +def sha256_file(f: Path): + hasher = hashlib.sha256() + + with open(f, 'rb') as h: + while data := h.read(1024 * 1024): + hasher.update(data) + + return hasher.hexdigest() + + +def make_artifacts_dir(eval_result, d: Path): + d.mkdir(exist_ok=True, parents=True) + version_dir = d / 'lix' / f'lix-{VERSION}' + version_dir.mkdir(exist_ok=True, parents=True) + + tarballs_drv = next(p for p in eval_result if p['attr'] == 'tarballs') + cp --no-preserve=mode -r @(tarballs_drv['outputs']['out'])/* @(version_dir) + + # FIXME: upgrade-nix searches for manifest.nix at root, which is rather annoying + with open(d / 'manifest.nix', 'w') as h: + h.write(make_manifest(eval_result)) + + with open(version_dir / 'manifest.nix', 'w') as h: + h.write(make_manifest(eval_result)) + + print('[+] Make sources tarball') + + filename = f'lix-{VERSION}.tar.gz' + git_tarball = version_dir / filename + make_git_tarball(git_tarball) + + file_hash = sha256_file(git_tarball) + + print(f'Hash: {file_hash}') + with open(version_dir / f'{filename}.sha256', 'w') as h: + h.write(file_hash) + + +def prepare_release_notes(): + print('[+] Preparing release notes') + RELEASE_NOTES_PATH = Path('doc/manual/rl-next') + + if RELEASE_NOTES_PATH.isdir(): + notes_body = subprocess.check_output(['build-release-notes', '--change-authors', 'doc/manual/change-authors.yml', 'doc/manual/rl-next']).decode() + else: + # I guess nobody put release notes on their changes? + print('[-] Warning: seemingly missing any release notes, not worrying about it') + notes_body = '' + + rl_path = Path(f'doc/manual/src/release-notes/rl-{MAJOR}.md') + + existing_rl = '' + try: + with open(rl_path, 'r') as fh: + existing_rl = fh.read() + except FileNotFoundError: + pass + + date = datetime.datetime.now().strftime('%Y-%m-%d') + + minor_header = f'# Lix {VERSION} ({date})' + + header = f'# Lix {MAJOR} "{RELEASE_NAME}"' + if existing_rl.startswith(header): + # strip the header off for minor releases + lines = existing_rl.splitlines() + header = lines[0] + existing_rl = '\n'.join(lines[1:]) + else: + header += f' ({date})\n\n' + + header += '\n' + minor_header + '\n' + + notes = header + notes += notes_body + notes += "\n\n" + notes += existing_rl + + # make pre-commit happy about one newline + notes = notes.rstrip() + notes += "\n" + + with open(rl_path, 'w') as fh: + fh.write(notes) + + commit_msg = textwrap.dedent("""\ + release: release notes for {VERSION} + + {RELENG_MSG} + """).format(VERSION=VERSION, RELENG_MSG=RELENG_MSG) + + git add @(rl_path) + git rm doc/manual/rl-next/*.md + + git commit -m @(commit_msg) + + +def verify_are_on_tag(): + current_tag = $(git describe --tag).strip() + assert current_tag == VERSION + + +def upload_artifacts(noconfirm=False, force_push_tag=False): + assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env + + tree @(ARTIFACTS) + + not noconfirm and confirm( + f'Would you like to release {ARTIFACTS} as {VERSION}? Type "I want to release this" to confirm\n', + 'I want to release this' + ) + + print('[+] Upload to cache') + with open(DRVS_TXT) as fh: + upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x]) + + + print('[+] Upload to release bucket') + aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/ + + print('[+] git push tag') + git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}' + + +def do_tag_merge(force_tag=False, no_check_git=False): + if not no_check_git: + git_preconditions() + prev_branch = official_release_commit_tag(force_tag=force_tag) + merge_to_release(prev_branch) + git switch --detach @(VERSION) + + +def build_artifacts(no_check_git=False): + if not no_check_git: + verify_are_on_tag() + git_preconditions() + + print('[+] Evaluating') + eval_result = eval_jobs() + drv_paths = [x['drvPath'] for x in eval_result] + + print('[+] Building') + realise(drv_paths) + + with open(DRVS_TXT, 'w') as fh: + fh.write('\n'.join(drv_paths)) + + make_artifacts_dir(eval_result, ARTIFACTS) + print(f'[+] Done! See {ARTIFACTS}') |