diff options
| -rw-r--r-- | package.nix | 3 | ||||
| -rw-r--r-- | releng/__init__.py | 12 | ||||
| -rw-r--r-- | releng/cli.py | 28 | ||||
| -rw-r--r-- | releng/create_release.xsh | 41 | ||||
| -rw-r--r-- | releng/docker.xsh | 13 | ||||
| -rw-r--r-- | releng/environment.py | 32 |
6 files changed, 96 insertions, 33 deletions
diff --git a/package.nix b/package.nix index 9c108308d..1ac54dab6 100644 --- a/package.nix +++ b/package.nix @@ -39,6 +39,7 @@ pkg-config, python3, rapidcheck, + skopeo, sqlite, toml11, util-linuxMinimal ? utillinuxMinimal, @@ -447,6 +448,8 @@ stdenv.mkDerivation (finalAttrs: { lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools_llvm ++ [ pythonEnv + # docker image tool + skopeo just nixfmt # Load-bearing order. Must come before clang-unwrapped below, but after clang_tools above. diff --git a/releng/__init__.py b/releng/__init__.py index 401b8e322..39d2beb51 100644 --- a/releng/__init__.py +++ b/releng/__init__.py @@ -2,11 +2,12 @@ from xonsh.main import setup setup() del setup -from releng import environment -from releng import create_release -from releng import keys -from releng import version -from releng import cli +from . import environment +from . import create_release +from . import keys +from . import version +from . import cli +from . import docker def reload(): import importlib @@ -15,3 +16,4 @@ def reload(): importlib.reload(keys) importlib.reload(version) importlib.reload(cli) + importlib.reload(docker) diff --git a/releng/cli.py b/releng/cli.py index bba50f534..89391e0a7 100644 --- a/releng/cli.py +++ b/releng/cli.py @@ -1,4 +1,8 @@ from . import create_release +from . import docker +from .environment import RelengEnvironment +from . import environment +import functools import argparse import sys @@ -18,13 +22,16 @@ def do_tag(args): no_check_git=args.no_check_git) -def do_upload(args): - create_release.setup_creds() +def do_upload(env: RelengEnvironment, args): + create_release.setup_creds(env) if args.target == 'all': - create_release.upload_artifacts(force_push_tag=args.force_push_tag, - noconfirm=args.noconfirm) + docker.check_all_logins(env) + create_release.upload_artifacts(env, + force_push_tag=args.force_push_tag, + noconfirm=args.noconfirm, + no_check_git=args.no_check_git) elif args.target == 'manual': - create_release.upload_manual() + create_release.upload_manual(env) else: raise ValueError('invalid target, unreachable') @@ -77,6 +84,10 @@ def main(): upload = sps.add_parser( 'upload', help='Upload artifacts to cache and releases bucket') + upload.add_argument( + '--no-check-git', + action='store_true', + help="Don't check git state before uploading. For testing.") upload.add_argument('--force-push-tag', action='store_true', help='Force push the tag. For testing.') @@ -90,7 +101,12 @@ def main(): '--noconfirm', action='store_true', help="Don't ask for confirmation. For testing/automation.") - upload.set_defaults(cmd=do_upload) + upload.add_argument('--environment', + choices=list(environment.ENVIRONMENTS.keys()), + default='staging', + help='Environment to release to') + upload.set_defaults(cmd=lambda args: do_upload( + environment.ENVIRONMENTS[args.environment], args)) args = ap.parse_args() args.cmd(args) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index c57a92b2f..128edb63f 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -7,19 +7,14 @@ import tempfile import hashlib import datetime from . import environment +from .environment import RelengEnvironment from . import keys +from . import docker from .version import VERSION, RELEASE_NAME, MAJOR $RAISE_SUBPROC_ERROR = True $XONSH_SHOW_TRACEBACK = True -RELENG_ENV = environment.STAGING - -RELEASES_BUCKET = RELENG_ENV.releases_bucket -DOCS_BUCKET = RELENG_ENV.docs_bucket -CACHE_STORE = RELENG_ENV.cache_store_uri() -REPO = RELENG_ENV.git_repo - GCROOTS_DIR = Path('./release/gcroots') BUILT_GCROOTS_DIR = Path('./release/gcroots-build') DRVS_TXT = Path('./release/drvs.txt') @@ -35,8 +30,8 @@ MAX_JOBS = 2 RELEASE_SYSTEMS = ["x86_64-linux"] -def setup_creds(): - key = keys.get_ephemeral_key(RELENG_ENV) +def setup_creds(env: RelengEnvironment): + key = keys.get_ephemeral_key(env) $AWS_SECRET_ACCESS_KEY = key.secret_key $AWS_ACCESS_KEY_ID = key.id $AWS_DEFAULT_REGION = 'garage' @@ -102,13 +97,13 @@ def eval_jobs(): ] -def upload_drv_paths_and_outputs(paths: list[str]): +def upload_drv_paths_and_outputs(env: RelengEnvironment, paths: list[str]): proc = subprocess.Popen([ 'nix', 'copy', '-v', '--to', - CACHE_STORE, + env.cache_store_uri(), '--stdin', ], stdin=subprocess.PIPE, @@ -250,7 +245,10 @@ def verify_are_on_tag(): assert current_tag == VERSION -def upload_artifacts(noconfirm=False, force_push_tag=False): +def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False, force_push_tag=False): + if not no_check_git: + verify_are_on_tag() + git_preconditions() assert 'AWS_SECRET_ACCESS_KEY' in __xonsh__.env tree @(ARTIFACTS) @@ -262,16 +260,21 @@ def upload_artifacts(noconfirm=False, force_push_tag=False): print('[+] Upload to cache') with open(DRVS_TXT) as fh: - upload_drv_paths_and_outputs([x.strip() for x in fh.readlines() if x]) + upload_drv_paths_and_outputs(env, [x.strip() for x in fh.readlines() if x]) + docker_images = (ARTIFACTS / f'lix/lix-{VERSION}').glob(f'lix-{VERSION}-docker-image-*.tar.gz') + print('[+] Upload docker images') + for image in docker_images: + for target in env.docker_targets: + docker.upload_docker_image(target, image) print('[+] Upload to release bucket') - aws s3 cp --recursive @(ARTIFACTS)/ @(RELEASES_BUCKET)/ + aws s3 cp --recursive @(ARTIFACTS)/ @(env.releases_bucket)/ print('[+] Upload manual') - upload_manual() + upload_manual(env) print('[+] git push tag') - git push @(['-f'] if force_push_tag else []) @(REPO) f'{VERSION}:refs/tags/{VERSION}' + git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' def do_tag_merge(force_tag=False, no_check_git=False): @@ -290,7 +293,7 @@ def build_manual(eval_result): cp --no-preserve=mode -vr @(manual)/share/doc/nix @(MANUAL) -def upload_manual(): +def upload_manual(env: RelengEnvironment): stable = json.loads($(nix eval --json '.#nix.officialRelease')) if stable: version = MAJOR @@ -298,9 +301,9 @@ def upload_manual(): version = 'nightly' print('[+] aws s3 sync manual') - aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/@(version)/ + aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/@(version)/ if stable: - aws s3 sync @(MANUAL)/ @(DOCS_BUCKET)/manual/lix/stable/ + aws s3 sync @(MANUAL)/ @(env.docs_bucket)/manual/lix/stable/ def build_artifacts(no_check_git=False): diff --git a/releng/docker.xsh b/releng/docker.xsh new file mode 100644 index 000000000..1ed2330cf --- /dev/null +++ b/releng/docker.xsh @@ -0,0 +1,13 @@ +from .environment import DockerTarget, RelengEnvironment +from .version import VERSION +from pathlib import Path + +def check_all_logins(env: RelengEnvironment): + for target in env.docker_targets: + check_login(target) + +def check_login(target: DockerTarget): + skopeo login @(target.registry_name()) + +def upload_docker_image(target: DockerTarget, path: Path): + skopeo --insecure-policy copy docker-archive:@(path) docker://@(target.resolve(version=VERSION)) diff --git a/releng/environment.py b/releng/environment.py index 58633d548..e8e7e771e 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -17,6 +17,21 @@ DEFAULT_STORE_URI_BITS = { @dataclasses.dataclass +class DockerTarget: + registry_path: str + + def resolve(self, version: str) -> str: + """Applies templates: + - version: the Lix version + """ + return self.registry_path.format(version=version) + + def registry_name(self) -> str: + [a, _, _] = self.registry_path.partition('/') + return a + + +@dataclasses.dataclass class RelengEnvironment: name: str @@ -26,22 +41,33 @@ class RelengEnvironment: docs_bucket: str git_repo: str + docker_targets: list[DockerTarget] + def cache_store_uri(self): qs = DEFAULT_STORE_URI_BITS.copy() qs.update(self.cache_store_overlay) return self.cache_bucket + "?" + urllib.parse.urlencode(qs) + STAGING = RelengEnvironment( name='staging', docs_bucket='s3://staging-docs', cache_bucket='s3://staging-cache', - cache_store_overlay={ - 'secret-key': 'staging.key' - }, + cache_store_overlay={'secret-key': 'staging.key'}, releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', + docker_targets=[ + DockerTarget( + 'git.lix.systems/lix-project/lix-releng-staging:{version}'), + DockerTarget( + 'ghcr.io/lix-project/lix-releng-staging:{version}'), + ], ) +ENVIRONMENTS = { + 'staging': STAGING, +} + @dataclasses.dataclass class S3Credentials: |