From 85d46c52c93405b15a046b842da0ef6494046087 Mon Sep 17 00:00:00 2001 From: Simon McVittie <smcv@collabora.com> Date: Mon, 7 Jun 2021 17:43:43 +0100 Subject: [PATCH] CI: Add infrastructure to upload pressure-vessel releases Signed-off-by: Simon McVittie <smcv@collabora.com> --- debian/gitlab-ci.yml | 59 ++++++++ pressure-vessel/deploy.py | 284 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100755 pressure-vessel/deploy.py diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml index 5bb9a71f1..9bf635f79 100644 --- a/debian/gitlab-ci.yml +++ b/debian/gitlab-ci.yml @@ -52,10 +52,18 @@ variables: BUILD_IMAGE: '${SCOUT_DOCKER_REGISTRY}/${SCOUT_DOCKER_IMAGE}' + PRESSURE_VESSEL_CI_UPLOAD_HOST: '' + PRESSURE_VESSEL_CI_UPLOAD_HOST_SSH_PUBLIC_KEYS_FILE: '' + PRESSURE_VESSEL_CI_UPLOAD_PATH: '' + PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH: '' + PRESSURE_VESSEL_CI_UPLOAD_USER: '' + PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY: '' + stages: - build - relocatable-install - test + - deploy package: extends: .build_package @@ -452,4 +460,55 @@ autopkgtest: ${NULL+} apt-get -y -f install +deploy: + stage: deploy + needs: + - package + - package:i386 + - relocatable-install:production + rules: + - if: '$DEVEL_DOCKER_REGISTRY == ""' + when: never + - if: '$DEVEL_DOCKER_IMAGE == ""' + when: never + - if: '$PRESSURE_VESSEL_CI_UPLOAD_HOST == ""' + when: never + - if: '$CI_COMMIT_TAG && $PRESSURE_VESSEL_CI_UPLOAD_PATH == ""' + when: never + - if: >- + $PRESSURE_VESSEL_CI_UPLOAD_PATH == "" + && $PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH == "" + when: never + - if: '$PRESSURE_VESSEL_CI_UPLOAD_USER == ""' + when: never + - if: '$PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE == ""' + when: never + - if: '$CI_COMMIT_TAG' + when: always + - when: manual + image: "${DEVEL_DOCKER_REGISTRY}/${DEVEL_DOCKER_IMAGE}" + variables: + STEAM_CI_DEPENDENCIES: >- + openssh-client + script: + - | + mkdir -p ~/.ssh + chmod 0700 ~/.ssh + cat "$PRESSURE_VESSEL_CI_UPLOAD_HOST_SSH_PUBLIC_KEYS_FILE" >> ~/.ssh/known_hosts + chmod 0644 ~/.ssh/known_hosts + eval "$(ssh-agent -s)" + tr -d '\r' < "$PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE" | ssh-add - + + if [ -n "${CI_COMMIT_TAG-}" ]; then + path="$PRESSURE_VESSEL_CI_UPLOAD_PATH" + else + path="$PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH" + fi + + python3.5 pressure-vessel/deploy.py \ + --host="$PRESSURE_VESSEL_CI_UPLOAD_HOST" \ + --path="$path" \ + --login="$PRESSURE_VESSEL_CI_UPLOAD_USER" \ + ${NULL+} + # vim:set sw=4 sts=4 et: diff --git a/pressure-vessel/deploy.py b/pressure-vessel/deploy.py new file mode 100755 index 000000000..10ff1f1c8 --- /dev/null +++ b/pressure-vessel/deploy.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 + +# Copyright © 2018-2021 Collabora Ltd. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import argparse +import contextlib +import logging +import os +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import textwrap +import typing +from pathlib import Path + + +logger = logging.getLogger('pressure-vessel.deploy') + +COMMAND = typing.Union[str, typing.List[str]] + + +@contextlib.contextmanager +def RemoteTemporaryDirectory( + ssh: typing.List[str], + parent: typing.Optional[str] = None, +): + argv = ssh + [ + 'mktemp', '-d', + ] + + if parent is not None: + argv.append('-p') + argv.append(parent) + + tmpdir = subprocess.check_output( + argv, + universal_newlines=True, + ).strip('\n') + + try: + yield tmpdir + finally: + subprocess.call(ssh + [ + 'rm', '-fr', tmpdir, + ]) + + +@contextlib.contextmanager +def SshMaster( + ssh: typing.List[str], +): + logger.debug('Opening persistent ssh connection...') + process = subprocess.Popen(ssh + [ + '-M', + 'cat', + ], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL) + + try: + assert process.stdin is not None + with process.stdin: + yield + finally: + logger.debug('Closing persistent ssh connection...') + process.wait() + logger.debug('Closed persistent ssh connection') + + +class Uploader: + def __init__( + self, + host: str, + path: str, + login: str, + ) -> None: + self.host = host + self.basedir = path + self.login = login + + self.ssh_target = '{}@{}'.format(self.login, self.host) + + self.stack = contextlib.ExitStack() + self.local_tmpdir = None # type: typing.Optional[str] + self.ssh = ['false'] # type: typing.List[str] + self.remote_tmpdir = None # type: typing.Optional[str] + + def __enter__(self) -> 'Uploader': + self.local_tmpdir = self.stack.enter_context( + tempfile.TemporaryDirectory() + ) + assert self.local_tmpdir is not None + self.ssh = [ + 'ssh', + '-oControlPath={}/socket'.format(self.local_tmpdir), + self.ssh_target, + ] + self.stack.enter_context(SshMaster(self.ssh)) + self.remote_tmpdir = self.stack.enter_context( + RemoteTemporaryDirectory(self.ssh) + ) + return self + + def __exit__(self, *exc) -> None: + self.stack.__exit__(*exc) + + def remote_command( + self, + command: COMMAND, + chdir=True, + shell=False, + ) -> str: + preamble = textwrap.dedent('''\ + set -eu; + umask 0022; + ''') + + if chdir: + preamble = preamble + 'cd {};\n'.format(shlex.quote(self.basedir)) + + if shell: + assert isinstance(command, str) + return preamble + command + else: + assert isinstance(command, list) + return preamble + ' '.join(map(shlex.quote, command)) + + def popen(self, command: COMMAND, chdir=True, shell=False, **kwargs): + logger.debug('remote: %s', command) + return subprocess.Popen(self.ssh + [ + self.remote_command(command, chdir=chdir, shell=shell), + ], **kwargs) + + def check_call(self, command: COMMAND, chdir=True, shell=False, **kwargs): + logger.debug('remote: %s', command) + subprocess.check_call(self.ssh + [ + self.remote_command(command, chdir=chdir, shell=shell), + ], **kwargs) + + def check_output( + self, + command: COMMAND, + chdir=True, + shell=False, + **kwargs + ): + logger.debug('remote: %s', command) + return subprocess.check_output(self.ssh + [ + self.remote_command(command, chdir=chdir, shell=shell), + ], **kwargs) + + def call(self, command: COMMAND, chdir=True, shell=False, **kwargs): + logger.debug('remote: %s', command) + return subprocess.call(self.ssh + [ + self.remote_command(command, chdir=chdir, shell=shell), + ], **kwargs) + + def run(self): + with self: + self.check_call([ + 'mkdir', '-p', self.basedir, + ], chdir=False) + + self.upload() + + def upload(self) -> None: + assert self.local_tmpdir is not None + assert self.remote_tmpdir is not None + + description = subprocess.check_output( + ['git', 'describe', '--match=v*'], + universal_newlines=True, + ) + version = description.lstrip('v') + + self.check_call([ + 'mkdir', version, + ], chdir=False) + + upload = Path('_build', 'upload') + upload.mkdir() + sources = Path('_build', 'upload', 'sources') + sources.mkdir() + + for a in Path('debian', 'tmp', 'artifacts', 'build').iterdir(): + if str(a).endswith('.dsc'): + subprocess.check_call([ + 'dcmd', 'ln', str(a), str(sources), + ]) + + for a in Path('_build', 'production').glob( + 'pressure-vessel-*-bin.tar.gz' + ): + os.link(str(a), upload / a.name) + + for a in Path('_build', 'production').glob( + 'pressure-vessel-*-bin+src.tar.gz' + ): + with tarfile.open(str(a), 'r') as unarchiver: + for member in unarchiver: + parts = member.name.split('/') + + if len(parts) >= 2 and parts[-2] == 'sources': + extract = unarchiver.extractfile(member) + assert extract is not None + with extract: + with open( + str(sources / parts[-1]), 'wb' + ) as writer: + shutil.copyfileobj(extract, writer) + + self.check_call( + 'if [ -d latest ]; then cp -al latest/sources {}; fi'.format( + shlex.quote(version) + ) + ) + subprocess.check_call([ + 'rsync', + '--chmod=a+rX,og-w', + '--links', + '--partial', + '--perms', + '--recursive', + '_build/upload/', + '{}/{}/'.format(self.basedir, version), + ]) + + self.check_call([ + 'ln', '-fns', version, 'latest', + ]) + + +def main() -> None: + if sys.stderr.isatty(): + try: + import colorlog + except ImportError: + logging.basicConfig() + else: + formatter = colorlog.ColoredFormatter( + '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s') + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logging.getLogger().addHandler(handler) + else: + logging.basicConfig() + + logging.getLogger().setLevel(logging.DEBUG) + + parser = argparse.ArgumentParser( + description='Upload a pressure-vessel release' + ) + + parser.add_argument('--host', required=True) + parser.add_argument('--path', required=True) + parser.add_argument('--login', required=True) + args = parser.parse_args() + Uploader(**vars(args)).run() + + +if __name__ == '__main__': + main() -- GitLab