From 85d46c52c93405b15a046b842da0ef6494046087 Mon Sep 17 00:00:00 2001
From: Simon McVittie <smcv@collabora.com>
Date: Mon, 7 Jun 2021 17:43:43 +0100
Subject: [PATCH] CI: Add infrastructure to upload pressure-vessel releases

Signed-off-by: Simon McVittie <smcv@collabora.com>
---
 debian/gitlab-ci.yml      |  59 ++++++++
 pressure-vessel/deploy.py | 284 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 343 insertions(+)
 create mode 100755 pressure-vessel/deploy.py

diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml
index 5bb9a71f1..9bf635f79 100644
--- a/debian/gitlab-ci.yml
+++ b/debian/gitlab-ci.yml
@@ -52,10 +52,18 @@ variables:
 
     BUILD_IMAGE: '${SCOUT_DOCKER_REGISTRY}/${SCOUT_DOCKER_IMAGE}'
 
+    PRESSURE_VESSEL_CI_UPLOAD_HOST: ''
+    PRESSURE_VESSEL_CI_UPLOAD_HOST_SSH_PUBLIC_KEYS_FILE: ''
+    PRESSURE_VESSEL_CI_UPLOAD_PATH: ''
+    PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH: ''
+    PRESSURE_VESSEL_CI_UPLOAD_USER: ''
+    PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY: ''
+
 stages:
     - build
     - relocatable-install
     - test
+    - deploy
 
 package:
     extends: .build_package
@@ -452,4 +460,55 @@ autopkgtest:
             ${NULL+}
             apt-get -y -f install
 
+deploy:
+    stage: deploy
+    needs:
+        - package
+        - package:i386
+        - relocatable-install:production
+    rules:
+        - if: '$DEVEL_DOCKER_REGISTRY == ""'
+          when: never
+        - if: '$DEVEL_DOCKER_IMAGE == ""'
+          when: never
+        - if: '$PRESSURE_VESSEL_CI_UPLOAD_HOST == ""'
+          when: never
+        - if: '$CI_COMMIT_TAG && $PRESSURE_VESSEL_CI_UPLOAD_PATH == ""'
+          when: never
+        - if: >-
+            $PRESSURE_VESSEL_CI_UPLOAD_PATH == ""
+            && $PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH == ""
+          when: never
+        - if: '$PRESSURE_VESSEL_CI_UPLOAD_USER == ""'
+          when: never
+        - if: '$PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE == ""'
+          when: never
+        - if: '$CI_COMMIT_TAG'
+          when: always
+        - when: manual
+    image: "${DEVEL_DOCKER_REGISTRY}/${DEVEL_DOCKER_IMAGE}"
+    variables:
+        STEAM_CI_DEPENDENCIES: >-
+            openssh-client
+    script:
+        - |
+            mkdir -p ~/.ssh
+            chmod 0700 ~/.ssh
+            cat "$PRESSURE_VESSEL_CI_UPLOAD_HOST_SSH_PUBLIC_KEYS_FILE" >> ~/.ssh/known_hosts
+            chmod 0644 ~/.ssh/known_hosts
+            eval "$(ssh-agent -s)"
+            tr -d '\r' < "$PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE" | ssh-add -
+
+            if [ -n "${CI_COMMIT_TAG-}" ]; then
+                path="$PRESSURE_VESSEL_CI_UPLOAD_PATH"
+            else
+                path="$PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH"
+            fi
+
+            python3.5 pressure-vessel/deploy.py \
+            --host="$PRESSURE_VESSEL_CI_UPLOAD_HOST" \
+            --path="$path" \
+            --login="$PRESSURE_VESSEL_CI_UPLOAD_USER" \
+            ${NULL+}
+
 # vim:set sw=4 sts=4 et:
diff --git a/pressure-vessel/deploy.py b/pressure-vessel/deploy.py
new file mode 100755
index 000000000..10ff1f1c8
--- /dev/null
+++ b/pressure-vessel/deploy.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+
+# Copyright © 2018-2021 Collabora Ltd.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import contextlib
+import logging
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import textwrap
+import typing
+from pathlib import Path
+
+
+logger = logging.getLogger('pressure-vessel.deploy')
+
+COMMAND = typing.Union[str, typing.List[str]]
+
+
+@contextlib.contextmanager
+def RemoteTemporaryDirectory(
+    ssh: typing.List[str],
+    parent: typing.Optional[str] = None,
+):
+    argv = ssh + [
+        'mktemp', '-d',
+    ]
+
+    if parent is not None:
+        argv.append('-p')
+        argv.append(parent)
+
+    tmpdir = subprocess.check_output(
+        argv,
+        universal_newlines=True,
+    ).strip('\n')
+
+    try:
+        yield tmpdir
+    finally:
+        subprocess.call(ssh + [
+            'rm', '-fr', tmpdir,
+        ])
+
+
+@contextlib.contextmanager
+def SshMaster(
+    ssh: typing.List[str],
+):
+    logger.debug('Opening persistent ssh connection...')
+    process = subprocess.Popen(ssh + [
+        '-M',
+        'cat',
+    ], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL)
+
+    try:
+        assert process.stdin is not None
+        with process.stdin:
+            yield
+    finally:
+        logger.debug('Closing persistent ssh connection...')
+        process.wait()
+        logger.debug('Closed persistent ssh connection')
+
+
+class Uploader:
+    def __init__(
+        self,
+        host: str,
+        path: str,
+        login: str,
+    ) -> None:
+        self.host = host
+        self.basedir = path
+        self.login = login
+
+        self.ssh_target = '{}@{}'.format(self.login, self.host)
+
+        self.stack = contextlib.ExitStack()
+        self.local_tmpdir = None    # type: typing.Optional[str]
+        self.ssh = ['false']        # type: typing.List[str]
+        self.remote_tmpdir = None   # type: typing.Optional[str]
+
+    def __enter__(self) -> 'Uploader':
+        self.local_tmpdir = self.stack.enter_context(
+            tempfile.TemporaryDirectory()
+        )
+        assert self.local_tmpdir is not None
+        self.ssh = [
+            'ssh',
+            '-oControlPath={}/socket'.format(self.local_tmpdir),
+            self.ssh_target,
+        ]
+        self.stack.enter_context(SshMaster(self.ssh))
+        self.remote_tmpdir = self.stack.enter_context(
+            RemoteTemporaryDirectory(self.ssh)
+        )
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.stack.__exit__(*exc)
+
+    def remote_command(
+        self,
+        command: COMMAND,
+        chdir=True,
+        shell=False,
+    ) -> str:
+        preamble = textwrap.dedent('''\
+            set -eu;
+            umask 0022;
+        ''')
+
+        if chdir:
+            preamble = preamble + 'cd {};\n'.format(shlex.quote(self.basedir))
+
+        if shell:
+            assert isinstance(command, str)
+            return preamble + command
+        else:
+            assert isinstance(command, list)
+            return preamble + ' '.join(map(shlex.quote, command))
+
+    def popen(self, command: COMMAND, chdir=True, shell=False, **kwargs):
+        logger.debug('remote: %s', command)
+        return subprocess.Popen(self.ssh + [
+            self.remote_command(command, chdir=chdir, shell=shell),
+        ], **kwargs)
+
+    def check_call(self, command: COMMAND, chdir=True, shell=False, **kwargs):
+        logger.debug('remote: %s', command)
+        subprocess.check_call(self.ssh + [
+            self.remote_command(command, chdir=chdir, shell=shell),
+        ], **kwargs)
+
+    def check_output(
+        self,
+        command: COMMAND,
+        chdir=True,
+        shell=False,
+        **kwargs
+    ):
+        logger.debug('remote: %s', command)
+        return subprocess.check_output(self.ssh + [
+            self.remote_command(command, chdir=chdir, shell=shell),
+        ], **kwargs)
+
+    def call(self, command: COMMAND, chdir=True, shell=False, **kwargs):
+        logger.debug('remote: %s', command)
+        return subprocess.call(self.ssh + [
+            self.remote_command(command, chdir=chdir, shell=shell),
+        ], **kwargs)
+
+    def run(self):
+        with self:
+            self.check_call([
+                'mkdir', '-p', self.basedir,
+            ], chdir=False)
+
+            self.upload()
+
+    def upload(self) -> None:
+        assert self.local_tmpdir is not None
+        assert self.remote_tmpdir is not None
+
+        description = subprocess.check_output(
+            ['git', 'describe', '--match=v*'],
+            universal_newlines=True,
+        )
+        version = description.lstrip('v')
+
+        self.check_call([
+            'mkdir', version,
+        ], chdir=False)
+
+        upload = Path('_build', 'upload')
+        upload.mkdir()
+        sources = Path('_build', 'upload', 'sources')
+        sources.mkdir()
+
+        for a in Path('debian', 'tmp', 'artifacts', 'build').iterdir():
+            if str(a).endswith('.dsc'):
+                subprocess.check_call([
+                    'dcmd', 'ln', str(a), str(sources),
+                ])
+
+        for a in Path('_build', 'production').glob(
+            'pressure-vessel-*-bin.tar.gz'
+        ):
+            os.link(str(a), upload / a.name)
+
+        for a in Path('_build', 'production').glob(
+            'pressure-vessel-*-bin+src.tar.gz'
+        ):
+            with tarfile.open(str(a), 'r') as unarchiver:
+                for member in unarchiver:
+                    parts = member.name.split('/')
+
+                    if len(parts) >= 2 and parts[-2] == 'sources':
+                        extract = unarchiver.extractfile(member)
+                        assert extract is not None
+                        with extract:
+                            with open(
+                                str(sources / parts[-1]), 'wb'
+                            ) as writer:
+                                shutil.copyfileobj(extract, writer)
+
+        self.check_call(
+            'if [ -d latest ]; then cp -al latest/sources {}; fi'.format(
+                shlex.quote(version)
+            )
+        )
+        subprocess.check_call([
+            'rsync',
+            '--chmod=a+rX,og-w',
+            '--links',
+            '--partial',
+            '--perms',
+            '--recursive',
+            '_build/upload/',
+            '{}/{}/'.format(self.basedir, version),
+        ])
+
+        self.check_call([
+            'ln', '-fns', version, 'latest',
+        ])
+
+
+def main() -> None:
+    if sys.stderr.isatty():
+        try:
+            import colorlog
+        except ImportError:
+            logging.basicConfig()
+        else:
+            formatter = colorlog.ColoredFormatter(
+                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
+            handler = logging.StreamHandler()
+            handler.setFormatter(formatter)
+            logging.getLogger().addHandler(handler)
+    else:
+        logging.basicConfig()
+
+    logging.getLogger().setLevel(logging.DEBUG)
+
+    parser = argparse.ArgumentParser(
+        description='Upload a pressure-vessel release'
+    )
+
+    parser.add_argument('--host', required=True)
+    parser.add_argument('--path', required=True)
+    parser.add_argument('--login', required=True)
+    args = parser.parse_args()
+    Uploader(**vars(args)).run()
+
+
+if __name__ == '__main__':
+    main()
-- 
GitLab