From a0bef6dfd31390d4f9b01ed7b0aec6e3d03933ca Mon Sep 17 00:00:00 2001
From: Simon McVittie <smcv@collabora.com>
Date: Mon, 7 Jun 2021 17:43:43 +0100
Subject: [PATCH] CI: Add infrastructure to upload pressure-vessel releases

Signed-off-by: Simon McVittie <smcv@collabora.com>
---
 debian/gitlab-ci.yml      |  69 ++++++++
 pressure-vessel/deploy.py | 336 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 405 insertions(+)
 create mode 100755 pressure-vessel/deploy.py

diff --git a/debian/gitlab-ci.yml b/debian/gitlab-ci.yml
index 5bb9a71f1..9dabcb001 100644
--- a/debian/gitlab-ci.yml
+++ b/debian/gitlab-ci.yml
@@ -52,10 +52,26 @@ variables:
 
     BUILD_IMAGE: '${SCOUT_DOCKER_REGISTRY}/${SCOUT_DOCKER_IMAGE}'
 
+    # These need to be configured in
+    # https://gitlab.steamos.cloud/groups/steamrt/-/settings/ci_cd
+    # Hostname of the machine that receives pressure-vessel releases
+    PRESSURE_VESSEL_CI_UPLOAD_HOST: ''
+    # Create a File variable with the public key(s) of P_V_CI_UPLOAD_HOST
+    PRESSURE_VESSEL_CI_UPLOAD_HOST_SSH_PUBLIC_KEYS_FILE: ''
+    # Path on P_V_CI_UPLOAD_HOST: /srv/VHOST/www/pressure-vessel/snapshots
+    PRESSURE_VESSEL_CI_UPLOAD_PATH: ''
+    # Similar path on P_V_CI_UPLOAD_HOST for unreleased test-builds
+    PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH: ''
+    # User to log in on P_V_CI_UPLOAD_HOST
+    PRESSURE_VESSEL_CI_UPLOAD_USER: ''
+    # Create a File variable with a private authorized for P_V_CI_UPLOAD_USER
+    PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE: ''
+
 stages:
     - build
     - relocatable-install
     - test
+    - deploy
 
 package:
     extends: .build_package
@@ -452,4 +468,57 @@ autopkgtest:
             ${NULL+}
             apt-get -y -f install
 
+deploy:
+    stage: deploy
+    tags:
+        - docker
+        - linux
+    needs:
+        - package
+        - package:i386
+        - relocatable-install:production
+    rules:
+        - if: '$DEVEL_DOCKER_REGISTRY == ""'
+          when: never
+        - if: '$DEVEL_DOCKER_IMAGE == ""'
+          when: never
+        - if: '$PRESSURE_VESSEL_CI_UPLOAD_HOST == ""'
+          when: never
+        - if: '$CI_COMMIT_TAG && $PRESSURE_VESSEL_CI_UPLOAD_PATH == ""'
+          when: never
+        - if: >-
+            $PRESSURE_VESSEL_CI_UPLOAD_PATH == ""
+            && $PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH == ""
+          when: never
+        - if: '$PRESSURE_VESSEL_CI_UPLOAD_USER == ""'
+          when: never
+        - if: '$PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE == ""'
+          when: never
+        - if: '$CI_COMMIT_TAG'
+          when: always
+        - when: manual
+    image: "${DEVEL_DOCKER_REGISTRY}/${DEVEL_DOCKER_IMAGE}"
+    variables:
+        STEAM_CI_DEPENDENCIES: >-
+            openssh-client
+    script:
+        - |
+            mkdir -p ~/.ssh
+            chmod 0700 ~/.ssh
+            cat "$PRESSURE_VESSEL_CI_UPLOAD_HOST_SSH_PUBLIC_KEYS_FILE" >> ~/.ssh/known_hosts
+            chmod 0644 ~/.ssh/known_hosts
+            tr -d '\r' < "$PRESSURE_VESSEL_CI_UPLOAD_SSH_PRIVATE_KEY_FILE" | ssh-add -
+
+            if [ -n "${CI_COMMIT_TAG-}" ]; then
+                path="$PRESSURE_VESSEL_CI_UPLOAD_PATH"
+            else
+                path="$PRESSURE_VESSEL_CI_UPLOAD_PLAYGROUND_PATH"
+            fi
+
+            python3.5 pressure-vessel/deploy.py \
+            --host="$PRESSURE_VESSEL_CI_UPLOAD_HOST" \
+            --path="$path" \
+            --login="$PRESSURE_VESSEL_CI_UPLOAD_USER" \
+            ${NULL+}
+
 # vim:set sw=4 sts=4 et:
diff --git a/pressure-vessel/deploy.py b/pressure-vessel/deploy.py
new file mode 100755
index 000000000..fabc82c11
--- /dev/null
+++ b/pressure-vessel/deploy.py
@@ -0,0 +1,336 @@
+#!/usr/bin/env python3
+
+# Copyright © 2018-2021 Collabora Ltd.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import contextlib
+import logging
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import textwrap
+import typing
+from pathlib import Path
+
+
+logger = logging.getLogger('pressure-vessel.deploy')
+
+COMMAND = typing.Union[str, typing.List[str]]
+
+
+@contextlib.contextmanager
+def RemoteTemporaryDirectory(
+    ssh: typing.List[str],
+    parent: typing.Optional[str] = None,
+):
+    argv = ssh + [
+        'mktemp', '-d',
+    ]
+
+    if parent is not None:
+        argv.append('-p')
+        argv.append(parent)
+
+    tmpdir = subprocess.check_output(
+        argv,
+        universal_newlines=True,
+    ).strip('\n')
+
+    try:
+        yield tmpdir
+    finally:
+        subprocess.call(ssh + [
+            'rm', '-fr', tmpdir,
+        ])
+
+
+@contextlib.contextmanager
+def SshAgent(tempdir: str):
+    logger.debug('Opening ssh agent...')
+    process = subprocess.Popen([
+        'ssh-agent',
+        '-D',
+        '-a', os.path.join(tempdir, 'ssh.sock'),
+    ], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
+
+    try:
+        yield os.path.join(tempdir, 'ssh.sock')
+    finally:
+        logger.debug('Closing ssh agent...')
+        process.kill()
+        process.wait()
+        logger.debug('Closed ssh agent')
+
+
+@contextlib.contextmanager
+def SshMaster(
+    ssh: typing.List[str],
+    ssh_agent_socket: typing.Optional[str] = None,
+):
+    logger.debug('Opening persistent ssh connection...')
+
+    env = dict(os.environ)
+
+    if ssh_agent_socket is not None:
+        env['SSH_AUTH_SOCK'] = ssh_agent_socket
+
+    process = subprocess.Popen(
+        ssh + ['-M', 'cat'],
+        env=env,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.DEVNULL,
+    )
+
+    try:
+        assert process.stdin is not None
+        with process.stdin:
+            yield
+    finally:
+        logger.debug('Closing persistent ssh connection...')
+        process.wait()
+        logger.debug('Closed persistent ssh connection')
+
+
+class Uploader:
+    def __init__(
+        self,
+        host: str,
+        path: str,
+        login: str,
+        dry_run: bool,
+    ) -> None:
+        self.host = host
+        self.basedir = path
+        self.login = login
+        self.dry_run = dry_run
+
+        self.ssh_target = '{}@{}'.format(self.login, self.host)
+
+        self.stack = contextlib.ExitStack()
+        self.local_tmpdir = None    # type: typing.Optional[str]
+        self.ssh = ['false']        # type: typing.List[str]
+        self.remote_tmpdir = None   # type: typing.Optional[str]
+
+    def __enter__(self) -> 'Uploader':
+        self.local_tmpdir = self.stack.enter_context(
+            tempfile.TemporaryDirectory()
+        )
+        assert self.local_tmpdir is not None
+        self.ssh = [
+            'ssh',
+            '-oControlPath={}/socket'.format(self.local_tmpdir),
+            self.ssh_target,
+        ]
+        ssh_agent_socket = self.stack.enter_context(
+            SshAgent(self.local_tmpdir)
+        )
+        self.stack.enter_context(SshMaster(self.ssh, ssh_agent_socket))
+        self.remote_tmpdir = self.stack.enter_context(
+            RemoteTemporaryDirectory(self.ssh)
+        )
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.stack.__exit__(*exc)
+
+    def remote_command(
+        self,
+        command: COMMAND,
+        chdir=True,
+        shell=False,
+    ) -> str:
+        preamble = textwrap.dedent('''\
+            set -eu;
+            umask 0022;
+        ''')
+
+        if chdir:
+            preamble = preamble + 'cd {};\n'.format(shlex.quote(self.basedir))
+
+        if shell:
+            assert isinstance(command, str)
+            return preamble + command
+        else:
+            assert isinstance(command, list)
+            return preamble + ' '.join(map(shlex.quote, command))
+
+    def popen(self, command: COMMAND, chdir=True, shell=False, **kwargs):
+        logger.debug('remote: %s', command)
+        if self.dry_run:
+            return None
+        else:
+            return subprocess.Popen(self.ssh + [
+                self.remote_command(command, chdir=chdir, shell=shell),
+            ], **kwargs)
+
+    def check_call(
+        self,
+        command: COMMAND,
+        chdir=True,
+        shell=False,
+        **kwargs
+    ) -> None:
+        logger.debug('remote: %s', command)
+        if not self.dry_run:
+            subprocess.check_call(self.ssh + [
+                self.remote_command(command, chdir=chdir, shell=shell),
+            ], **kwargs)
+
+    def check_output(
+        self,
+        command: COMMAND,
+        chdir=True,
+        shell=False,
+        **kwargs
+    ):
+        logger.debug('remote: %s', command)
+        if self.dry_run:
+            return None
+        else:
+            return subprocess.check_output(self.ssh + [
+                self.remote_command(command, chdir=chdir, shell=shell),
+            ], **kwargs)
+
+    def call(self, command: COMMAND, chdir=True, shell=False, **kwargs):
+        logger.debug('remote: %s', command)
+        return subprocess.call(self.ssh + [
+            self.remote_command(command, chdir=chdir, shell=shell),
+        ], **kwargs)
+
+    def run(self):
+        with self:
+            self.check_call([
+                'mkdir', '-p', self.basedir,
+            ], chdir=False)
+
+            self.upload()
+
+    def upload(self) -> None:
+        assert self.local_tmpdir is not None
+        assert self.remote_tmpdir is not None
+
+        description = subprocess.check_output(
+            ['git', 'describe', '--match=v*'],
+            universal_newlines=True,
+        )
+        version = description.lstrip('v')
+
+        self.check_call([
+            'mkdir', version,
+        ], chdir=False)
+
+        upload = Path('_build', 'upload')
+        upload.mkdir()
+        sources = Path('_build', 'upload', 'sources')
+        sources.mkdir()
+
+        for a in Path('debian', 'tmp', 'artifacts', 'build').iterdir():
+            if str(a).endswith('.dsc'):
+                subprocess.check_call([
+                    'dcmd', 'ln', str(a), str(sources),
+                ])
+
+        for a in Path('_build', 'production').glob(
+            'pressure-vessel-*-bin.tar.gz'
+        ):
+            os.link(str(a), upload / a.name)
+
+        for a in Path('_build', 'production').glob(
+            'pressure-vessel-*-bin+src.tar.gz'
+        ):
+            with tarfile.open(str(a), 'r') as unarchiver:
+                for member in unarchiver:
+                    parts = member.name.split('/')
+
+                    if len(parts) >= 2 and parts[-2] == 'sources':
+                        extract = unarchiver.extractfile(member)
+                        assert extract is not None
+                        with extract:
+                            with open(
+                                str(sources / parts[-1]), 'wb'
+                            ) as writer:
+                                shutil.copyfileobj(extract, writer)
+
+        self.check_call(
+            'if [ -d latest ]; then cp -al latest/sources {}; fi'.format(
+                shlex.quote(version)
+            )
+        )
+
+        argv = [
+            'rsync',
+            '--chmod=a+rX,og-w',
+            '--links',
+            '--partial',
+            '--perms',
+            '--recursive',
+            '_build/upload/',
+            '{}/{}/'.format(self.basedir, version),
+        ]
+
+        if self.dry_run:
+            logger.info('Would run: %r', argv)
+        else:
+            subprocess.check_call(argv)
+
+        self.check_call([
+            'ln', '-fns', version, 'latest',
+        ])
+
+
+def main() -> None:
+    if sys.stderr.isatty():
+        try:
+            import colorlog
+        except ImportError:
+            logging.basicConfig()
+        else:
+            formatter = colorlog.ColoredFormatter(
+                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
+            handler = logging.StreamHandler()
+            handler.setFormatter(formatter)
+            logging.getLogger().addHandler(handler)
+    else:
+        logging.basicConfig()
+
+    logging.getLogger().setLevel(logging.DEBUG)
+
+    parser = argparse.ArgumentParser(
+        description='Upload a pressure-vessel release'
+    )
+
+    parser.add_argument('--host', required=True)
+    parser.add_argument('--path', required=True)
+    parser.add_argument('--login', required=True)
+    parser.add_argument('--dry-run', action='store_true')
+    args = parser.parse_args()
+    Uploader(**vars(args)).run()
+
+
+if __name__ == '__main__':
+    main()
-- 
GitLab