Skip to content
Snippets Groups Projects
Commit e3ebf1b9 authored by Simon McVittie's avatar Simon McVittie
Browse files

populate-depot: New script to download one or more runtimes


As we start to add more runtimes, we need to decouple the
SteamLinuxRuntime depot from any individual runtime. Instead of producing
a pre-populated depot as part of a scout build, we can clone this git
repository in some convenient place and run ./populate-depot.py to
download the necessary files.

Also add convenience Makefile targets to download scout and optionally
soldier - these are mostly here as manual tests and as executable
examples.

This partially resolves T17590 and T20721.

Signed-off-by: default avatarSimon McVittie <smcv@collabora.com>
parent 6981b373
No related branches found
No related tags found
No related merge requests found
......@@ -41,10 +41,10 @@ todo:
test:stretch:
stage: test
image: docker.internal.steamos.cloud/steamos/package-builder:stretch
script: make check
script: make check TESTS_ONLY=true
# The same, but for brewmaster and Python 3.4
test:brewmaster:
stage: test
image: docker.internal.steamos.cloud/steamos/package-builder:brewmaster
script: make check
script: make check TESTS_ONLY=true
all:
all: scout
scout:
./populate-depot.py --ssh --depot=depot scout='{"version": "latest-steam-client-main-branch"}'
soldier:
./populate-depot.py --ssh --depot=depot scout='{"version": "latest-steam-client-main-branch"}' soldier
clean:
rm -fr depot/pressure-vessel depot/com.valvesoftware.SteamRuntime.*
check:
prove -v tests/*.sh
#!/usr/bin/env python3
# Copyright © 2019-2020 Collabora Ltd.
#
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
Build the steam-container-runtime (aka SteamLinuxRuntime) depot, either
from just-built files or by downloading a previous build.
"""
import argparse
import json
import logging
import os
import shlex
import shutil
import subprocess
import tempfile
import urllib.request
from contextlib import suppress
from typing import (
Any,
Dict,
List,
Optional,
Sequence,
)
logger = logging.getLogger('populate-depot')
class InvocationError(Exception):
pass
class Runtime:
def __init__(
self,
name,
*,
suite: str,
architecture: str = 'amd64,i386',
path: Optional[str] = None,
version: str = 'latest',
) -> None:
self.architecture = architecture
self.name = name
self.path = path
self.suite = suite
self.version = version
self.pinned_version = None # type: Optional[str]
self.prefix = 'com.valvesoftware.SteamRuntime'
self.platform = self.prefix + '.Platform'
self.sdk = self.prefix + '.Sdk'
self.tarball = '{}-{}-{}-runtime.tar.gz'.format(
self.platform,
self.architecture,
self.suite,
)
self.build_id_file = '{}-{}-{}-buildid.txt'.format(
self.platform,
self.architecture,
self.suite,
)
def __str__(self) -> str:
return self.name
@classmethod
def from_details(
cls,
name: str,
details: Dict[str, Any],
default_architecture: str = 'amd64,i386',
default_suite: str = '',
default_version: str = 'latest',
):
return cls(
name,
architecture=details.get(
'architecture', default_architecture,
),
path=details.get('path', None),
suite=details.get('suite', default_suite or name),
version=details.get('version', default_version),
)
def get_uri(
self,
filename: str,
version: Optional[str] = None,
) -> str:
suite = self.suite
v = version or self.pinned_version or self.version
return (
f'https://images.steamos.cloud/steamrt-{suite}/'
f'snapshots/{v}/{filename}'
)
def get_ssh_path(
self,
filename: str,
version: Optional[str] = None,
) -> str:
suite = self.suite
v = version or self.pinned_version or self.version
return (
f'/srv/images.internal.steamos.cloud/www/steamrt-{suite}/'
f'snapshots/{v}/{filename}'
)
def fetch(
self,
filename: str,
destdir: str,
opener: urllib.request.OpenerDirector,
ssh: bool = False,
version: Optional[str] = None,
) -> None:
if ssh:
path = self.get_ssh_path(filename)
logger.info('Downloading %r...', path)
subprocess.run([
'rsync',
'--archive',
'--partial',
'--progress',
'images.internal.steamos.cloud:' + path,
os.path.join(destdir, filename),
], check=True)
else:
uri = self.get_uri(filename)
logger.info('Downloading %r...', uri)
with opener.open(uri) as response:
with open(os.path.join(destdir, filename), 'wb') as writer:
shutil.copyfileobj(response, writer)
def pin_version(
self,
opener: urllib.request.OpenerDirector,
ssh: bool = False,
) -> str:
pinned = self.pinned_version
if pinned is None:
if ssh:
path = self.get_ssh_path(filename='VERSION.txt')
logger.info('Determining version number from %r...', path)
pinned = subprocess.run([
'ssh', 'images.internal.steamos.cloud',
'cat {}'.format(shlex.quote(path)),
], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
else:
uri = self.get_uri(filename='VERSION.txt')
logger.info('Determining version number from %r...', uri)
with opener.open(uri) as response:
pinned = response.read().decode('utf-8').strip()
self.pinned_version = pinned
return pinned
class Main:
def __init__(
self,
architecture: str = 'amd64,i386',
credential_envs: Sequence[str] = (),
depot: str = 'depot',
pressure_vessel: str = 'scout',
runtimes: Sequence[str] = (),
ssh: bool = False,
suite: str = '',
unpack_ld_library_path: str = '',
version: str = 'latest',
**kwargs: Dict[str, Any],
) -> None:
openers: List[urllib.request.BaseHandler] = []
if not runtimes:
runtimes = ('scout',)
if credential_envs:
password_manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
for cred in credential_envs:
if ':' in cred:
username_env, password_env = cred.split(':', 1)
logger.info(
'Using username from $%s and password from $%s',
username_env, password_env)
username = os.environ[username_env]
password = os.environ[password_env]
else:
logger.info(
'Using username and password from $%s', cred)
username, password = os.environ[cred].split(':', 1)
password_manager.add_password(
None, # type: ignore
'https://images.steamos.cloud/',
username,
password,
)
openers.append(
urllib.request.HTTPBasicAuthHandler(password_manager)
)
self.opener = urllib.request.build_opener(*openers)
self.default_architecture = architecture
self.default_suite = suite
self.default_version = version
self.depot = os.path.abspath(depot)
self.pressure_vessel = pressure_vessel
self.runtimes = [] # type: List[Runtime]
self.ssh = ssh
self.unpack_ld_library_path = unpack_ld_library_path
for runtime in runtimes:
if '=' in runtime:
name, rhs = runtime.split('=', 1)
if rhs.startswith('{'):
details = json.loads(rhs)
else:
with open(rhs, 'rb') as reader:
details = json.load(reader)
else:
name = runtime
details = {}
self.runtimes.append(self.new_runtime(name, details))
def new_runtime(self, name: str, details: Dict[str, Any]) -> Runtime:
return Runtime.from_details(
name,
details,
default_architecture=self.default_architecture,
default_suite=self.default_suite,
default_version=self.default_version,
)
def run(self) -> None:
for runtime in self.runtimes:
if runtime.name == self.pressure_vessel:
logger.info(
'Downloading pressure-vessel from %s', runtime.name)
pressure_vessel_runtime = runtime
self.download_pressure_vessel(pressure_vessel_runtime)
break
else:
if self.pressure_vessel.startswith('{'):
logger.info(
'Downloading pressure-vessel using JSON from command-line')
pressure_vessel_runtime = self.new_runtime(
'scout', json.loads(self.pressure_vessel),
)
self.download_pressure_vessel(pressure_vessel_runtime)
elif os.path.isdir(self.pressure_vessel):
logger.info(
'Unpacking pressure-vessel from local directory %s',
self.pressure_vessel)
self.use_local_pressure_vessel(self.pressure_vessel)
pressure_vessel_runtime = self.new_runtime(
'scout', {'path': self.pressure_vessel},
)
else:
logger.info(
'Downloading pressure-vessel using JSON from %r',
self.pressure_vessel)
with open(self.pressure_vessel, 'rb') as reader:
details = json.load(reader)
pressure_vessel_runtime = self.new_runtime('scout', details)
self.download_pressure_vessel(pressure_vessel_runtime)
if self.unpack_ld_library_path:
logger.info(
'Downloading LD_LIBRARY_PATH Steam Runtime from same place '
'as pressure-vessel into %r',
self.unpack_ld_library_path)
self.download_scout_tarball(pressure_vessel_runtime)
for runtime in self.runtimes:
if runtime.path:
logger.info(
'Using Platform from local directory %r',
runtime.path)
self.use_local_platform(runtime)
else:
logger.info(
'Downloading Platform from %s',
runtime)
self.download_platform(runtime)
def use_local_pressure_vessel(self, path: str = '.') -> None:
os.makedirs(self.depot, exist_ok=True)
argv = [
'tar', '-C', self.depot, '-xvf',
os.path.join(path, 'pressure-vessel-bin.tar.gz'),
]
logger.info('%r', argv)
subprocess.run(argv, check=True)
def download_pressure_vessel(self, runtime: Runtime) -> None:
filename = 'pressure-vessel-bin.tar.gz'
runtime.pin_version(self.opener, ssh=self.ssh)
with tempfile.TemporaryDirectory(prefix='populate-depot.') as tmp:
runtime.fetch(filename, tmp, self.opener, ssh=self.ssh)
os.makedirs(self.depot, exist_ok=True)
subprocess.run(
[
'tar', '-C', self.depot, '-xvf',
os.path.join(tmp, filename),
],
check=True,
)
def use_local_platform(self, runtime: Runtime) -> None:
assert runtime.path
src = os.path.join(runtime.path, runtime.tarball)
dest = os.path.join(self.depot, runtime.tarball)
logger.info('Hard-linking local runtime %r to %r', src, dest)
with suppress(FileNotFoundError):
os.unlink(dest)
os.link(src, dest)
with open(
os.path.join(self.depot, runtime.build_id_file), 'w',
) as writer:
writer.write(f'{runtime.version}\n')
def download_platform(self, runtime: Runtime) -> None:
"""
Download a pre-prepared Platform from a previous container
runtime build.
"""
pinned = runtime.pin_version(self.opener, ssh=self.ssh)
runtime.fetch(runtime.tarball, self.depot, self.opener, ssh=self.ssh)
with open(
os.path.join(self.depot, runtime.build_id_file), 'w',
) as writer:
writer.write(f'{pinned}\n')
def download_scout_tarball(self, runtime: Runtime) -> None:
"""
Download a pre-prepared LD_LIBRARY_PATH Steam Runtime from a
previous scout build.
"""
filename = 'steam-runtime.tar.xz'
pinned = runtime.pin_version(self.opener, ssh=self.ssh)
logger.info('Downloading steam-runtime build %s', pinned)
os.makedirs(self.unpack_ld_library_path, exist_ok=True)
with tempfile.TemporaryDirectory(prefix='populate-depot.') as tmp:
runtime.fetch(filename, tmp, self.opener, ssh=self.ssh)
subprocess.run(
[
'tar', '-C', self.unpack_ld_library_path, '-xvf',
os.path.join(tmp, filename),
],
check=True,
)
def main() -> None:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
'--architecture', default='amd64,i386',
help=(
'Default dpkg architecture or comma-separated list of '
'architectures'
)
)
parser.add_argument(
'--suite', default='',
help=(
'Default suite to use if none is specified'
)
)
parser.add_argument(
'--version', default='latest',
help=(
'Default version to use if none is specified'
)
)
parser.add_argument(
'--credential-env',
action='append',
default=[],
dest='credential_envs',
help=(
'Environment variable to be evaluated for login:password, '
'or a pair of environment variables VAR1:VAR2 to be evaluated '
'for login and password respectively'
),
)
parser.add_argument(
'--ssh', default=False, action='store_true',
help='Use ssh and rsync to download files',
)
parser.add_argument(
'--depot', default='depot',
help=(
'Download runtimes into this existing directory'
)
)
parser.add_argument(
'--pressure-vessel', default='scout', metavar='NAME|PATH|DETAILS',
help=(
'Get pressure-vessel from the named runtime (default "scout"), '
'or from a runtime version given as a JSON object, '
'or from a given directory (use ./ to disambiguate if necessary).'
)
)
parser.add_argument(
'--unpack-ld-library-path', metavar='PATH', default='',
help=(
'Get the steam-runtime.tar.xz from the same place as '
'pressure-vessel and unpack it into the given PATH, '
'for use in regression testing.'
)
)
parser.add_argument(
'runtimes',
default=[],
metavar='NAME[="DETAILS"]',
nargs='*',
help=(
'Runtimes to download, in the form NAME or NAME="DETAILS". '
'DETAILS is a JSON object containing something like '
'{"path": "../prebuilt", "suite: "scout", "version": "latest", '
'"architecture": "amd64,i386"}, or the path to a file with '
'the same JSON object in. All JSON fields are optional.'
),
)
try:
args = parser.parse_args()
Main(**vars(args)).run()
except InvocationError as e:
parser.error(str(e))
if __name__ == '__main__':
main()
......@@ -9,10 +9,16 @@ set -u
export MYPYPATH="${PYTHONPATH:=$(pwd)}"
set --
if [ -z "${TESTS_ONLY-}" ]; then
set -- "$@" ./*.py
fi
set -- "$@" tests/depot/*.py
i=0
for script in \
tests/depot/*.py \
; do
for script in "$@"; do
i=$((i + 1))
if [ "x${MYPY:="$(command -v mypy || echo false)"}" = xfalse ]; then
echo "ok $i - $script # SKIP mypy not found"
......
......@@ -7,12 +7,18 @@
set -e
set -u
set --
if [ -z "${TESTS_ONLY-}" ]; then
set -- "$@" ./*.py
fi
set -- "$@" tests/depot/*.py
if [ "x${PYCODESTYLE:=pycodestyle}" = xfalse ] || \
[ -z "$(command -v "$PYCODESTYLE")" ]; then
echo "1..0 # SKIP pycodestyle not found"
elif "${PYCODESTYLE}" \
tests/depot/*.py \
>&2; then
elif "${PYCODESTYLE}" "$@" >&2; then
echo "1..1"
echo "ok 1 - $PYCODESTYLE reported no issues"
else
......
......@@ -7,12 +7,18 @@
set -e
set -u
set --
if [ -z "${TESTS_ONLY-}" ]; then
set -- "$@" ./*.py
fi
set -- "$@" tests/depot/*.py
if [ "x${PYFLAKES:=pyflakes3}" = xfalse ] || \
[ -z "$(command -v "$PYFLAKES")" ]; then
echo "1..0 # SKIP pyflakes3 not found"
elif "${PYFLAKES}" \
tests/depot/*.py \
>&2; then
elif "${PYFLAKES}" "$@" >&2; then
echo "1..1"
echo "ok 1 - $PYFLAKES reported no issues"
else
......
......@@ -10,9 +10,15 @@ set -u
n=0
fail=
for script in \
tests/depot/*.py \
; do
set --
if [ -z "${TESTS_ONLY-}" ]; then
set -- "$@" ./*.py
fi
set -- "$@" tests/depot/*.py
for script in "$@"; do
n=$(( n + 1 ))
if python3 "$script" --help >/dev/null; then
echo "ok $n - $script --help succeeded with python3"
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment