Skip to content
Snippets Groups Projects
Commit 5658072a authored by Simon McVittie's avatar Simon McVittie
Browse files

populate-depot.py: Update from SteamLinuxRuntime v0.20210806.0


Originally part or all of these commits:

* steamlinuxruntime@8b0b5a03
  "populate-depot: Remove unused support for loading JSON from a file"
* steamlinuxruntime@084f4e11
  "populate-depot: Add more explicit versions of --pressure-vessel"
* steamlinuxruntime@392878db
  "Add support for downloading a standalone pressure-vessel release"
* steamlinuxruntime@4e2a5628
  "scout-on-soldier: Fail if toolmanifest.vdf has not been reloaded"
* steamlinuxruntime@ae46edab
  "populate-depot: Use a standalone pressure-vessel release by default"
* steamlinuxruntime@b3f00ee2
  "populate-depot: Use latest scout build for LD_LIBRARY_PATH runtime"
* steamlinuxruntime@5eb43c40
  "populate-depot: Store LD_LIBRARY_PATH version in layered scout runtime"
* steamlinuxruntime@a10a407e
  "populate-depot: Have an overall version number for the depot as a whole"

all written by me.

Signed-off-by: default avatarSimon McVittie <smcv@collabora.com>
parent ab5c2f08
No related branches found
No related tags found
1 merge request!359populate-depot.py: Update from SteamLinuxRuntime v0.20210806.0
#!/usr/bin/env python3 #!/usr/bin/env python3
# Copyright © 2019-2020 Collabora Ltd. # Copyright © 2019-2021 Collabora Ltd.
# #
# SPDX-License-Identifier: MIT # SPDX-License-Identifier: MIT
# #
...@@ -53,6 +53,7 @@ from typing import ( ...@@ -53,6 +53,7 @@ from typing import (
Optional, Optional,
Sequence, Sequence,
Set, Set,
Tuple,
) )
from debian.deb822 import ( from debian.deb822 import (
...@@ -78,6 +79,10 @@ sys.path[:0] = [ ...@@ -78,6 +79,10 @@ sys.path[:0] = [
logger = logging.getLogger('populate-depot') logger = logging.getLogger('populate-depot')
DEFAULT_PRESSURE_VESSEL_URI = (
'https://repo.steampowered.com/pressure-vessel/snapshots'
)
DEFAULT_IMAGES_URI = ( DEFAULT_IMAGES_URI = (
'https://repo.steampowered.com/steamrt-images-SUITE/snapshots' 'https://repo.steampowered.com/steamrt-images-SUITE/snapshots'
) )
...@@ -87,6 +92,102 @@ class InvocationError(Exception): ...@@ -87,6 +92,102 @@ class InvocationError(Exception):
pass pass
class PressureVesselRelease:
def __init__(
self,
*,
cache: str = '.cache',
ssh_host: str = '',
ssh_path: str = '',
uri: str = DEFAULT_PRESSURE_VESSEL_URI,
version: str = ''
) -> None:
self.cache = cache
self.pinned_version = None # type: Optional[str]
self.ssh_host = ssh_host
self.ssh_path = ssh_path
self.uri = uri
self.version = version
def get_uri(
self,
filename: str,
version: Optional[str] = None,
) -> str:
uri = self.uri
v = version or self.pinned_version or self.version or 'latest'
return f'{uri}/{v}/{filename}'
def get_ssh_path(
self,
filename: str,
version: Optional[str] = None,
) -> str:
ssh_host = self.ssh_host
ssh_path = self.ssh_path
v = version or self.pinned_version or self.version or 'latest'
if not ssh_host or not ssh_path:
raise RuntimeError('ssh host/path not configured')
return f'{ssh_path}/{v}/{filename}'
def fetch(
self,
filename: str,
opener: urllib.request.OpenerDirector,
version: Optional[str] = None,
) -> str:
dest = os.path.join(self.cache, filename)
if self.ssh_host and self.ssh_path:
path = self.get_ssh_path(filename)
logger.info('Downloading %r...', path)
subprocess.run([
'rsync',
'--archive',
'--partial',
'--progress',
self.ssh_host + ':' + path,
dest,
], check=True)
else:
uri = self.get_uri(filename)
logger.info('Downloading %r...', uri)
with opener.open(uri) as response:
with open(dest + '.new', 'wb') as writer:
shutil.copyfileobj(response, writer)
os.rename(dest + '.new', dest)
return dest
def pin_version(
self,
opener: urllib.request.OpenerDirector,
) -> str:
pinned = self.pinned_version
if pinned is None:
if self.ssh_host and self.ssh_path:
path = self.get_ssh_path(filename='VERSION.txt')
logger.info('Determining version number from %r...', path)
pinned = subprocess.run([
'ssh', self.ssh_host,
'cat {}'.format(shlex.quote(path)),
], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
else:
uri = self.get_uri(filename='VERSION.txt')
logger.info('Determining version number from %r...', uri)
with opener.open(uri) as response:
pinned = response.read().decode('utf-8').strip()
self.pinned_version = pinned
return pinned
class Runtime: class Runtime:
def __init__( def __init__(
self, self,
...@@ -381,11 +482,16 @@ exec "${{pressure_vessel}}/bin/pressure-vessel-unruntime" "$@" ...@@ -381,11 +482,16 @@ exec "${{pressure_vessel}}/bin/pressure-vessel-unruntime" "$@"
class ComponentVersion: class ComponentVersion:
def __init__(self, name: str = '') -> None: def __init__(
self,
name: str = '',
sort_weight: int = 0,
) -> None:
self.name = name self.name = name
self.version = '' self.version = ''
self.runtime = '' self.runtime = ''
self.runtime_version = '' self.runtime_version = ''
self.sort_weight = sort_weight
self.comment = '' self.comment = ''
def __str__(self) -> str: def __str__(self) -> str:
...@@ -399,6 +505,9 @@ class ComponentVersion: ...@@ -399,6 +505,9 @@ class ComponentVersion:
return ret return ret
def to_sort_key(self) -> Tuple[int, str]:
return (self.sort_weight, self.to_tsv())
def to_tsv(self) -> str: def to_tsv(self) -> str:
if self.comment: if self.comment:
comment = '# ' + self.comment comment = '# ' + self.comment
...@@ -420,12 +529,20 @@ class Main: ...@@ -420,12 +529,20 @@ class Main:
credential_envs: Sequence[str] = (), credential_envs: Sequence[str] = (),
credential_hosts: Sequence[str] = (), credential_hosts: Sequence[str] = (),
depot: str = 'depot', depot: str = 'depot',
depot_version: str = '',
images_uri: str = DEFAULT_IMAGES_URI, images_uri: str = DEFAULT_IMAGES_URI,
include_archives: bool = False, include_archives: bool = False,
include_sdk: bool = False, include_sdk: bool = False,
layered: bool = False, layered: bool = False,
minimize: bool = False, minimize: bool = False,
pressure_vessel: str = 'scout', pressure_vessel_archive: str = '',
pressure_vessel_from_runtime: str = '',
pressure_vessel_from_runtime_json: str = '',
pressure_vessel_guess: str = '',
pressure_vessel_ssh_host: str = '',
pressure_vessel_ssh_path: str = '',
pressure_vessel_uri: str = DEFAULT_PRESSURE_VESSEL_URI,
pressure_vessel_version: str = '',
runtime: str = 'scout', runtime: str = 'scout',
source_dir: str = HERE, source_dir: str = HERE,
ssh_host: str = '', ssh_host: str = '',
...@@ -485,11 +602,14 @@ class Main: ...@@ -485,11 +602,14 @@ class Main:
self.default_suite = suite self.default_suite = suite
self.default_version = version self.default_version = version
self.depot = os.path.abspath(depot) self.depot = os.path.abspath(depot)
self.depot_version = depot_version
self.images_uri = images_uri self.images_uri = images_uri
self.include_archives = include_archives self.include_archives = include_archives
self.layered = layered self.layered = layered
self.minimize = minimize self.minimize = minimize
self.pressure_vessel = pressure_vessel self.pressure_vessel_ssh_host = pressure_vessel_ssh_host or ssh_host
self.pressure_vessel_ssh_path = pressure_vessel_ssh_path
self.pressure_vessel_uri = pressure_vessel_uri
self.source_dir = source_dir self.source_dir = source_dir
self.ssh_host = ssh_host self.ssh_host = ssh_host
self.ssh_path = ssh_path self.ssh_path = ssh_path
...@@ -500,6 +620,30 @@ class Main: ...@@ -500,6 +620,30 @@ class Main:
self.unpack_sources_into = unpack_sources_into self.unpack_sources_into = unpack_sources_into
self.versioned_directories = versioned_directories self.versioned_directories = versioned_directories
n_sources = 0
for source in (
pressure_vessel_archive,
pressure_vessel_from_runtime,
pressure_vessel_from_runtime_json,
pressure_vessel_guess,
pressure_vessel_version,
):
if source:
n_sources += 1
if n_sources == 0:
pressure_vessel_version = 'latest'
elif n_sources > 1:
raise RuntimeError(
'Cannot combine more than one of '
'--pressure-vessel, '
'--pressure-vessel-archive, '
'--pressure-vessel-from-runtime, '
'--pressure-vessel-from-runtime-json and '
'--pressure-vessel-version'
)
os.makedirs(self.cache, exist_ok=True) os.makedirs(self.cache, exist_ok=True)
if not (self.include_archives or self.unpack_runtime): if not (self.include_archives or self.unpack_runtime):
...@@ -524,6 +668,48 @@ class Main: ...@@ -524,6 +668,48 @@ class Main:
self.versions = [] # type: List[ComponentVersion] self.versions = [] # type: List[ComponentVersion]
if pressure_vessel_guess:
if self.runtime.name == pressure_vessel_guess:
pressure_vessel_from_runtime = pressure_vessel_guess
elif pressure_vessel_guess.startswith('{'):
pressure_vessel_from_runtime_json = pressure_vessel_guess
elif os.path.isdir(pressure_vessel_guess):
pressure_vessel_archive = os.path.join(
pressure_vessel_guess, 'pressure-vessel-bin.tar.gz',
)
elif (
os.path.isfile(pressure_vessel_guess)
and pressure_vessel_guess.endswith('.tar.gz')
):
pressure_vessel_archive = pressure_vessel_guess
else:
pressure_vessel_from_runtime = pressure_vessel_guess
self.pressure_vessel_runtime = None # type: Optional[Runtime]
self.pressure_vessel_version = ''
if pressure_vessel_version:
self.pressure_vessel_version = pressure_vessel_version
elif pressure_vessel_archive:
self.pressure_vessel_runtime = self.new_runtime(
'scout',
{'path': pressure_vessel_archive},
default_suite='local',
)
elif self.runtime.name == pressure_vessel_from_runtime:
self.pressure_vessel_runtime = self.runtime
elif pressure_vessel_from_runtime:
self.pressure_vessel_runtime = self.new_runtime(
pressure_vessel_from_runtime, {},
default_suite=pressure_vessel_from_runtime,
)
elif pressure_vessel_from_runtime_json:
self.pressure_vessel_runtime = self.new_runtime(
'scout',
json.loads(pressure_vessel_from_runtime_json),
default_suite='scout',
)
def new_runtime( def new_runtime(
self, self,
name: str, name: str,
...@@ -599,15 +785,6 @@ class Main: ...@@ -599,15 +785,6 @@ class Main:
self.merge_dir_into_depot( self.merge_dir_into_depot(
os.path.join(self.source_dir, 'runtimes', 'scout-on-soldier') os.path.join(self.source_dir, 'runtimes', 'scout-on-soldier')
) )
# Copy the launcher script for compatibility
shutil.copy(
os.path.join(self.depot, 'run-in-scout'),
os.path.join(self.depot, 'run')
)
shutil.copy(
os.path.join(self.depot, 'run-in-scout'),
os.path.join(self.depot, '_v2-entry-point'),
)
if self.runtime.version: if self.runtime.version:
self.unpack_ld_library_path = self.depot self.unpack_ld_library_path = self.depot
...@@ -618,6 +795,8 @@ class Main: ...@@ -618,6 +795,8 @@ class Main:
local_version.version = version local_version.version = version
local_version.runtime = 'scout' local_version.runtime = 'scout'
local_version.runtime_version = version local_version.runtime_version = version
local_version.comment = 'steam-runtime/'
self.versions.append(local_version)
else: else:
unspecified_version = ComponentVersion('LD_LIBRARY_PATH') unspecified_version = ComponentVersion('LD_LIBRARY_PATH')
unspecified_version.version = '-' unspecified_version.version = '-'
...@@ -661,83 +840,69 @@ class Main: ...@@ -661,83 +840,69 @@ class Main:
if os.path.exists(root): if os.path.exists(root):
self.merge_dir_into_depot(root) self.merge_dir_into_depot(root)
if self.runtime.name == self.pressure_vessel: pressure_vessel_runtime = self.pressure_vessel_runtime
if self.pressure_vessel_version:
logger.info( logger.info(
'Downloading pressure-vessel from %s', self.runtime.name) 'Downloading standalone pressure-vessel release'
pressure_vessel_runtime = self.runtime )
pv_version.comment = self.download_pressure_vessel( pv_version.version = self.download_pressure_vessel_standalone(
pressure_vessel_runtime self.pressure_vessel_version
) )
else: else:
if self.pressure_vessel.startswith('{'): assert pressure_vessel_runtime is not None
logger.info(
'Downloading pressure-vessel using JSON from command-line') if pressure_vessel_runtime.path:
pressure_vessel_runtime = self.new_runtime( self.use_local_pressure_vessel(pressure_vessel_runtime.path)
'scout',
json.loads(self.pressure_vessel),
default_suite='scout',
)
pv_version.comment = self.download_pressure_vessel(
pressure_vessel_runtime
)
elif (
os.path.isdir(self.pressure_vessel)
or (
os.path.isfile(self.pressure_vessel)
and self.pressure_vessel.endswith('.tar.gz')
)
):
logger.info(
'Unpacking pressure-vessel from local file or '
'directory %s',
self.pressure_vessel)
self.use_local_pressure_vessel(self.pressure_vessel)
pressure_vessel_runtime = self.new_runtime(
'scout', {'path': self.pressure_vessel},
default_suite='local',
)
pv_version.comment = 'from local file' pv_version.comment = 'from local file'
elif os.path.isfile(self.pressure_vessel):
logger.info(
'Downloading pressure-vessel using JSON from %r',
self.pressure_vessel)
with open(self.pressure_vessel, 'rb') as reader:
details = json.load(reader)
pressure_vessel_runtime = self.new_runtime(
'scout',
details,
default_suite='scout',
)
pv_version.comment = self.download_pressure_vessel(
pressure_vessel_runtime
)
else: else:
logger.info( pv_version.comment = (
'Assuming %r is a suite containing pressure-vessel', self.download_pressure_vessel_from_runtime(
self.pressure_vessel) pressure_vessel_runtime
pressure_vessel_runtime = self.new_runtime( )
self.pressure_vessel, {},
default_suite=self.pressure_vessel,
)
pv_version.comment = self.download_pressure_vessel(
pressure_vessel_runtime
) )
for path in ('metadata/VERSION.txt', 'sources/VERSION.txt'): for path in ('metadata/VERSION.txt', 'sources/VERSION.txt'):
full = os.path.join(self.depot, 'pressure-vessel', path) full = os.path.join(self.depot, 'pressure-vessel', path)
if os.path.exists(full): if os.path.exists(full):
with open(full) as text_reader: with open(full) as text_reader:
pv_version.version = text_reader.read().rstrip('\n') v = text_reader.read().rstrip('\n')
if pv_version.version:
if pv_version.version != v:
raise RuntimeError(
'Inconsistent version! '
'{} says {}, but expected {}'.format(
path, v, pv_version.version,
)
)
else:
pv_version.version = v
break break
pv_version.runtime = pressure_vessel_runtime.suite or '' if pressure_vessel_runtime is not None:
pv_version.runtime_version = ( pv_version.runtime = pressure_vessel_runtime.suite or ''
pressure_vessel_runtime.pinned_version or '' pv_version.runtime_version = (
) pressure_vessel_runtime.pinned_version or ''
)
self.versions.append(pv_version) self.versions.append(pv_version)
if self.unpack_ld_library_path: if self.unpack_ld_library_path and pressure_vessel_runtime is None:
if self.runtime.name == 'scout':
scout = self.runtime
else:
scout = self.new_runtime(
'scout',
dict(version='latest'),
default_suite='scout',
)
logger.info(
'Downloading LD_LIBRARY_PATH Steam Runtime from scout into %r',
self.unpack_ld_library_path)
self.download_scout_tarball(scout)
elif self.unpack_ld_library_path:
assert pressure_vessel_runtime is not None
logger.info( logger.info(
'Downloading LD_LIBRARY_PATH Steam Runtime from same place ' 'Downloading LD_LIBRARY_PATH Steam Runtime from same place '
'as pressure-vessel into %r', 'as pressure-vessel into %r',
...@@ -984,7 +1149,13 @@ class Main: ...@@ -984,7 +1149,13 @@ class Main:
except (OSError, subprocess.SubprocessError): except (OSError, subprocess.SubprocessError):
version = 'unknown' version = 'unknown'
component_version = ComponentVersion('SteamLinuxRuntime') if self.depot_version:
component_version = ComponentVersion('depot', sort_weight=-1)
component_version.version = self.depot_version
component_version.comment = 'Overall version number'
self.versions.append(component_version)
component_version = ComponentVersion('scripts')
component_version.version = version component_version.version = version
component_version.comment = 'Entry point scripts, etc.' component_version.comment = 'Entry point scripts, etc.'
self.versions.append(component_version) self.versions.append(component_version)
...@@ -994,7 +1165,7 @@ class Main: ...@@ -994,7 +1165,7 @@ class Main:
'#Name\tVersion\t\tRuntime\tRuntime_Version\tComment\n' '#Name\tVersion\t\tRuntime\tRuntime_Version\tComment\n'
) )
for entry in sorted(self.versions, key=lambda v: v.to_tsv()): for entry in sorted(self.versions, key=lambda v: v.to_sort_key()):
logger.info('Component version: %s', entry) logger.info('Component version: %s', entry)
writer.write(entry.to_tsv()) writer.write(entry.to_tsv())
...@@ -1011,7 +1182,24 @@ class Main: ...@@ -1011,7 +1182,24 @@ class Main:
logger.info('%r', argv) logger.info('%r', argv)
subprocess.run(argv, check=True) subprocess.run(argv, check=True)
def download_pressure_vessel(self, runtime: Runtime) -> str: def download_pressure_vessel_standalone(
self,
version: str,
) -> str:
pv = PressureVesselRelease(
cache=self.cache,
ssh_host=self.pressure_vessel_ssh_host,
ssh_path=self.pressure_vessel_ssh_path,
uri=self.pressure_vessel_uri,
version=version,
)
pinned = pv.pin_version(self.opener)
self.use_local_pressure_vessel(
pv.fetch('pressure-vessel-bin.tar.gz', self.opener, pinned)
)
return pinned
def download_pressure_vessel_from_runtime(self, runtime: Runtime) -> str:
filename = 'pressure-vessel-bin.tar.gz' filename = 'pressure-vessel-bin.tar.gz'
runtime.pin_version(self.opener) runtime.pin_version(self.opener)
...@@ -1475,13 +1663,73 @@ def main() -> None: ...@@ -1475,13 +1663,73 @@ def main() -> None:
) )
) )
parser.add_argument( parser.add_argument(
'--pressure-vessel', default='scout', metavar='NAME|PATH|DETAILS', '--depot-version', default='',
help=( help=(
'Get pressure-vessel from the named runtime (default "scout"), ' 'Set an overall version number for the depot contents'
'or from a runtime version given as a JSON object, '
'or from a given directory (use ./ to disambiguate if necessary).'
) )
) )
parser.add_argument(
'--pressure-vessel-uri',
default=DEFAULT_PRESSURE_VESSEL_URI,
metavar='URI',
help=(
'Download pressure-vessel from a versioned subdirectory of URI'
),
)
parser.add_argument(
'--pressure-vessel-ssh-host', default='', metavar='HOST',
help=(
'Use ssh and rsync to download pressure-vessel from HOST '
'[default: same as --ssh-host]'
),
)
parser.add_argument(
'--pressure-vessel-ssh-path', default='', metavar='PATH',
help=(
'Use ssh and rsync to download pressure-vessel from a versioned '
'subdirectory of PATH on HOST'
),
)
parser.add_argument(
'--pressure-vessel-version', default='', metavar='0.x.y|latest',
help=(
'Use this version of pressure-vessel from --pressure-vessel-uri '
'or --pressure-vessel-ssh-path'
)
)
parser.add_argument(
'--pressure-vessel-archive', default='', metavar='PATH',
help=(
'Unpack pressure-vessel from the named archive'
),
)
parser.add_argument(
'--pressure-vessel-from-runtime', default='', metavar='NAME',
help=(
'Get pressure-vessel from the named runtime (default "scout")'
),
)
parser.add_argument(
'--pressure-vessel-from-runtime-json', default='', metavar='NAME',
help=(
'Get pressure-vessel from a separate runtime version given as a '
'JSON object'
),
)
parser.add_argument(
'--pressure-vessel', default='', metavar='NAME|PATH|DETAILS',
dest='pressure_vessel_guess',
help=(
'--pressure-vessel-archive=ARCHIVE, '
'--pressure-vessel-archive=DIRECTORY/pressure-vessel-bin.tar.gz, '
'--pressure-vessel-from-runtime=NAME or '
'--pressure-vessel-from-runtime-json=DETAILS, '
'based on form of argument given '
'(disambiguate with ./ if necessary)'
),
)
parser.add_argument( parser.add_argument(
'--include-archives', action='store_true', default=False, '--include-archives', action='store_true', default=False,
help=( help=(
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment