From 85e4016ba66010a9e0d8ba2bd2adbbcd125a91b0 Mon Sep 17 00:00:00 2001 From: Simon McVittie <smcv@collabora.com> Date: Tue, 15 Jun 2021 16:16:57 +0100 Subject: [PATCH] Update populate-depot.py from SteamLinuxRuntime v0.20210608.0 Signed-off-by: Simon McVittie <smcv@collabora.com> --- pressure-vessel/populate-depot.py | 788 +++++++++++++++++++++++------- 1 file changed, 622 insertions(+), 166 deletions(-) diff --git a/pressure-vessel/populate-depot.py b/pressure-vessel/populate-depot.py index 507d473df..5ae2831f3 100755 --- a/pressure-vessel/populate-depot.py +++ b/pressure-vessel/populate-depot.py @@ -29,13 +29,19 @@ from just-built files or by downloading a previous build. """ import argparse +import errno +import gzip +import hashlib import json import logging import os +import re import shlex import shutil +import stat import subprocess import sys +import tarfile import tempfile import urllib.parse import urllib.request @@ -46,6 +52,7 @@ from typing import ( List, Optional, Sequence, + Set, ) from debian.deb822 import ( @@ -88,14 +95,16 @@ class Runtime: suite: str, architecture: str = 'amd64,i386', + cache: str = '.cache', images_uri: str = DEFAULT_IMAGES_URI, include_sdk: bool = False, path: Optional[str] = None, ssh_host: str = '', ssh_path: str = '', - version: str = 'latest', + version: str = '', ) -> None: self.architecture = architecture + self.cache = cache self.images_uri = images_uri self.include_sdk = include_sdk self.name = name @@ -105,6 +114,9 @@ class Runtime: self.ssh_path = ssh_path self.version = version self.pinned_version = None # type: Optional[str] + self.sha256 = {} # type: Dict[str, str] + + os.makedirs(self.cache, exist_ok=True) self.prefix = 'com.valvesoftware.SteamRuntime' self.platform = self.prefix + '.Platform' @@ -166,10 +178,11 @@ class Runtime: cls, name: str, details: Dict[str, Any], + cache: str = '.cache', default_architecture: str = 'amd64,i386', default_include_sdk: bool = False, default_suite: str = '', - default_version: str = 'latest', + default_version: str = '', images_uri: str = DEFAULT_IMAGES_URI, ssh_host: str = '', ssh_path: str = '', @@ -179,6 +192,7 @@ class Runtime: architecture=details.get( 'architecture', default_architecture, ), + cache=cache, images_uri=images_uri, include_sdk=details.get('include_sdk', default_include_sdk), path=details.get('path', None), @@ -195,7 +209,7 @@ class Runtime: ) -> str: suite = self.suite uri = self.images_uri.replace('SUITE', suite) - v = version or self.pinned_version or self.version + v = version or self.pinned_version or self.version or 'latest' return f'{uri}/{v}/{filename}' def get_ssh_path( @@ -206,7 +220,7 @@ class Runtime: ssh_host = self.ssh_host suite = self.suite ssh_path = self.ssh_path.replace('SUITE', suite) - v = version or self.pinned_version or self.version + v = version or self.pinned_version or self.version or 'latest' if not ssh_host or not ssh_path: raise RuntimeError('ssh host/path not configured') @@ -216,10 +230,32 @@ class Runtime: def fetch( self, filename: str, - destdir: str, opener: urllib.request.OpenerDirector, version: Optional[str] = None, - ) -> None: + ) -> str: + dest = os.path.join(self.cache, filename) + + if filename in self.sha256: + try: + with open(dest, 'rb') as reader: + hasher = hashlib.sha256() + + while True: + blob = reader.read(4096) + + if not blob: + break + + hasher.update(blob) + + digest = hasher.hexdigest() + except OSError: + pass + else: + if digest == self.sha256[filename]: + logger.info('Using cached %r', dest) + return dest + if self.ssh_host and self.ssh_path: path = self.get_ssh_path(filename) logger.info('Downloading %r...', path) @@ -229,21 +265,26 @@ class Runtime: '--partial', '--progress', self.ssh_host + ':' + path, - os.path.join(destdir, filename), + dest, ], check=True) else: uri = self.get_uri(filename) logger.info('Downloading %r...', uri) with opener.open(uri) as response: - with open(os.path.join(destdir, filename), 'wb') as writer: + with open(dest + '.new', 'wb') as writer: shutil.copyfileobj(response, writer) + os.rename(dest + '.new', dest) + + return dest + def pin_version( self, opener: urllib.request.OpenerDirector, ) -> str: pinned = self.pinned_version + sha256 = {} # type: Dict[str, str] if pinned is None: if self.ssh_host and self.ssh_path: @@ -253,18 +294,67 @@ class Runtime: 'ssh', self.ssh_host, 'cat {}'.format(shlex.quote(path)), ], stdout=subprocess.PIPE).stdout.decode('utf-8').strip() + + path = self.get_ssh_path(filename='SHA256SUMS') + + sha256sums = subprocess.run([ + 'ssh', self.ssh_host, + 'cat {}'.format(shlex.quote(path)), + ], stdout=subprocess.PIPE).stdout + assert sha256sums is not None + else: uri = self.get_uri(filename='VERSION.txt') logger.info('Determining version number from %r...', uri) with opener.open(uri) as response: pinned = response.read().decode('utf-8').strip() + uri = self.get_uri(filename='SHA256SUMS') + + with opener.open(uri) as response: + sha256sums = response.read() + + for line in sha256sums.splitlines(): + sha256_bytes, name_bytes = line.split(maxsplit=1) + name = name_bytes.decode('utf-8') + + if name.startswith('*'): + name = name[1:] + + sha256[name] = sha256_bytes.decode('ascii') + + self.sha256 = sha256 self.pinned_version = pinned return pinned -RUN_IN_WHATEVER_SOURCE = '''\ +RUN_IN_DIR_SOURCE = '''\ +#!/bin/sh +# {source_for_generated_file} + +set -eu + +me="$(readlink -f "$0")" +here="${{me%/*}}" +me="${{me##*/}}" + +dir={escaped_dir} +pressure_vessel="${{PRESSURE_VESSEL_PREFIX:-"${{here}}/pressure-vessel"}}" + +export PRESSURE_VESSEL_GC_LEGACY_RUNTIMES=1 +export PRESSURE_VESSEL_RUNTIME="${{dir}}" +unset PRESSURE_VESSEL_RUNTIME_ARCHIVE +export PRESSURE_VESSEL_RUNTIME_BASE="${{here}}" + +if [ -z "${{PRESSURE_VESSEL_VARIABLE_DIR-}}" ]; then + export PRESSURE_VESSEL_VARIABLE_DIR="${{here}}/var" +fi + +exec "${{pressure_vessel}}/bin/pressure-vessel-unruntime" "$@" +''' + +RUN_IN_ARCHIVE_SOURCE = '''\ #!/bin/sh # {source_for_generated_file} @@ -326,31 +416,32 @@ class Main: def __init__( self, architecture: str = 'amd64,i386', - cache: str = '', + cache: str = '.cache', credential_envs: Sequence[str] = (), credential_hosts: Sequence[str] = (), depot: str = 'depot', images_uri: str = DEFAULT_IMAGES_URI, + include_archives: bool = False, include_sdk: bool = False, + layered: bool = False, + minimize: bool = False, pressure_vessel: str = 'scout', - runtimes: Sequence[str] = (), + runtime: str = 'scout', source_dir: str = HERE, ssh_host: str = '', ssh_path: str = '', suite: str = '', toolmanifest: bool = False, unpack_ld_library_path: str = '', - unpack_runtimes: bool = False, + unpack_runtime: bool = True, unpack_sources: Sequence[str] = (), unpack_sources_into: str = '.', - version: str = 'latest', + version: str = '', + versioned_directories: bool = False, **kwargs: Dict[str, Any], ) -> None: openers: List[urllib.request.BaseHandler] = [] - if not runtimes: - runtimes = ('scout',) - if not credential_hosts: credential_hosts = [] host = urllib.parse.urlparse(images_uri).hostname @@ -395,34 +486,41 @@ class Main: self.default_version = version self.depot = os.path.abspath(depot) self.images_uri = images_uri + self.include_archives = include_archives + self.layered = layered + self.minimize = minimize self.pressure_vessel = pressure_vessel - self.runtimes = [] # type: List[Runtime] self.source_dir = source_dir self.ssh_host = ssh_host self.ssh_path = ssh_path self.toolmanifest = toolmanifest self.unpack_ld_library_path = unpack_ld_library_path - self.unpack_runtimes = unpack_runtimes + self.unpack_runtime = unpack_runtime self.unpack_sources = unpack_sources self.unpack_sources_into = unpack_sources_into + self.versioned_directories = versioned_directories - if cache: - os.makedirs(self.cache, exist_ok=True) + os.makedirs(self.cache, exist_ok=True) - for runtime in runtimes: - if '=' in runtime: - name, rhs = runtime.split('=', 1) + if not (self.include_archives or self.unpack_runtime): + raise RuntimeError( + 'Cannot use both --no-include-archives and ' + '--no-unpack-runtime' + ) - if rhs.startswith('{'): - details = json.loads(rhs) - else: - with open(rhs, 'rb') as reader: - details = json.load(reader) + if '=' in runtime: + name, rhs = runtime.split('=', 1) + + if rhs.startswith('{'): + details = json.loads(rhs) else: - name = runtime - details = {} + with open(rhs, 'rb') as reader: + details = json.load(reader) + else: + name = runtime + details = {} - self.runtimes.append(self.new_runtime(name, details)) + self.runtime = self.new_runtime(name, details) self.versions = [] # type: List[ComponentVersion] @@ -435,6 +533,7 @@ class Main: return Runtime.from_details( name, details, + cache=self.cache, default_architecture=self.default_architecture, default_include_sdk=self.default_include_sdk, default_suite=default_suite or self.default_suite, @@ -468,25 +567,107 @@ class Main: shutil.copy(source, merged) def run(self) -> None: + if self.layered: + self.do_layered_runtime() + else: + self.do_container_runtime() + + def do_layered_runtime(self) -> None: + if self.runtime.name != 'scout': + raise InvocationError('Can only layer scout onto soldier') + + if self.unpack_ld_library_path: + raise InvocationError( + 'Cannot use --unpack-ld-library-path with --layered' + ) + + if self.include_archives: + raise InvocationError( + 'Cannot use --include-archives with --layered' + ) + + if self.default_include_sdk: + raise InvocationError( + 'Cannot use --include-sdk with --layered' + ) + + if self.unpack_sources: + raise InvocationError( + 'Cannot use --unpack-source with --layered' + ) + + self.merge_dir_into_depot( + os.path.join(self.source_dir, 'runtimes', 'scout-on-soldier') + ) + # Copy the launcher script for compatibility + shutil.copy( + os.path.join(self.depot, 'run-in-scout'), + os.path.join(self.depot, 'run') + ) + shutil.copy( + os.path.join(self.depot, 'run-in-scout'), + os.path.join(self.depot, '_v2-entry-point'), + ) + + if self.runtime.version: + self.unpack_ld_library_path = self.depot + self.download_scout_tarball(self.runtime) + local_version = ComponentVersion('LD_LIBRARY_PATH') + version = self.runtime.pinned_version + assert version is not None + local_version.version = version + local_version.runtime = 'scout' + local_version.runtime_version = version + else: + unspecified_version = ComponentVersion('LD_LIBRARY_PATH') + unspecified_version.version = '-' + unspecified_version.runtime = 'scout' + unspecified_version.runtime_version = '-' + unspecified_version.comment = ( + 'see ~/.steam/root/ubuntu12_32/steam-runtime/version.txt' + ) + self.versions.append(unspecified_version) + + self.write_component_versions() + + def ensure_ref(self, path: str) -> None: + ''' + Create $path/files/.ref as an empty regular file. + + This is useful because pressure-vessel would create this file + during processing. If it gets committed to the depot, then Steampipe + will remove it when superseded. + ''' + ref = os.path.join(path, 'files', '.ref') + + try: + statinfo = os.stat(ref, follow_symlinks=False) + except FileNotFoundError: + with open(ref, 'x'): + pass + else: + if statinfo.st_size > 0 or not stat.S_ISREG(statinfo.st_mode): + raise RuntimeError( + 'Expected {} to be an empty regular file'.format(path) + ) + + def do_container_runtime(self) -> None: pv_version = ComponentVersion('pressure-vessel') self.merge_dir_into_depot(os.path.join(self.source_dir, 'common')) - for runtime in self.runtimes: - root = os.path.join(self.source_dir, runtime.name) + root = os.path.join(self.source_dir, 'runtimes', self.runtime.name) - if os.path.exists(root): - self.merge_dir_into_depot(root) + if os.path.exists(root): + self.merge_dir_into_depot(root) - for runtime in self.runtimes: - if runtime.name == self.pressure_vessel: - logger.info( - 'Downloading pressure-vessel from %s', runtime.name) - pressure_vessel_runtime = runtime - pv_version.comment = self.download_pressure_vessel( - pressure_vessel_runtime - ) - break + if self.runtime.name == self.pressure_vessel: + logger.info( + 'Downloading pressure-vessel from %s', self.runtime.name) + pressure_vessel_runtime = self.runtime + pv_version.comment = self.download_pressure_vessel( + pressure_vessel_runtime + ) else: if self.pressure_vessel.startswith('{'): logger.info( @@ -569,13 +750,12 @@ class Main: ', '.join(self.unpack_sources), self.unpack_sources_into) os.makedirs(self.unpack_sources_into, exist_ok=True) - for runtime in self.runtimes: - os.makedirs( - os.path.join(self.unpack_sources_into, runtime.name), - exist_ok=True, - ) + os.makedirs( + os.path.join(self.unpack_sources_into, self.runtime.name), + exist_ok=True, + ) - for runtime in self.runtimes: + for runtime in (self.runtime,): # too much to reindent right now if runtime.path: logger.info( 'Using runtime from local directory %r', @@ -589,21 +769,28 @@ class Main: component_version = ComponentVersion(runtime.name) - version = runtime.pinned_version - comment = ', '.join(sorted(runtime.runtime_files)) + if runtime.path: + with open( + os.path.join(runtime.path, runtime.build_id_file), 'r', + ) as text_reader: + version = text_reader.read().strip() + else: + version = runtime.pinned_version or '' + assert version - if version is None: - version = runtime.version - comment += ' (from local build)' + if self.include_archives: + runtime_files = set(runtime.runtime_files) + else: + runtime_files = set() - component_version.version = version - component_version.runtime = runtime.suite - component_version.runtime_version = version - component_version.comment = comment - self.versions.append(component_version) + if self.unpack_runtime: + if self.versioned_directories: + subdir = '{}_platform_{}'.format(runtime.name, version) + else: + subdir = runtime.name - if self.unpack_runtimes: - dest = os.path.join(self.depot, runtime.name) + dest = os.path.join(self.depot, subdir) + runtime_files.add(subdir + '/') with suppress(FileNotFoundError): shutil.rmtree(dest) @@ -613,13 +800,28 @@ class Main: 'tar', '-C', dest, '-xf', - os.path.join(self.depot, runtime.tarball), + os.path.join(self.cache, runtime.tarball), ] logger.info('%r', argv) subprocess.run(argv, check=True) + self.write_lookaside( + os.path.join(self.cache, runtime.tarball), + dest, + ) + + if self.minimize: + self.minimize_runtime(dest) + + self.ensure_ref(dest) if runtime.include_sdk: - dest = os.path.join(self.depot, runtime.name + '_sdk') + if self.versioned_directories: + sdk_subdir = '{}_sdk_{}'.format(runtime.name, version) + else: + sdk_subdir = '{}_sdk'.format(runtime.name) + + dest = os.path.join(self.depot, sdk_subdir) + runtime_files.add(sdk_subdir + '/') with suppress(FileNotFoundError): shutil.rmtree(os.path.join(dest, 'files')) @@ -634,24 +836,39 @@ class Main: argv = [ 'tar', '-C', dest, - '-xf', os.path.join(self.depot, runtime.sdk_tarball), + '-xf', os.path.join(self.cache, runtime.sdk_tarball), ] logger.info('%r', argv) subprocess.run(argv, check=True) + self.write_lookaside( + os.path.join(self.cache, runtime.sdk_tarball), + dest, + ) + + if self.minimize: + self.minimize_runtime(dest) + + self.ensure_ref(dest) argv = [ 'tar', '-C', os.path.join(dest, 'files', 'lib', 'debug'), '--transform', r's,^\(\./\)\?files\(/\|$\),,', '-xf', - os.path.join(self.depot, runtime.debug_tarball), + os.path.join(self.cache, runtime.debug_tarball), ] logger.info('%r', argv) subprocess.run(argv, check=True) - sysroot = os.path.join( - self.depot, runtime.name + '_sysroot', - ) + if self.versioned_directories: + sysroot_subdir = '{}_sysroot_{}'.format( + runtime.name, version, + ) + else: + sysroot_subdir = '{}_sysroot'.format(runtime.name) + + sysroot = os.path.join(self.depot, sysroot_subdir) + runtime_files.add(sysroot_subdir + '/') with suppress(FileNotFoundError): shutil.rmtree(sysroot) @@ -662,7 +879,7 @@ class Main: '-C', os.path.join(sysroot, 'files'), '--exclude', 'dev/*', '-xf', - os.path.join(self.depot, runtime.sysroot_tarball), + os.path.join(self.cache, runtime.sysroot_tarball), ] logger.info('%r', argv) subprocess.run(argv, check=True) @@ -678,25 +895,46 @@ class Main: with open( os.path.join(self.depot, 'run-in-' + runtime.name), 'w' ) as writer: - writer.write( - RUN_IN_WHATEVER_SOURCE.format( - escaped_arch=shlex.quote(runtime.architecture), - escaped_name=shlex.quote(runtime.name), - escaped_runtime=shlex.quote(runtime.platform), - escaped_suite=shlex.quote(runtime.suite), - source_for_generated_file=( - 'Generated file, do not edit' - ), + if self.unpack_runtime: + writer.write( + RUN_IN_DIR_SOURCE.format( + escaped_dir=shlex.quote(subdir), + source_for_generated_file=( + 'Generated file, do not edit' + ), + ) + ) + else: + writer.write( + RUN_IN_ARCHIVE_SOURCE.format( + escaped_arch=shlex.quote(runtime.architecture), + escaped_name=shlex.quote(runtime.name), + escaped_runtime=shlex.quote(runtime.platform), + escaped_suite=shlex.quote(runtime.suite), + source_for_generated_file=( + 'Generated file, do not edit' + ), + ) ) - ) os.chmod(os.path.join(self.depot, 'run-in-' + runtime.name), 0o755) - for runtime in self.runtimes[0:]: + comment = ', '.join(sorted(runtime_files)) + + if runtime.path: + comment += ' (from local build)' + + component_version.version = version + component_version.runtime = runtime.suite + component_version.runtime_version = version + component_version.comment = comment + self.versions.append(component_version) + + for runtime in (self.runtime,): # too much to reindent right now if not self.toolmanifest: continue with open( - os.path.join(self.depot, 'toolmanifest.v2.vdf'), 'w' + os.path.join(self.depot, 'toolmanifest.vdf'), 'w' ) as writer: import vdf # noqa @@ -713,32 +951,19 @@ class Main: use_tool_subprocess_reaper='1', ) ) # type: Dict[str, Any] - if runtime.suite == 'soldier': + if runtime.suite != 'scout': content['manifest']['unlisted'] = '1' vdf.dump(content, writer, pretty=True, escaped=True) - with open( - os.path.join(self.depot, 'run'), 'w' - ) as writer: - writer.write( - RUN_IN_WHATEVER_SOURCE.format( - escaped_arch=shlex.quote(runtime.architecture), - escaped_name=shlex.quote(runtime.name), - escaped_runtime=shlex.quote(runtime.platform), - escaped_suite=shlex.quote(runtime.suite), - source_for_generated_file=( - 'Generated file, do not edit' - ), - ) - ) + shutil.copy2( + os.path.join(self.depot, 'run-in-' + runtime.name), + os.path.join(self.depot, 'run'), + ) os.chmod(os.path.join(self.depot, 'run'), 0o755) - if self.toolmanifest: - shutil.copy2( - os.path.join(self.depot, 'toolmanifest.v2.vdf'), - os.path.join(self.depot, 'toolmanifest.vdf'), - ) + self.write_component_versions() + def write_component_versions(self) -> None: try: with subprocess.Popen( [ @@ -790,21 +1015,18 @@ class Main: filename = 'pressure-vessel-bin.tar.gz' runtime.pin_version(self.opener) - with tempfile.TemporaryDirectory(prefix='populate-depot.') as tmp: - runtime.fetch( - filename, - self.cache or tmp, - self.opener, - ) + downloaded = runtime.fetch( + filename, + self.opener, + ) - os.makedirs(self.depot, exist_ok=True) - subprocess.run( - [ - 'tar', '-C', self.depot, '-xf', - os.path.join(self.cache or tmp, filename), - ], - check=True, - ) + os.makedirs(self.depot, exist_ok=True) + subprocess.run( + [ + 'tar', '-C', self.depot, '-xf', downloaded, + ], + check=True, + ) return filename @@ -813,7 +1035,7 @@ class Main: for basename in runtime.runtime_files: src = os.path.join(runtime.path, basename) - dest = os.path.join(self.depot, basename) + dest = os.path.join(self.cache, basename) logger.info('Hard-linking local runtime %r to %r', src, dest) with suppress(FileNotFoundError): @@ -821,17 +1043,27 @@ class Main: os.link(src, dest) - with open( - os.path.join(self.depot, runtime.build_id_file), 'w', - ) as writer: - writer.write(f'{runtime.version}\n') + if self.include_archives: + dest = os.path.join(self.depot, basename) + logger.info('Hard-linking local runtime %r to %r', src, dest) - if runtime.include_sdk: + with suppress(FileNotFoundError): + os.unlink(dest) + + os.link(src, dest) + + if self.include_archives: with open( - os.path.join(self.depot, runtime.sdk_build_id_file), 'w', + os.path.join(self.depot, runtime.build_id_file), 'w', ) as writer: writer.write(f'{runtime.version}\n') + if runtime.include_sdk: + with open( + os.path.join(self.depot, runtime.sdk_build_id_file), 'w', + ) as writer: + writer.write(f'{runtime.version}\n') + if self.unpack_sources: with open( os.path.join(runtime.path, runtime.sources), 'rb', @@ -876,32 +1108,33 @@ class Main: """ pinned = runtime.pin_version(self.opener) + for basename in runtime.runtime_files: - runtime.fetch(basename, self.depot, self.opener) + downloaded = runtime.fetch(basename, self.opener) - with open( - os.path.join(self.depot, runtime.build_id_file), 'w', - ) as writer: - writer.write(f'{pinned}\n') + if self.include_archives: + os.link(downloaded, os.path.join(self.depot, basename)) - if runtime.include_sdk: + if self.include_archives: with open( - os.path.join(self.depot, runtime.sdk_build_id_file), 'w', + os.path.join(self.depot, runtime.build_id_file), 'w', ) as writer: writer.write(f'{pinned}\n') + if runtime.include_sdk: + with open( + os.path.join(self.depot, runtime.sdk_build_id_file), 'w', + ) as writer: + writer.write(f'{pinned}\n') + if self.unpack_sources: with tempfile.TemporaryDirectory(prefix='populate-depot.') as tmp: want = set(self.unpack_sources) - runtime.fetch( + downloaded = runtime.fetch( runtime.sources, - self.cache or tmp, self.opener, ) - with open( - os.path.join(self.cache or tmp, runtime.sources), - 'rb' - ) as reader: + with open(downloaded, 'rb') as reader: for stanza in Sources.iter_paragraphs( sequence=reader, use_apt_pkg=True, @@ -917,11 +1150,12 @@ class Main: exist_ok=True, ) + file_path = {} # type: Dict[str, str] + for f in stanza['files']: name = f['name'] - runtime.fetch( + file_path[name] = runtime.fetch( os.path.join('sources', name), - self.cache or tmp, self.opener, ) @@ -943,11 +1177,7 @@ class Main: [ 'dpkg-source', '-x', - os.path.join( - self.cache or tmp, - 'sources', - name, - ), + file_path[name], dest, ], check=True, @@ -970,19 +1200,192 @@ class Main: logger.info('Downloading steam-runtime build %s', pinned) os.makedirs(self.unpack_ld_library_path, exist_ok=True) - with tempfile.TemporaryDirectory(prefix='populate-depot.') as tmp: - runtime.fetch( - filename, - self.cache or tmp, - self.opener, - ) - subprocess.run( - [ - 'tar', '-C', self.unpack_ld_library_path, '-xf', - os.path.join(self.cache or tmp, filename), - ], - check=True, - ) + downloaded = runtime.fetch( + filename, + self.opener, + ) + subprocess.run( + [ + 'tar', '-C', self.unpack_ld_library_path, '-xf', + downloaded, + ], + check=True, + ) + + def octal_escape_char(self, match: 're.Match') -> str: + ret = [] # type: List[str] + + for byte in match.group(0).encode('utf-8', 'surrogateescape'): + ret.append('\\%03o' % byte) + + return ''.join(ret) + + _NEEDS_OCTAL_ESCAPE = re.compile(r'[^-A-Za-z0-9+,./:@_]') + + def octal_escape(self, s: str) -> str: + return self._NEEDS_OCTAL_ESCAPE.sub(self.octal_escape_char, s) + + def filename_is_windows_friendly(self, s: str) -> bool: + for c in s: + # This is the set of characters that are reserved in Windows + # filenames, excluding '/' which obviously we're fine with + # using as a directory separator. + if c in r'<>:"\|?*': + return False + + if c >= '\uDC80' and c <= '\uDCFF': + # surrogate escape, not Unicode + return False + + return True + + def write_lookaside(self, archive: str, dest: str) -> None: + with tarfile.open( + archive, mode='r' + ) as unarchiver, gzip.open( + os.path.join(dest, 'usr-mtree.txt.gz'), 'wt' + ) as writer: + lc_names = {} # type: Dict[str, str] + differ_only_by_case = set() # type: Set[str] + not_windows_friendly = set() # type: Set[str] + sha256 = {} # type: Dict[str, str] + sizes = {} # type: Dict[str, int] + + writer.write('#mtree\n') + writer.write('. type=dir\n') + + for member in unarchiver: + name = member.name + + if name.startswith('./'): + name = name[len('./'):] + + if not name.startswith('files/'): + continue + + name = name[len('files/'):] + + if not self.filename_is_windows_friendly(name): + not_windows_friendly.add(name) + + if name.lower() in lc_names: + differ_only_by_case.add(lc_names[name.lower()]) + differ_only_by_case.add(name) + else: + lc_names[name.lower()] = name + + fields = ['./' + self.octal_escape(name)] + + if member.isfile() or member.islnk(): + fields.append('type=file') + fields.append('mode=%o' % member.mode) + + # We only store mtime to 1-second precision for now, + # but some mtree implementations require the dot. + # + # If we add sub-second precision, note that some + # versions of mtree use the part after the dot + # as integer nanoseconds, so "1.234" is actually + # 1 second + 234 nanoseconds, or what normal people + # would write as 1.000000234 - we can be compatible + # with both by always showing the time in %d.%09d format. + fields.append('time=%d.0' % member.mtime) + + if member.islnk(): + writer.write( + '# hard link to {}\n'.format( + self.octal_escape(member.linkname), + ), + ) + assert member.linkname in sizes + fields.append('size=%d' % sizes[member.linkname]) + + if sizes[member.linkname] > 0: + assert member.linkname in sha256 + fields.append('sha256=' + sha256[member.linkname]) + else: + fields.append('size=%d' % member.size) + sizes[member.name] = member.size + + if member.size > 0: + hasher = hashlib.sha256() + extract = unarchiver.extractfile(member) + assert extract is not None + with extract: + while True: + blob = extract.read(4096) + + if not blob: + break + + hasher.update(blob) + + fields.append('sha256=' + hasher.hexdigest()) + sha256[member.name] = hasher.hexdigest() + + elif member.issym(): + fields.append('type=link') + fields.append('link=' + self.octal_escape(member.linkname)) + elif member.isdir(): + fields.append('type=dir') + else: + writer.write( + '# unknown file type: {}\n'.format( + self.octal_escape(member.name), + ), + ) + continue + + writer.write(' '.join(fields) + '\n') + + if '.ref' not in lc_names: + writer.write('./.ref type=file size=0 mode=644\n') + + if differ_only_by_case: + writer.write('\n') + writer.write('# Files whose names differ only by case:\n') + + for name in sorted(differ_only_by_case): + writer.write('# {}\n'.format(self.octal_escape(name))) + + if not_windows_friendly: + writer.write('\n') + writer.write('# Files whose names are not Windows-friendly:\n') + + for name in sorted(not_windows_friendly): + writer.write('# {}\n'.format(self.octal_escape(name))) + + def minimize_runtime(self, root: str) -> None: + ''' + Remove files that pressure-vessel can reconstitute from the manifest. + + This is the equivalent of: + + find $root/files -type l -delete + find $root/files -empty -delete + + Note that this needs to be done before ensure_ref(), otherwise + it will delete files/.ref too. + ''' + for (dirpath, dirnames, filenames) in os.walk( + os.path.join(root, 'files'), + topdown=False, + ): + for f in filenames + dirnames: + path = os.path.join(dirpath, f) + + try: + statinfo = os.lstat(path) + except FileNotFoundError: + continue + + if stat.S_ISLNK(statinfo.st_mode) or statinfo.st_size == 0: + os.remove(path) + try: + os.rmdir(dirpath) + except OSError as e: + if e.errno != errno.ENOTEMPTY: + raise def main() -> None: @@ -1008,14 +1411,14 @@ def main() -> None: ) ) parser.add_argument( - '--version', default='latest', + '--version', default='', help=( 'Default version to use if none is specified' ) ) parser.add_argument( - '--cache', default='', + '--cache', default='.cache', help=( 'Cache downloaded files that are not in --depot here' ), @@ -1068,7 +1471,7 @@ def main() -> None: parser.add_argument( '--depot', default='depot', help=( - 'Download runtimes into this existing directory' + 'Download runtime into this existing directory' ) ) parser.add_argument( @@ -1079,10 +1482,42 @@ def main() -> None: 'or from a given directory (use ./ to disambiguate if necessary).' ) ) + parser.add_argument( + '--include-archives', action='store_true', default=False, + help=( + 'Provide the runtime as an archive to be unpacked' + ) + ) + parser.add_argument( + '--no-include-archives', action='store_false', dest='include_archives', + help=( + 'Do not provide the runtime as an archive to be unpacked ' + '[default]' + ) + ) parser.add_argument( '--include-sdk', default=False, action='store_true', help='Include a corresponding SDK', ) + parser.add_argument( + '--layered', default=False, action='store_true', + help='Produce a layered runtime that runs scout on soldier', + ) + parser.add_argument( + '--minimize', action='store_true', default=False, + help=( + 'Omit empty files, empty directories and symlinks from ' + 'runtime content, requiring pressure-vessel to fill them in ' + 'from the mtree manifest' + ) + ) + parser.add_argument( + '--no-minimize', action='store_false', dest='minimize', + help=( + 'Include empty files, empty directories and symlinks in ' + 'runtime content [default]' + ) + ) parser.add_argument( '--source-dir', default=HERE, help=( @@ -1102,10 +1537,18 @@ def main() -> None: ) ) parser.add_argument( - '--unpack-runtimes', action='store_true', default=False, + '--unpack-runtime', '--unpack-runtimes', + action='store_true', default=True, help=( - "Unpack the runtimes into the --depot, for use with " - "pressure-vessel's tests/containers.py." + "Unpack the runtime into the --depot, for use with " + "pressure-vessel's tests/containers.py. [default]" + ) + ) + parser.add_argument( + '--no-unpack-runtime', '--no-unpack-runtimes', + action='store_false', dest='unpack_runtime', + help=( + "Don't unpack the runtime into the --depot" ) ) parser.add_argument( @@ -1124,12 +1567,25 @@ def main() -> None: ) ) parser.add_argument( - 'runtimes', - default=[], + '--versioned-directories', action='store_true', default=True, + help=( + 'Include version number in unpacked runtime directories ' + '[default]' + ) + ) + parser.add_argument( + '--no-versioned-directories', action='store_false', + dest='versioned_directories', + help=( + 'Do not include version number in unpacked runtime directories' + ) + ) + parser.add_argument( + 'runtime', + default='', metavar='NAME[="DETAILS"]', - nargs='*', help=( - 'Runtimes to download, in the form NAME or NAME="DETAILS". ' + 'Runtime to download, in the form NAME or NAME="DETAILS". ' 'DETAILS is a JSON object containing something like ' '{"path": "../prebuilt", "suite: "scout", "version": "latest", ' '"architecture": "amd64,i386", "include_sdk": true}, or the ' -- GitLab