Newer
Older
'make-flatpak-friendly',
'platformize',
'prepare-runtime',
'symlink-alternatives',
'usrmerge',
'write-manifest',
):
dest = os.path.join(scratch, helper)
shutil.copyfile(
os.path.join(
os.path.dirname(__file__),
'flatdeb',
helper,
),
dest,
)
os.chmod(dest, 0o755)
prefix = self.runtime_details['id_prefix']
# Do the Platform first, because we download its source
# packages as part of preparing the Sdk
if sdk and not self.do_sdk:
continue
if not sdk and not self.do_platform:
continue
packages = list(self.get_runtime_packages(
self.runtime_details.get('add_packages', [])
))
packages.extend(self.get_runtime_packages(
self.runtime_details.get('add_packages_multiarch', []),
multiarch=True,
))
if sdk:
runtime = prefix + '.Sdk'
else:
runtime = prefix + '.Platform'
artifact_prefix = '{}-{}-{}'.format(
ostree_prefix = artifact_prefix + '-runtime'
out_tarball = ostree_prefix + '.tar.gz'
sources_prefix = artifact_prefix + '-sources'
argv = [
'debos',
'--artifactdir={}'.format(self.build_area),
'--scratchsize=8G',
'-t', 'architecture:{}'.format(self.primary_dpkg_arch),
'-t', 'foreignarchs:{}'.format(
' '.join(self.dpkg_archs[1:]),
),
'-t', 'flatpak_arch:{}'.format(self.flatpak_arch),
'-t', 'suite:{}'.format(self.apt_suite),
'-t', 'ospack:{}'.format(tarball),
'-t', 'artifact_prefix:{}'.format(artifact_prefix),
'-t', 'ostree_prefix:{}'.format(ostree_prefix),
'-t', 'ostree_tarball:{}'.format(out_tarball + '.new'),
'-t', 'runtime:{}'.format(runtime),
'-t', 'runtime_branch:{}'.format(self.runtime_branch),
'-t', 'strip_source_version_suffix:{}'.format(
self.strip_source_version_suffix),
]
if self.apt_debug:
argv.append('-t')
argv.append('apt_debug:true')
if self.build_id is not None:
argv.append('-t')
argv.append('build_id:{}'.format(self.build_id))
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
if sdk:
variant_name = self.sdk_variant_name
variant_id = self.sdk_variant_id
if variant_name is None and self.variant_name is not None:
variant_name = self.variant_name + ' (SDK)'
else:
variant_name = self.variant_name
variant_id = self.variant_id
if variant_name is None:
variant_name = artifact_prefix
if variant_id is None:
variant_id = artifact_prefix
argv.append('-t')
argv.append('variant:{}'.format(variant_name))
argv.append('-t')
argv.append('variant_id:{}'.format(
self.escape_variant_id(variant_id)
))
if packages:
logger.info('Installing packages:')
packages.sort()
for p in packages:
logger.info('- %s', p)
argv.append('-t')
argv.append('packages:{}'.format(
self.yaml_dump_one_line(packages)))
dest = os.path.join(scratch, 'runtimes', runtime)
os.makedirs(dest, 0o755, exist_ok=True)
dest = os.path.join(dest, 'packages.yaml')
with open(dest, 'w', encoding='utf-8') as writer:
yaml.safe_dump(packages, stream=writer)
script = self.runtime_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append('post_script:post_script')
pre_apt_script = self.runtime_details.get('pre_apt_script', '')
if pre_apt_script:
dest = os.path.join(scratch, 'pre_apt_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(pre_apt_script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append('pre_apt_script:pre_apt_script')
sources_tarball = sources_prefix + '.tar.gz'
debug_prefix = artifact_prefix + '-debug'
debug_tarball = debug_prefix + '.tar.gz'
sysroot_prefix = None # type: typing.Optional[str]
sysroot_tarball = None # type: typing.Optional[str]
if generate_sysroot_tarball:
sysroot_prefix = artifact_prefix + '-sysroot'
sysroot_tarball = sysroot_prefix + '.tar.gz'
argv.append('-t')
argv.append('sysroot_prefix:{}'.format(sysroot_prefix))
argv.append('-t')
argv.append(
'sysroot_tarball:{}'.format(
sysroot_tarball + '.new'))
if generate_source_directory:
os.makedirs(
os.path.join(
self.build_area,
generate_source_directory,
),
0o755,
exist_ok=True,
)
argv.append('-t')
argv.append(
'sources_directory:{}'.format(
generate_source_directory))
if generate_source_tarball is None:
generate_source_tarball = not generate_source_directory
if self.collect_source_code and generate_source_tarball:
sources_tarball = sources_prefix + '.tar.gz'
argv.append('-t')
argv.append(
'sources_tarball:{}'.format(
sources_tarball + '.new'))
sdk_details = self.runtime_details.get('sdk', {})
argv.append('-t')
argv.append('sdk:yes')
argv.append('-t')
if self.debug_symbols:
argv.append('debug_symbols:yes')
else:
argv.append('debug_symbols:')
argv.append('-t')
if self.collect_source_code:
argv.append('collect_source_code:yes')
else:
argv.append('collect_source_code:')
argv.append('-t')
if self.automatic_dbgsym:
argv.append('automatic_dbgsym:yes')
else:
argv.append('automatic_dbgsym:')
argv.append('-t')
argv.append('debug_tarball:' + debug_tarball + '.new')
argv.append('-t')
argv.append('debug_prefix:' + debug_prefix)
argv.append('-t')
argv.append('sources_prefix:' + sources_prefix)
sdk_packages = list(self.get_runtime_packages(
sdk_details.get('add_packages', [])
))
sdk_packages.extend(self.get_runtime_packages(
sdk_details.get('add_packages_multiarch', []),
multiarch=True,
))
# We probably have this anyway, but we need it for
# dpkg-scansources
if 'dpkg-dev' not in sdk_packages:
sdk_packages.append('dpkg-dev')
if sdk_packages:
logger.info('Installing extra packages for SDK:')
sdk_packages.sort()
for p in sdk_packages:
logger.info('- %s', p)
argv.append('-t')
argv.append(
'sdk_packages:{}'.format(
self.yaml_dump_one_line(sdk_packages)))
dest = os.path.join(scratch, 'runtimes', runtime)
os.makedirs(dest, 0o755, exist_ok=True)
dest = os.path.join(dest, 'sdk_packages.yaml')
with open(dest, 'w', encoding='utf-8') as writer:
yaml.safe_dump(sdk_packages, stream=writer)
script = sdk_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'sdk_post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append('sdk_post_script:sdk_post_script')
else: # not sdk
platform_details = self.runtime_details.get('platform', {})
script = platform_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'platform_post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append(
'platform_post_script:platform_post_script')
overlay = os.path.join(
scratch, 'runtimes', runtime, 'flatpak-overlay')
self.create_flatpak_manifest_overlay(
overlay, prefix, runtime, sdk=sdk)
overlay = os.path.join(
scratch, 'runtimes', runtime, 'apt-overlay')
self.configure_apt(overlay, self.final_apt_sources)
subprocess.check_call(argv)
output = os.path.join(
self.build_area,
sources_prefix + '.MISSING.txt',
)
if self.collect_source_code:
if os.path.exists(output) and self.strict:
raise SystemExit(
'Some source code was missing: aborting'
)
else:
with open(output, 'w') as writer:
writer.write('EVERYTHING\n')
if sysroot_prefix is not None:
assert sysroot_tarball is not None
output = os.path.join(self.build_area, sysroot_tarball)
os.rename(output + '.new', output)
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
output = os.path.join(
self.build_area, sysroot_prefix + '.Dockerfile')
with open(
os.path.join(
os.path.dirname(__file__), 'flatdeb',
'Dockerfile.in'),
'r',
encoding='utf-8',
) as reader:
content = reader.read()
content = content.replace(
'@sysroot_tarball@', sysroot_tarball)
with open(
output + '.new', 'w', encoding='utf-8'
) as writer:
writer.write(content)
os.rename(output + '.new', output)
if self.debug_symbols:
output = os.path.join(self.build_area, debug_tarball)
os.rename(output + '.new', output)
if self.ostree_commit and self.debug_symbols:
logger.info('Committing %s to OSTree', debug_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Debug/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
'--add-metadata-string',
'xa.metadata=' + self.metadata_debug.to_data()[0],
])
if self.collect_source_code and generate_source_tarball:
output = os.path.join(self.build_area, sources_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info(
'Committing %s to OSTree', sources_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Sources/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
'--add-metadata-string',
('xa.metadata='
+ self.metadata_sources.to_data()[0]),
output = os.path.join(self.build_area, out_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', out_tarball)
subprocess.check_call([
'time',
'ostree',
'--branch=runtime/{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
'--add-metadata-string',
'xa.metadata=' + self.metadata.to_data()[0],
if self.ostree_commit:
# Don't keep the history in this working repository:
# if history is desired, mirror the commits into a public
# repository and maintain history there.
subprocess.check_call([
'time',
'ostree',
'prune',
'--refs-only',
'--depth=1',
subprocess.check_call([
'time',
'flatpak',
'build-update-repo',
self.ostree_repo,
])
if self.export_bundles:
for suffix in ('.Platform', '.Sdk'):
bundle = '{}-{}-{}.bundle'.format(
prefix + suffix,
','.join(self.dpkg_archs),
self.runtime_branch,
)
output = os.path.join(self.build_area, bundle)
subprocess.check_call([
'time',
'flatpak',
'build-bundle',
'--runtime',
self.ostree_repo,
output + '.new',
prefix + suffix,
self.runtime_branch,
])
os.rename(output + '.new', output)
def configure_apt(self, overlay, apt_sources):
# type: (str, typing.Iterable[AptSource]) -> None
"""
Configure apt. We only do this once, so that all chroots
created from the same base have their version numbers
aligned.
"""
os.makedirs(
os.path.join(overlay, 'etc', 'apt', 'trusted.gpg.d'),
0o755,
exist_ok=True,
)
with open(
os.path.join(overlay, 'etc', 'apt', 'sources.list'),
'w',
encoding='utf-8'
) as writer:
for source in apt_sources:
writer.write('{}\n'.format(source))
for keyring in self.apt_keyrings:
if os.path.exists(os.path.join('suites', keyring)):
keyring = os.path.join('suites', keyring)
elif os.path.exists(keyring):
pass
raise RuntimeError('Cannot open {}'.format(keyring))
shutil.copyfile(
os.path.abspath(keyring),
os.path.join(
overlay,
'etc', 'apt', 'trusted.gpg.d',
os.path.basename(keyring),
),
)
def create_flatpak_manifest_overlay(
self,
overlay, # type: str
prefix, # type: str
runtime, # type: str
sdk=False,
):
metadata = os.path.join(overlay, 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = self.metadata
keyfile.set_string('Runtime', 'name', runtime)
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
keyfile.set_string(
'Runtime', 'x-flatdeb-sources',
'{}.Sdk.Sources/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
),
)
keyfile.set_string(
'Environment', 'XDG_DATA_DIRS',
':'.join([
'/app/share', '/usr/share', '/usr/share/runtime/share',
]),
)
if sdk:
keyfile.set_string(
'Extension {}.Sdk.Debug'.format(prefix),
'directory', 'lib/debug',
)
keyfile.set_boolean(
'Extension {}.Sdk.Debug'.format(prefix),
'autodelete', True,
)
keyfile.set_boolean(
'Extension {}.Sdk.Debug'.format(prefix),
'no-autodownload', True,
)
keyfile.set_string(
'Extension {}.Sdk.Sources'.format(prefix),
'directory', 'runtime/src',
)
keyfile.set_boolean(
'Extension {}.Sdk.Sources'.format(prefix),
'autodelete', True,
)
keyfile.set_boolean(
'Extension {}.Sdk.Sources'.format(prefix),
'no-autodownload', True,
)
search_path = []
for arch in self.dpkg_archs:
search_path.append('/app/lib/{}'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib')
keyfile.set_string(
'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
)
if True: # TODO: 'libgstreamer1.0-0' in installed:
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/extensions/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/extensions/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/gstreamer-1.0')
keyfile.set_string(
'Environment', 'GST_PLUGIN_SYSTEM_PATH',
':'.join(search_path),
)
if True: # TODO: 'libgirepository-1.0-1' in installed:
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/girepository-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/girepository-1.0')
'Environment', 'GI_TYPELIB_PATH',
':'.join(search_path),
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
for ext, detail in self.runtime_details.get(
'add-extensions', {}
).items():
group = 'Extension {}'.format(ext)
os.makedirs(
os.path.join(
overlay, 'usr',
detail['directory'],
),
0o755,
exist_ok=True,
)
for k, v in detail.items():
if isinstance(v, str):
keyfile.set_string(group, k, v)
elif isinstance(v, bool):
keyfile.set_boolean(group, k, v)
else:
raise RuntimeError(
'Unknown type {} in {}'.format(v, ext))
keyfile.save_to_file(metadata)
metadata = os.path.join(overlay, 'debug', 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = self.metadata_debug
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
keyfile.set_string('Runtime', 'name', runtime + '.Debug')
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
keyfile.save_to_file(metadata)
metadata = os.path.join(overlay, 'src', 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = self.metadata_sources
keyfile.set_string('Runtime', 'name', runtime + '.Sources')
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
keyfile.save_to_file(metadata)
def command_app(
self,
*,
app_branch, # type: str
yaml_manifest, # type: str
**kwargs
):
# type: (...) -> None
if not self.ostree_commit:
logger.error(
'flatdeb app --no-ostree-commit cannot work')
raise SystemExit(1)
with open(yaml_manifest, encoding='utf-8') as reader:
if self.runtime_branch is None:
self.runtime_branch = manifest.get('runtime-version')
if self.runtime_branch is None:
self.runtime_branch = self.apt_suite
self.app_branch = app_branch
if self.app_branch is None:
self.app_branch = manifest.get('branch')
if self.app_branch is None:
self.app_branch = 'master'
manifest['branch'] = self.app_branch
manifest['runtime-version'] = self.runtime_branch
if self.remote_url is None:
self.remote_url = 'file://{}'.format(
urllib.parse.quote(self.ostree_repo))
# We assume the build area has xattr support
self.ensure_build_area()
scratch = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb.',
dir=os.path.join(self.build_area, 'tmp'),
)
)
os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'remote-add', '--if-not-exists', '--no-gpg-verify',
'{}'.format(self.remote_url),
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'--url={}'.format(self.remote_url),
for runtime in (manifest['sdk'], manifest['runtime']):
# This may fail: we might already have it.
subprocess.call([
'XDG_DATA_HOME={}/home'.format(scratch),
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'flatpak', '--user',
'update',
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
for module in manifest.get('modules', []):
if isinstance(module, dict):
sources = module.setdefault('sources', [])
for source in sources:
if 'path' in source:
if source.get('type') == 'git':
clone = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-git.',
dir=scratch,
),
)
uploader = subprocess.Popen([
'tar',
'-cf-',
'-C', source['path'],
'.',
], stdout=subprocess.PIPE)
subprocess.check_call([
'tar',
'-xf-',
'-C', clone,
], stdin=uploader.stdout)
uploader.wait()
source['path'] = clone
else:
d = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-path.',
dir=scratch,
),
)
clone = os.path.join(
shutil.copyfile(
source['path'],
clone,
)
if GLib.file_test(
source['path'],
GLib.FileTest.IS_EXECUTABLE,
):
os.chmod(clone, 0o755)
else:
os.chmod(clone, 0o644)
source['path'] = clone
if 'x-flatdeb-apt-packages' in module:
packages = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-debs.',
dir=scratch,
),
)
shutil.copy2(
os.path.join(
os.path.dirname(__file__),
'flatdeb',
'collect-app-source-code',
),
packages
)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'flatpak', 'run',
'--filesystem={}'.format(packages),
'--share=network',
'--command=/usr/bin/env',
'{}/{}/{}'.format(
manifest['sdk'],
self.flatpak_arch,
'{}/collect-app-source-code'.format(packages),
'--export={}'.format(packages),
'--strip-source-version-suffix={}'.format(
self.strip_source_version_suffix),
os.remove(
os.path.join(packages, 'collect-app-source-code')
)
obtained = subprocess.check_output([
'sh', '-euc',
'cd "$1"\n'
'find * -type f -print0 | xargs -0 sha256sum -b\n'
'',
'sh', # argv[0]
packages,
for line in obtained:
sha256, f = line.split(' *', 1)
sources.append({
'dest': (os.path.dirname(f) or '.'),
'type': 'file',
'sha256': sha256,
'url': urllib.parse.urlunsplit((
'file',
'',
urllib.parse.quote(path),
'',
'',
))
})
json_manifest = os.path.join(scratch, manifest['id'] + '.json')
os.makedirs(
os.path.join(self.build_area, '.flatpak-builder'),
exist_ok=True,
)
if self.build_area != scratch:
subprocess.check_call([
'ln', '-nsf',
os.path.join(self.build_area, '.flatpak-builder'),
'{}/'.format(scratch),
with open(json_manifest, 'w', encoding='utf-8') as writer:
json.dump(manifest, writer, indent=2, sort_keys=True)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'sh', '-euc',
'cd "$1"; shift; exec "$@"',
'sh', # argv[0]
scratch, # directory to cd into
'--arch={}'.format(self.flatpak_arch),
'--bundle-sources',
os.path.join(scratch, 'workdir'),
json_manifest,
bundle = '{}-{}-{}.bundle'.format(
manifest['id'],
self.flatpak_arch,
manifest['branch'],
)
output = os.path.join(self.build_area, bundle)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
output + '.new',
manifest['id'],
manifest['branch'],
])
os.rename(output + '.new', output)
if sys.stderr.isatty():
try:
import colorlog
except ImportError:
logging.basicConfig()
else:
formatter = colorlog.ColoredFormatter(
'%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
else:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)