Newer
Older
if generate_sysroot_tarball:
sysroot_prefix = artifact_prefix + '-sysroot'
sysroot_tarball = sysroot_prefix + '.tar.gz'
argv.append('-t')
argv.append('sysroot_prefix:{}'.format(sysroot_prefix))
argv.append('-t')
argv.append(
'sysroot_tarball:{}'.format(
sysroot_tarball + '.new'))
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
if generate_source_directory:
os.makedirs(
os.path.join(
self.build_area,
generate_source_directory,
),
0o755,
exist_ok=True,
)
argv.append('-t')
argv.append(
'sources_directory:{}'.format(
generate_source_directory))
if generate_source_tarball is None:
generate_source_tarball = not generate_source_directory
if generate_source_tarball:
sources_tarball = sources_prefix + '.tar.gz'
argv.append('-t')
argv.append(
'sources_tarball:{}'.format(
sources_tarball + '.new'))
sdk_details = self.runtime_details.get('sdk', {})
sdk_packages = list(sdk_details.get('add_packages', []))
argv.append('-t')
argv.append('sdk:yes')
argv.append('-t')
argv.append('debug_tarball:' + debug_tarball + '.new')
argv.append('-t')
argv.append('debug_prefix:' + debug_prefix)
argv.append('-t')
argv.append('sources_prefix:' + sources_prefix)
for p in sdk_details.get('add_packages_multiarch', []):
for a in self.dpkg_archs:
sdk_packages.append(p + ':' + a)
if sdk_packages:
logger.info('Installing extra packages for SDK:')
sdk_packages.sort()
for p in sdk_packages:
logger.info('- %s', p)
argv.append('-t')
argv.append(
'sdk_packages:{}'.format(
self.yaml_dump_one_line(sdk_packages)))
dest = os.path.join(scratch, 'runtimes', runtime)
os.makedirs(dest, 0o755, exist_ok=True)
dest = os.path.join(dest, 'sdk_packages.yaml')
with open(dest, 'w', encoding='utf-8') as writer:
yaml.safe_dump(sdk_packages, stream=writer)
script = sdk_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'sdk_post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append('sdk_post_script:sdk_post_script')
else: # not sdk
platform_details = self.runtime_details.get('platform', {})
script = platform_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'platform_post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append(
'platform_post_script:platform_post_script')
overlay = os.path.join(scratch, 'runtimes', runtime, 'overlay')
self.create_flatpak_manifest_overlay(
overlay, prefix, runtime, sdk=sdk)
self.configure_apt(
os.path.join(scratch, 'runtimes', runtime, 'overlay'),
self.final_apt_sources,
)
subprocess.check_call(argv)
if sysroot_prefix is not None:
assert sysroot_tarball is not None
output = os.path.join(self.build_area, sysroot_tarball)
os.rename(output + '.new', output)
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
output = os.path.join(
self.build_area, sysroot_prefix + '.Dockerfile')
with open(
os.path.join(
os.path.dirname(__file__), 'flatdeb',
'Dockerfile.in'),
'r',
encoding='utf-8',
) as reader:
content = reader.read()
content = content.replace(
'@sysroot_tarball@', sysroot_tarball)
with open(
output + '.new', 'w', encoding='utf-8'
) as writer:
writer.write(content)
os.rename(output + '.new', output)
output = os.path.join(self.build_area, debug_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', debug_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Debug/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
if generate_source_tarball:
output = os.path.join(self.build_area, sources_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', sources_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Sources/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
output = os.path.join(self.build_area, out_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', out_tarball)
subprocess.check_call([
'time',
'ostree',
'--branch=runtime/{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
if self.ostree_commit:
# Don't keep the history in this working repository:
# if history is desired, mirror the commits into a public
# repository and maintain history there.
subprocess.check_call([
'time',
'ostree',
'prune',
'--refs-only',
'--depth=1',
subprocess.check_call([
'time',
'flatpak',
'build-update-repo',
self.ostree_repo,
])
if self.export_bundles:
for suffix in ('.Platform', '.Sdk'):
bundle = '{}-{}-{}.bundle'.format(
prefix + suffix,
','.join(self.dpkg_archs),
self.runtime_branch,
)
output = os.path.join(self.build_area, bundle)
subprocess.check_call([
'time',
'flatpak',
'build-bundle',
'--runtime',
self.ostree_repo,
output + '.new',
prefix + suffix,
self.runtime_branch,
])
os.rename(output + '.new', output)
def configure_apt(self, overlay, apt_sources):
# type: (str, typing.Iterable[AptSource]) -> None
"""
Configure apt. We only do this once, so that all chroots
created from the same base have their version numbers
aligned.
"""
os.makedirs(
os.path.join(overlay, 'etc', 'apt', 'trusted.gpg.d'),
0o755,
exist_ok=True,
)
with open(
os.path.join(overlay, 'etc', 'apt', 'sources.list'),
'w',
encoding='utf-8'
) as writer:
for source in apt_sources:
writer.write('{}\n'.format(source))
for keyring in self.apt_keyrings:
if os.path.exists(os.path.join('suites', keyring)):
keyring = os.path.join('suites', keyring)
elif os.path.exists(keyring):
pass
raise RuntimeError('Cannot open {}'.format(keyring))
shutil.copyfile(
os.path.abspath(keyring),
os.path.join(
overlay,
'etc', 'apt', 'trusted.gpg.d',
os.path.basename(keyring),
),
)
def create_flatpak_manifest_overlay(
self,
overlay, # type: str
prefix, # type: str
runtime, # type: str
sdk=False,
):
metadata = os.path.join(overlay, 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime)
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
keyfile.set_string(
'Runtime', 'x-flatdeb-sources',
'{}.Sdk.Sources/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
),
)
keyfile.set_string(
'Environment', 'XDG_DATA_DIRS',
':'.join([
'/app/share', '/usr/share', '/usr/share/runtime/share',
]),
)
if sdk:
keyfile.set_string(
'Extension {}.Sdk.Debug'.format(prefix),
'directory', 'lib/debug',
)
keyfile.set_boolean(
'Extension {}.Sdk.Debug'.format(prefix),
'autodelete', True,
)
keyfile.set_boolean(
'Extension {}.Sdk.Debug'.format(prefix),
'no-autodownload', True,
)
keyfile.set_string(
'Extension {}.Sdk.Sources'.format(prefix),
'directory', 'runtime/src',
)
keyfile.set_boolean(
'Extension {}.Sdk.Sources'.format(prefix),
'autodelete', True,
)
keyfile.set_boolean(
'Extension {}.Sdk.Sources'.format(prefix),
'no-autodownload', True,
)
search_path = []
for arch in self.dpkg_archs:
search_path.append('/app/lib/{}'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib')
keyfile.set_string(
'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
)
if True: # TODO: 'libgstreamer1.0-0' in installed:
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/extensions/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/extensions/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/gstreamer-1.0')
keyfile.set_string(
'Environment', 'GST_PLUGIN_SYSTEM_PATH',
':'.join(search_path),
)
if True: # TODO: 'libgirepository-1.0-1' in installed:
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/girepository-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/girepository-1.0')
'Environment', 'GI_TYPELIB_PATH',
':'.join(search_path),
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
for ext, detail in self.runtime_details.get(
'add-extensions', {}
).items():
group = 'Extension {}'.format(ext)
os.makedirs(
os.path.join(
overlay, 'usr',
detail['directory'],
),
0o755,
exist_ok=True,
)
for k, v in detail.items():
if isinstance(v, str):
keyfile.set_string(group, k, v)
elif isinstance(v, bool):
keyfile.set_boolean(group, k, v)
else:
raise RuntimeError(
'Unknown type {} in {}'.format(v, ext))
keyfile.save_to_file(metadata)
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
metadata = os.path.join(overlay, 'debug', 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime + '.Debug')
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
keyfile.save_to_file(metadata)
metadata = os.path.join(overlay, 'src', 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime + '.Sources')
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
keyfile.save_to_file(metadata)
def command_app(
self,
*,
app_branch, # type: str
yaml_manifest, # type: str
**kwargs
):
# type: (...) -> None
if not self.ostree_commit:
logger.error(
'flatdeb app --no-ostree-commit cannot work')
raise SystemExit(1)
with open(yaml_manifest, encoding='utf-8') as reader:
if self.runtime_branch is None:
self.runtime_branch = manifest.get('runtime-version')
if self.runtime_branch is None:
self.runtime_branch = self.apt_suite
self.app_branch = app_branch
if self.app_branch is None:
self.app_branch = manifest.get('branch')
if self.app_branch is None:
self.app_branch = 'master'
manifest['branch'] = self.app_branch
manifest['runtime-version'] = self.runtime_branch
# We assume the build area has xattr support
self.ensure_build_area()
scratch = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb.',
dir=os.path.join(self.build_area, 'tmp'),
)
)
os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'remote-add', '--if-not-exists', '--no-gpg-verify',
'file://{}'.format(urllib.parse.quote(self.ostree_repo)),
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'--url=file://{}'.format(urllib.parse.quote(self.ostree_repo)),
for runtime in (manifest['sdk'], manifest['runtime']):
# This may fail: we might already have it.
subprocess.call([
'XDG_DATA_HOME={}/home'.format(scratch),
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'flatpak', '--user',
'update',
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
for module in manifest.get('modules', []):
if isinstance(module, dict):
sources = module.setdefault('sources', [])
for source in sources:
if 'path' in source:
if source.get('type') == 'git':
clone = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-git.',
dir=scratch,
),
)
uploader = subprocess.Popen([
'tar',
'-cf-',
'-C', source['path'],
'.',
], stdout=subprocess.PIPE)
subprocess.check_call([
'tar',
'-xf-',
'-C', clone,
], stdin=uploader.stdout)
uploader.wait()
source['path'] = clone
else:
d = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-path.',
dir=scratch,
),
)
clone = os.path.join(
shutil.copyfile(
source['path'],
clone,
)
if GLib.file_test(
source['path'],
GLib.FileTest.IS_EXECUTABLE,
):
os.chmod(clone, 0o755)
else:
os.chmod(clone, 0o644)
source['path'] = clone
if 'x-flatdeb-apt-packages' in module:
packages = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-debs.',
dir=scratch,
),
)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'flatpak', 'run',
'--filesystem={}'.format(packages),
'--share=network',
'--command=/usr/bin/env',
'{}/{}/{}'.format(
manifest['sdk'],
self.flatpak_arch,
'export={}'.format(packages),
'sh',
'-euc',
'install -d /var/cache/apt/archives/partial\n'
'fakeroot apt-get update\n'
'fakeroot apt-get -y --download-only \\\n'
' --no-install-recommends install "$@"\n'
'for x in /var/cache/apt/archives/*.deb; do\n'
' package="$(dpkg-deb -f "$x" Package)"\n'
' source="$(dpkg-deb -f "$x" Source)"\n'
' bu="$(dpkg-deb -f "$x" Built-Using)"\n'
' version="$(dpkg-deb -f "$x" Version)"\n'
' if [ -z "$source" ]; then\n'
' source="$package"\n'
' fi\n'
' if [ "${source% (*}" != "$source" ]; then\n'
' version="${source#* (}"\n'
' version="${version%)}"\n'
' source="${source% (*}"\n'
' fi\n'
' ( cd "$export" && \\\n'
' apt-get -y --download-only \\\n'
' -oAPT::Get::Only-Source=true source \\\n'
' "$source=$version"\n'
' )\n'
' if [ -n "$bu" ]; then\n'
' oldIFS="$IFS"\n'
' IFS=","\n'
' for dep in $bu; do\n'
' bu="$(echo "$bu" | tr -d " ")"\n'
' version="${bu#*(=}"\n'
' version="${version%)}"\n'
' source="${bu%(*}"\n'
' ( cd "$export" && \\\n'
' apt-get -y --download-only \\\n'
' -oAPT::Get::Only-Source=true \\\n'
' source "$source=$version"\n'
' )\n'
' done\n'
' IFS="$oldIFS"\n'
' fi\n'
'mv /var/cache/apt/archives/*.deb "$export"\n'
'mv /var/lib/apt/lists "$export"\n'
'',
'sh', # argv[0]
] + module['x-flatdeb-apt-packages'])
obtained = subprocess.check_output([
'sh', '-euc',
'cd "$1"\n'
'find * -type f -print0 | xargs -0 sha256sum -b\n'
'',
'sh', # argv[0]
packages,
for line in obtained:
sha256, f = line.split(' *', 1)
sources.append({
'dest': (os.path.dirname(f) or '.'),
'type': 'file',
'sha256': sha256,
'url': urllib.parse.urlunsplit((
'file',
'',
urllib.parse.quote(path),
'',
'',
))
})
json_manifest = os.path.join(scratch, manifest['id'] + '.json')
os.makedirs(
os.path.join(self.build_area, '.flatpak-builder'),
exist_ok=True,
)
if self.build_area != scratch:
subprocess.check_call([
'ln', '-nsf',
os.path.join(self.build_area, '.flatpak-builder'),
'{}/'.format(scratch),
with open(json_manifest, 'w', encoding='utf-8') as writer:
json.dump(manifest, writer, indent=2, sort_keys=True)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'sh', '-euc',
'cd "$1"; shift; exec "$@"',
'sh', # argv[0]
scratch, # directory to cd into
'--arch={}'.format(self.flatpak_arch),
'--bundle-sources',
os.path.join(scratch, 'workdir'),
json_manifest,
bundle = '{}-{}-{}.bundle'.format(
manifest['id'],
self.flatpak_arch,
manifest['branch'],
)
output = os.path.join(self.build_area, bundle)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
output + '.new',
manifest['id'],
manifest['branch'],
])
os.rename(output + '.new', output)
if sys.stderr.isatty():
try:
import colorlog
except ImportError:
pass
else:
formatter = colorlog.ColoredFormatter(
'%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
else:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
try:
Builder().run_command_line()
except KeyboardInterrupt:
raise SystemExit(130)
except subprocess.CalledProcessError as e:
logger.error('%s', e)
raise SystemExit(1)