Newer
Older
self.yaml_dump_one_line(sdk_packages)))
dest = os.path.join(scratch, 'runtimes', runtime)
os.makedirs(dest, 0o755, exist_ok=True)
dest = os.path.join(dest, 'sdk_packages.yaml')
with open(dest, 'w', encoding='utf-8') as writer:
yaml.safe_dump(sdk_packages, stream=writer)
script = sdk_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'sdk_post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append('sdk_post_script:sdk_post_script')
else: # not sdk
platform_details = self.runtime_details.get('platform', {})
script = platform_details.get('post_script', '')
if script:
dest = os.path.join(scratch, 'platform_post_script')
with open(dest, 'w', encoding='utf-8') as writer:
writer.write('#!/bin/sh\n')
writer.write(script)
writer.write('\n')
os.chmod(dest, 0o755)
argv.append('-t')
argv.append(
'platform_post_script:platform_post_script')
overlay = os.path.join(scratch, 'runtimes', runtime, 'overlay')
self.create_flatpak_manifest_overlay(
overlay, prefix, runtime, sdk=sdk)
self.configure_apt(
os.path.join(scratch, 'runtimes', runtime, 'overlay'),
self.final_apt_sources,
)
subprocess.check_call(argv)
if sysroot_prefix is not None:
assert sysroot_tarball is not None
output = os.path.join(self.build_area, sysroot_tarball)
os.rename(output + '.new', output)
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
output = os.path.join(
self.build_area, sysroot_prefix + '.Dockerfile')
with open(
os.path.join(
os.path.dirname(__file__), 'flatdeb',
'Dockerfile.in'),
'r',
encoding='utf-8',
) as reader:
content = reader.read()
content = content.replace(
'@sysroot_tarball@', sysroot_tarball)
with open(
output + '.new', 'w', encoding='utf-8'
) as writer:
writer.write(content)
os.rename(output + '.new', output)
output = os.path.join(self.build_area, debug_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', debug_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Debug/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
output = os.path.join(self.build_area, sources_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', sources_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Sources/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
output = os.path.join(self.build_area, out_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', out_tarball)
subprocess.check_call([
'time',
'ostree',
'--branch=runtime/{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
if self.ostree_commit:
# Don't keep the history in this working repository:
# if history is desired, mirror the commits into a public
# repository and maintain history there.
subprocess.check_call([
'time',
'ostree',
'prune',
'--refs-only',
'--depth=1',
subprocess.check_call([
'time',
'flatpak',
'build-update-repo',
self.ostree_repo,
])
if self.export_bundles:
for suffix in ('.Platform', '.Sdk'):
bundle = '{}-{}-{}.bundle'.format(
prefix + suffix,
','.join(self.dpkg_archs),
self.runtime_branch,
)
output = os.path.join(self.build_area, bundle)
subprocess.check_call([
'time',
'flatpak',
'build-bundle',
'--runtime',
self.ostree_repo,
output + '.new',
prefix + suffix,
self.runtime_branch,
])
os.rename(output + '.new', output)
def configure_apt(self, overlay, apt_sources):
# type: (str, typing.Iterable[AptSource]) -> None
"""
Configure apt. We only do this once, so that all chroots
created from the same base have their version numbers
aligned.
"""
os.makedirs(
os.path.join(overlay, 'etc', 'apt', 'trusted.gpg.d'),
0o755,
exist_ok=True,
)
with open(
os.path.join(overlay, 'etc', 'apt', 'sources.list'),
'w',
encoding='utf-8'
) as writer:
for source in apt_sources:
writer.write('{}\n'.format(source))
for keyring in self.apt_keyrings:
if os.path.exists(os.path.join('suites', keyring)):
keyring = os.path.join('suites', keyring)
elif os.path.exists(keyring):
pass
raise RuntimeError('Cannot open {}'.format(keyring))
shutil.copyfile(
os.path.abspath(keyring),
os.path.join(
overlay,
'etc', 'apt', 'trusted.gpg.d',
os.path.basename(keyring),
),
)
def create_flatpak_manifest_overlay(
self,
overlay, # type: str
prefix, # type: str
runtime, # type: str
sdk=False,
):
metadata = os.path.join(overlay, 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime)
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
keyfile.set_string(
'Runtime', 'x-flatdeb-sources',
'{}.Sdk.Sources/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
),
)
keyfile.set_string(
'Environment', 'XDG_DATA_DIRS',
':'.join([
'/app/share', '/usr/share', '/usr/share/runtime/share',
]),
)
if sdk:
keyfile.set_string(
'Extension {}.Sdk.Debug'.format(prefix),
'directory', 'lib/debug',
)
keyfile.set_boolean(
'Extension {}.Sdk.Debug'.format(prefix),
'autodelete', True,
)
keyfile.set_boolean(
'Extension {}.Sdk.Debug'.format(prefix),
'no-autodownload', True,
)
keyfile.set_string(
'Extension {}.Sdk.Sources'.format(prefix),
'directory', 'runtime/src',
)
keyfile.set_boolean(
'Extension {}.Sdk.Sources'.format(prefix),
'autodelete', True,
)
keyfile.set_boolean(
'Extension {}.Sdk.Sources'.format(prefix),
'no-autodownload', True,
)
search_path = []
for arch in self.dpkg_archs:
search_path.append('/app/lib/{}'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib')
keyfile.set_string(
'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
)
if True: # TODO: 'libgstreamer1.0-0' in installed:
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/extensions/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/extensions/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/gstreamer-1.0')
keyfile.set_string(
'Environment', 'GST_PLUGIN_SYSTEM_PATH',
':'.join(search_path),
)
if True: # TODO: 'libgirepository-1.0-1' in installed:
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/girepository-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/girepository-1.0')
'Environment', 'GI_TYPELIB_PATH',
':'.join(search_path),
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
for ext, detail in self.runtime_details.get(
'add-extensions', {}
).items():
group = 'Extension {}'.format(ext)
os.makedirs(
os.path.join(
overlay, 'usr',
detail['directory'],
),
0o755,
exist_ok=True,
)
for k, v in detail.items():
if isinstance(v, str):
keyfile.set_string(group, k, v)
elif isinstance(v, bool):
keyfile.set_boolean(group, k, v)
else:
raise RuntimeError(
'Unknown type {} in {}'.format(v, ext))
keyfile.save_to_file(metadata)
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
metadata = os.path.join(overlay, 'debug', 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime + '.Debug')
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
keyfile.save_to_file(metadata)
metadata = os.path.join(overlay, 'src', 'metadata')
os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime + '.Sources')
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}'.format(
prefix,
self.flatpak_arch,
self.runtime_branch,
)
)
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
if self.build_id is not None:
keyfile.set_string(
'Runtime', 'x-flatdeb-build-id', self.build_id,
)
keyfile.save_to_file(metadata)
def command_app(
self,
*,
app_branch, # type: str
yaml_manifest, # type: str
**kwargs
):
# type: (...) -> None
if not self.ostree_commit:
logger.error(
'flatdeb app --no-ostree-commit cannot work')
raise SystemExit(1)
with open(yaml_manifest, encoding='utf-8') as reader:
if self.runtime_branch is None:
self.runtime_branch = manifest.get('runtime-version')
if self.runtime_branch is None:
self.runtime_branch = self.apt_suite
self.app_branch = app_branch
if self.app_branch is None:
self.app_branch = manifest.get('branch')
if self.app_branch is None:
self.app_branch = 'master'
manifest['branch'] = self.app_branch
manifest['runtime-version'] = self.runtime_branch
# We assume the build area has xattr support
self.ensure_build_area()
scratch = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb.',
dir=os.path.join(self.build_area, 'tmp'),
)
)
os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'remote-add', '--if-not-exists', '--no-gpg-verify',
'file://{}'.format(urllib.parse.quote(self.ostree_repo)),
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'--url=file://{}'.format(urllib.parse.quote(self.ostree_repo)),
for runtime in (manifest['sdk'], manifest['runtime']):
# This may fail: we might already have it.
subprocess.call([
'XDG_DATA_HOME={}/home'.format(scratch),
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'flatpak', '--user',
'update',
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
for module in manifest.get('modules', []):
if isinstance(module, dict):
sources = module.setdefault('sources', [])
for source in sources:
if 'path' in source:
if source.get('type') == 'git':
clone = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-git.',
dir=scratch,
),
)
uploader = subprocess.Popen([
'tar',
'-cf-',
'-C', source['path'],
'.',
], stdout=subprocess.PIPE)
subprocess.check_call([
'tar',
'-xf-',
'-C', clone,
], stdin=uploader.stdout)
uploader.wait()
source['path'] = clone
else:
d = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-path.',
dir=scratch,
),
)
clone = os.path.join(
shutil.copyfile(
source['path'],
clone,
)
if GLib.file_test(
source['path'],
GLib.FileTest.IS_EXECUTABLE,
):
os.chmod(clone, 0o755)
else:
os.chmod(clone, 0o644)
source['path'] = clone
if 'x-flatdeb-apt-packages' in module:
packages = stack.enter_context(
TemporaryDirectory(
prefix='flatdeb-debs.',
dir=scratch,
),
)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'flatpak', 'run',
'--filesystem={}'.format(packages),
'--share=network',
'--command=/usr/bin/env',
'{}/{}/{}'.format(
manifest['sdk'],
self.flatpak_arch,
'export={}'.format(packages),
'sh',
'-euc',
'install -d /var/cache/apt/archives/partial\n'
'fakeroot apt-get update\n'
'fakeroot apt-get -y --download-only \\\n'
' --no-install-recommends install "$@"\n'
'for x in /var/cache/apt/archives/*.deb; do\n'
' package="$(dpkg-deb -f "$x" Package)"\n'
' source="$(dpkg-deb -f "$x" Source)"\n'
' bu="$(dpkg-deb -f "$x" Built-Using)"\n'
' version="$(dpkg-deb -f "$x" Version)"\n'
' if [ -z "$source" ]; then\n'
' source="$package"\n'
' fi\n'
' if [ "${source% (*}" != "$source" ]; then\n'
' version="${source#* (}"\n'
' version="${version%)}"\n'
' source="${source% (*}"\n'
' fi\n'
' ( cd "$export" && \\\n'
' apt-get -y --download-only \\\n'
' -oAPT::Get::Only-Source=true source \\\n'
' "$source=$version"\n'
' )\n'
' if [ -n "$bu" ]; then\n'
' oldIFS="$IFS"\n'
' IFS=","\n'
' for dep in $bu; do\n'
' bu="$(echo "$bu" | tr -d " ")"\n'
' version="${bu#*(=}"\n'
' version="${version%)}"\n'
' source="${bu%(*}"\n'
' ( cd "$export" && \\\n'
' apt-get -y --download-only \\\n'
' -oAPT::Get::Only-Source=true \\\n'
' source "$source=$version"\n'
' )\n'
' done\n'
' IFS="$oldIFS"\n'
' fi\n'
'mv /var/cache/apt/archives/*.deb "$export"\n'
'mv /var/lib/apt/lists "$export"\n'
'',
'sh', # argv[0]
] + module['x-flatdeb-apt-packages'])
obtained = subprocess.check_output([
'sh', '-euc',
'cd "$1"\n'
'find * -type f -print0 | xargs -0 sha256sum -b\n'
'',
'sh', # argv[0]
packages,
for line in obtained:
sha256, f = line.split(' *', 1)
sources.append({
'dest': (os.path.dirname(f) or '.'),
'type': 'file',
'sha256': sha256,
'url': urllib.parse.urlunsplit((
'file',
'',
urllib.parse.quote(path),
'',
'',
))
})
json_manifest = os.path.join(scratch, manifest['id'] + '.json')
os.makedirs(
os.path.join(self.build_area, '.flatpak-builder'),
exist_ok=True,
)
if self.build_area != scratch:
subprocess.check_call([
'ln', '-nsf',
os.path.join(self.build_area, '.flatpak-builder'),
'{}/'.format(scratch),
with open(json_manifest, 'w', encoding='utf-8') as writer:
json.dump(manifest, writer, indent=2, sort_keys=True)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
'sh', '-euc',
'cd "$1"; shift; exec "$@"',
'sh', # argv[0]
scratch, # directory to cd into
'--arch={}'.format(self.flatpak_arch),
'--bundle-sources',
os.path.join(scratch, 'workdir'),
json_manifest,
bundle = '{}-{}-{}.bundle'.format(
manifest['id'],
self.flatpak_arch,
manifest['branch'],
)
output = os.path.join(self.build_area, bundle)
subprocess.check_call([
'XDG_DATA_HOME={}/home'.format(scratch),
output + '.new',
manifest['id'],
manifest['branch'],
])
os.rename(output + '.new', output)
if sys.stderr.isatty():
try:
import colorlog
except ImportError:
pass
else:
formatter = colorlog.ColoredFormatter(
'%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
else:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
try:
Builder().run_command_line()
except KeyboardInterrupt:
raise SystemExit(130)
except subprocess.CalledProcessError as e:
logger.error('%s', e)
raise SystemExit(1)