Newer
Older
'--force-depends',
] + unwanted)
# Second round of removals.
for package in [
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
'busybox-initramfs',
'debconf',
'debian-archive-keyring',
'e2fsprogs',
'init',
'init-system-helpers',
'insserv',
'iproute',
'login',
'mount',
'mountall',
'passwd',
'systemd',
'systemd-sysv',
'sysv-rc',
'ubuntu-archive-keyring',
'ubuntu-keyring',
'udev',
'upstart',
]:
if package in installed:
unwanted.append(package)
if 'perl' not in installed:
unwanted.append('perl-base')
logger.info('Packages we will forcibly remove (second round):')
for p in sorted(unwanted):
logger.info('- %s', p)
if unwanted:
nspawn.check_call([
'dpkg', '--purge', '--force-remove-essential',
'--force-depends',
] + unwanted)
installed = nspawn.list_packages_ignore_arch()
logger.info('Packages included in platform:')
for p in sorted(installed):
if p != 'dpkg':
logger.info('- %s', p)
script = self.runtime_details.get('post_script', '')
if script:
logger.info('Running custom script...')
nspawn.check_call([
'sh', '-c', script,
])
logger.info('... done')
script = platform_details.get('post_script', '')
if script:
logger.info('Running custom platform script...')
nspawn.check_call([
'sh', '-c', script,
])
logger.info('... done')
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
# We have to do this before removing dpkg :-)
nspawn.write_manifest()
# This has to be last for obvious reasons!
nspawn.check_call([
'dpkg', '--purge', '--force-remove-essential',
'--force-depends',
'dpkg',
])
return installed
def ostreeify(self, prefix, chroot, sdk=False, packages=()):
"""
Move things around to turn a chroot into a runtime.
"""
if sdk:
installed = self.sdkize(chroot)
else:
installed = self.platformize(chroot)
with NspawnWorker(
self.root_worker,
chroot,
) as nspawn:
nspawn.check_call([
'find', '/', '-xdev', '(',
'-lname', '/etc/alternatives/*', '-o',
'-lname', '/etc/locale.alias',
')', '-exec', 'sh', '-euc',
'set -e\n'
'while [ $# -gt 0 ]; do\n'
' old="$(readlink "$1")"\n'
' echo "Making $1 a hard link to $target (was $old)"\n'
' rm -f "$1"\n'
' cp -al "$target" "$1"\n'
' fi\n'
' shift\n'
'done'
'',
'sh', # argv[0] for the one-line shell script
'{}', '+',
])
self.root_worker.check_call([
'chmod', '-R', '--changes', 'a-s,o-t,u=rwX,og=rX', chroot,
])
self.root_worker.check_call([
'chown', '-R', '--changes', 'root:root', chroot,
])
self.root_worker.check_call([
'rm', '-fr', '--one-file-system',
'{}/usr/local'.format(chroot),
])
# Merge /usr the hard way, if necessary.
if not self.suite_details.get('can_merge_usr', False):
self.usrmerge(chroot)
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
if sdk:
runtime = prefix + '.Sdk'
self.root_worker.check_call([
'rm', '-fr', '--one-file-system',
'{}/etc/group-'.format(chroot),
'{}/etc/gshadow-'.format(chroot),
'{}/etc/passwd-'.format(chroot),
'{}/etc/shadow-'.format(chroot),
'{}/etc/subuid-'.format(chroot),
'{}/etc/subgid-'.format(chroot),
'{}/var/backups'.format(chroot),
'{}/var/cache'.format(chroot),
'{}/var/lib/dpkg/status-old'.format(chroot),
'{}/var/lib/dpkg/statoverride'.format(chroot),
])
self.root_worker.check_call([
'install', '-d',
'{}/var/cache/apt/archives/partial'.format(chroot),
'{}/var/lib/extrausers'.format(chroot),
])
self.root_worker.check_call([
'touch', '{}/var/cache/apt/archives/partial/.exists'.format(chroot),
])
# This is only useful if the SDK has libnss-extrausers
self.root_worker.check_call([
'cp', '{}/etc/passwd'.format(chroot),
'{}/var/lib/extrausers/passwd'.format(chroot),
])
self.root_worker.check_call([
'cp', '{}/etc/group'.format(chroot),
'{}/var/lib/extrausers/groups'.format(chroot),
])
self.root_worker.check_call([
'mv', '{}/etc'.format(chroot),
'{}/usr/etc'.format(chroot),
])
self.root_worker.check_call([
'mv', '{}/var'.format(chroot),
'{}/usr/var'.format(chroot),
])
else:
runtime = prefix + '.Platform'
self.root_worker.check_call([
'rm', '-fr', '--one-file-system',
'{}/etc'.format(chroot),
'{}/share/bash-completion'.format(chroot),
'{}/share/bug'.format(chroot),
'{}/var'.format(chroot),
])
# TODO: Move lib/debug, zoneinfo, locales into extensions
# TODO: Hook point for GL, instead of just Mesa
# TODO: GStreamer extension
# TODO: Icon theme, Gtk theme extension
# TODO: VAAPI extension
# TODO: SDK extension
self.root_worker.check_call([
'install', '-d', '{}/ostree/main'.format(chroot),
])
self.root_worker.check_call([
'mv', '{}/usr'.format(chroot),
'{}/ostree/main/files'.format(chroot),
])
ref = 'runtime/{}/{}/{}'.format(
runtime, self.flatpak_arch, self.runtime_branch,
with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
metadata = os.path.join(t, 'metadata')
keyfile = GLib.KeyFile()
keyfile.set_string('Runtime', 'name', runtime)
keyfile.set_string(
'Runtime', 'runtime',
'{}.Platform/{}/{}\n'.format(
prefix,
self.flatpak_arch,
)
)
keyfile.set_string(
'Runtime', 'sdk',
'{}.Sdk/{}/{}\n'.format(
prefix,
self.flatpak_arch,
)
)
keyfile.set_string(
'Environment', 'XDG_DATA_DIRS',
':'.join([
'/app/share', '/usr/share', '/usr/share/runtime/share',
]),
)
search_path = []
for arch in self.dpkg_archs:
search_path.append('/app/lib/{}'.format(self.multiarch_tuple(arch)))
search_path.append('/app/lib')
keyfile.set_string(
'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
)
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/extensions/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/extensions/gstreamer-1.0')
for arch in self.dpkg_archs:
search_path.append(
'/usr/lib/{}/gstreamer-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/usr/lib/gstreamer-1.0')
keyfile.set_string(
'Environment', 'GST_PLUGIN_SYSTEM_PATH',
':'.join(search_path),
search_path = []
for arch in self.dpkg_archs:
search_path.append(
'/app/lib/{}/girepository-1.0'.format(
self.multiarch_tuple(arch)))
search_path.append('/app/lib/girepository-1.0')
keyfile.set_string(
'Environment', 'GI_TYPELIB_PATH',
':'.join(search_path),
keyfile.set_string(
'Runtime', 'x-flatdeb-version', VERSION,
)
for ext, detail in self.runtime_details.get(
'add-extensions', {}
).items():
group = 'Extension {}'.format(ext)
self.root_worker.check_call([
'install', '-d',
'{}/ostree/main/files/{}'.format(chroot, detail['directory']),
])
for k, v in detail.items():
if isinstance(v, str):
keyfile.set_string(group, k, v)
elif isinstance(v, bool):
keyfile.set_boolean(group, k, v)
else:
raise RuntimeError(
'Unknown type {} in {}'.format(v, ext))
keyfile.save_to_file(metadata)
self.root_worker.install_file(
metadata,
'{}/ostree/main/metadata'.format(chroot),
)
self.worker.check_call([
'time',
'ostree',
'--repo=' + self.remote_repo,
'commit',
'--branch=' + ref,
'--subject=Update',
'--tree=dir={}/ostree/main'.format(chroot),
'--fsync=false',
])
# Don't keep the history in this working repository:
# if history is desired, mirror the commits into a public
# repository and maintain history there.
self.worker.check_call([
'time',
'ostree',
'--repo=' + self.remote_repo,
'prune',
'--refs-only',
'--depth=1',
])
if not isinstance(self.worker, HostWorker):
self.worker.check_call([
'time',
'flatpak',
'build-update-repo',
self.remote_repo,
])
with self.worker.remote_dir_context(self.remote_repo) as mount:
'ostree',
'--repo={}'.format(self.repo),
'remote',
'delete',
'flatdeb-worker',
])
print('^ It is OK if that failed with "remote not found"')
'ostree',
'--repo={}'.format(self.repo),
'remote',
'add',
'--no-gpg-verify',
'flatdeb-worker',
'file://' + urllib.parse.quote(mount),
])
'ostree',
'--repo={}'.format(self.repo),
'pull',
'--disable-fsync',
'--mirror',
'--untrusted',
'flatdeb-worker',
'runtime/{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
'ostree',
'--repo={}'.format(self.repo),
'remote',
'delete',
'flatdeb-worker',
])
self.host_worker.check_call([
'time',
'flatpak',
'build-update-repo',
self.repo,
])
def command_app(self, *, app_branch, yaml_manifest, **kwargs):
with open(yaml_manifest) as reader:
if self.runtime_branch is None:
self.runtime_branch = manifest.get('runtime-version')
if self.runtime_branch is None:
self.runtime_branch = self.apt_suite
self.app_branch = app_branch
if self.app_branch is None:
self.app_branch = manifest.get('branch')
if self.app_branch is None:
self.app_branch = 'master'
manifest['branch'] = self.app_branch
manifest['runtime-version'] = self.runtime_branch
with ExitStack() as stack:
stack.enter_context(self.worker)
self.ensure_build_area()
self.ensure_remote_repo()
t = stack.enter_context(
TemporaryDirectory(prefix='flatpak-app.')
)
'mkdir', '-p', '{}/home'.format(self.remote_build_area),
if isinstance(self.worker, SshWorker):
# ostree over sshfs is painfully slow so do something different
self.host_worker.call([
'time',
'rsync',
'-aW',
'--no-i-r',
'--info=progress2',
'{}/'.format(self.repo),
'{}:{}/'.format(self.worker.remote, self.remote_repo),
])
elif not isinstance(self.worker, HostWorker):
with self.worker.remote_dir_context(self.remote_repo) as mount:
'ostree',
'--repo={}'.format(mount),
'remote',
'delete',
'flatdeb-host',
])
print('^ It is OK if that failed with "remote not found"')
'ostree',
'--repo={}'.format(mount),
'remote',
'add',
'--no-gpg-verify',
'flatdeb-host',
'file://' + urllib.parse.quote(self.repo),
])
'ostree',
'--repo={}'.format(mount),
'pull',
'--disable-fsync',
'--mirror',
'flatdeb-host',
'runtime/{}/{}/{}'.format(
manifest['sdk'],
self.flatpak_arch,
self.runtime_branch,
'ostree',
'--repo={}'.format(mount),
'pull',
'--disable-fsync',
'--mirror',
'flatdeb-host',
'runtime/{}/{}/{}'.format(
manifest['runtime'],
self.flatpak_arch,
self.runtime_branch,
'ostree',
'--repo={}'.format(mount),
'remote',
'delete',
'flatdeb-host',
])
self.worker.check_call([
'time',
'flatpak',
'build-update-repo',
self.remote_repo,
])
'env',
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'flatpak', '--user',
'remote-add', '--if-not-exists', '--no-gpg-verify',
'file://{}'.format(urllib.parse.quote(self.remote_repo)),
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'remote-modify', '--no-gpg-verify',
'--url=file://{}'.format(urllib.parse.quote(self.remote_repo)),
'flatdeb',
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
for runtime in (manifest['sdk'], manifest['runtime']):
if self.worker.call([
'env',
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'flatpak', '--user',
'install', 'flatdeb',
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
]) != 0:
self.worker.check_call([
'env',
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'flatpak', '--user',
'update',
'{}/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
])
for module in manifest.get('modules', []):
if isinstance(module, dict):
sources = module.setdefault('sources', [])
for source in sources:
if 'path' in source:
if source.get('type') == 'git':
clone = self.worker.check_output([
'mktemp', '-d',
'-p', self.worker.scratch,
'flatdeb-git.XXXXXX',
'tar',
'-cf-',
'-C', source['path'],
'.',
], stdout=subprocess.PIPE)
self.worker.check_call([
'tar',
'-xf-',
'-C', clone,
], stdin=uploader.stdout)
uploader.wait()
source['path'] = clone
else:
d = self.worker.check_output([
'mktemp', '-d',
'-p', self.worker.scratch,
'flatdeb-path.XXXXXX',
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
]).decode('utf-8').rstrip('\n')
clone = '{}/{}'.format(
d, os.path.basename(source['path']),
)
permissions = 0o644
if GLib.file_test(
source['path'],
GLib.FileTest.IS_EXECUTABLE,
):
permissions = 0o755
self.worker.install_file(
source['path'],
clone,
permissions,
)
source['path'] = clone
if 'x-flatdeb-apt-packages' in module:
packages = self.worker.check_output([
'mktemp', '-d',
'-p', self.worker.scratch,
'flatdeb-debs.XXXXXX',
]).decode('utf-8').rstrip('\n')
self.worker.check_call([
'env',
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'flatpak', 'run',
'--filesystem={}'.format(packages),
'--share=network',
'--command=/usr/bin/env',
'{}/{}/{}'.format(
manifest['sdk'],
self.flatpak_arch,
'http_proxy=http://192.168.122.1:3142',
'export={}'.format(packages),
'sh',
'-euc',
'cp -a /usr/var /\n'
'install -d /var/cache/apt/archives/partial\n'
'fakeroot apt-get update\n'
'fakeroot apt-get -y --download-only \\\n'
' --no-install-recommends install "$@"\n'
'mv /var/cache/apt/archives/*.deb "$export"\n'
'mv /var/lib/apt/lists "$export"\n'
'',
'sh', # argv[0]
] + module['x-flatdeb-apt-packages'])
obtained = self.worker.check_output([
'sh', '-euc',
'cd "$1"\n'
'find * -type f -print0 | xargs -0 sha256sum -b\n'
'',
'sh', # argv[0]
packages,
for line in obtained:
sha256, f = line.split(' *', 1)
path = '{}/{}'.format(packages, f)
if f.endswith('.deb'):
sources.append({
'dest': '.',
'type': 'file',
'sha256': sha256,
'url': urllib.parse.urlunsplit((
'file',
'',
urllib.parse.quote(path),
'',
'',
))
remote_manifest = '{}/{}.json'.format(
self.worker.scratch,
manifest['id'],
)
self.worker.check_call([
'mkdir', '-p',
'{}/.flatpak-builder'.format(self.remote_build_area),
])
self.worker.check_call([
'ln', '-nsf',
'{}/.flatpak-builder'.format(self.remote_build_area),
'{}/'.format(self.worker.scratch),
])
with TemporaryDirectory(prefix='flatdeb-manifest.') as t:
json_manifest = os.path.join(t, manifest['id'] + '.json')
with open(
json_manifest, 'w', encoding='utf-8',
) as writer:
json.dump(manifest, writer, indent=2, sort_keys=True)
self.worker.install_file(json_manifest, remote_manifest)
self.worker.check_call([
'env',
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'http_proxy=http://192.168.122.1:3142',
'--arch={}'.format(self.flatpak_arch),
'--repo={}'.format(self.remote_repo),
'{}/workdir'.format(self.worker.scratch),
remote_manifest,
])
if not isinstance(self.worker, HostWorker):
self.worker.check_call([
'time',
'flatpak',
'build-update-repo',
self.remote_repo,
])
with self.worker.remote_dir_context(self.remote_repo) as mount:
'ostree',
'--repo={}'.format(self.repo),
'remote',
'delete',
'flatdeb-worker',
])
'ostree',
'--repo={}'.format(self.repo),
'remote',
'add',
'--no-gpg-verify',
'flatdeb-worker',
'file://' + urllib.parse.quote(mount),
])
'ostree',
'--repo={}'.format(self.repo),
'pull',
'--disable-fsync',
'--mirror',
'--untrusted',
'app/{}/{}/{}'.format(
manifest['id'],
self.flatpak_arch,
manifest['branch'],
),
])
'ostree',
'--repo={}'.format(self.repo),
'remote',
'delete',
'flatdeb-worker',
])
self.host_worker.check_call([
'time',
'flatpak',
'build-update-repo',
self.repo,
])
'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
'flatpak',
'build-bundle',
self.remote_repo,
manifest['id'],
manifest['branch'],
])
bundle = '{}-{}-{}.bundle'.format(
manifest['id'],
self.flatpak_arch,
manifest['branch'],
)
output = os.path.join(self.build_area, bundle)
with open(output + '.new', 'wb') as writer:
self.worker.check_call([
'cat',
'{}/bundle'.format(self.worker.scratch),
], stdout=writer)
os.rename(output + '.new', output)
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
def usrmerge(self, chroot):
self.root_worker.check_call([
'time',
'chroot', chroot,
'sh',
'-euc',
'usrmerge () {\n'
' local f="$1"\n'
'\n'
' ls -dl "$f" "/usr$f" >&2 || true\n'
' if [ "$(readlink "$f")" = "/usr$f" ]; then\n'
' echo "Removing $f in favour of /usr$f" >&2\n'
' rm -v -f "$f"\n'
' elif [ "$(readlink "/usr$f")" = "$f" ]; then\n'
' echo "Removing /usr$f in favour of $f" >&2\n'
' rm -v -f "/usr$f"\n'
' elif [ "$(readlink -f "/usr$f")" = \\\n'
' "$(readlink -f "$f")" ]; then\n'
' echo "/usr$f and $f are functionally identical" >&2\n'
' rm -v -f "$f"\n'
' else\n'
' echo "Cannot merge $f with /usr$f" >&2\n'
' exit 1\n'
' fi\n'
'}\n'
'\n'
'find /bin /sbin /lib* -not -xtype d |\n'
'while read f; do\n'
' if [ -e "/usr$f" ]; then\n'
' usrmerge "$f"\n'
' fi\n'
'done\n'
'',
'sh', # argv[0]
chroot,
])
self.root_worker.check_call([
'time',
'sh', '-euc',
'cd "$1"; tar -cf- bin sbin lib* | tar -C usr -xf-',
'sh', chroot,
])
self.root_worker.check_call([
'time',
'sh', '-euc', 'cd "$1"; rm -fr bin sbin lib*',
'sh', chroot,
])
self.root_worker.check_call([
'time',
'sh', '-euc', 'cd "$1"; ln -vs usr/bin usr/sbin usr/lib* .',
'sh', chroot,
])
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
logging.getLogger().setLevel(logging.DEBUG)
if sys.stderr.isatty():
try:
import colorlog
except ImportError:
pass
else:
formatter = colorlog.ColoredFormatter(
'%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
try:
Builder().run_command_line()
except KeyboardInterrupt:
raise SystemExit(130)
except subprocess.CalledProcessError as e:
logger.error('%s', e)
raise SystemExit(1)