Skip to content
Snippets Groups Projects
run.py 68.2 KiB
Newer Older
            logger.info('Packages we will forcibly remove (first round):')

            for p in sorted(unwanted):
                logger.info('- %s', p)

            if unwanted:
                nspawn.check_call([
                    'dpkg', '--purge', '--force-remove-essential',
                    '--force-depends',
                ] + unwanted)

            # Second round of removals.
            for package in [
Simon McVittie's avatar
Simon McVittie committed
                    'busybox-initramfs',
                    'debconf',
                    'debian-archive-keyring',
                    'e2fsprogs',
                    'init',
                    'init-system-helpers',
                    'insserv',
                    'iproute',
                    'login',
                    'mount',
                    'mountall',
                    'passwd',
                    'systemd',
                    'systemd-sysv',
                    'sysv-rc',
                    'ubuntu-archive-keyring',
                    'ubuntu-keyring',
                    'udev',
                    'upstart',
            ]:
                if package in installed:
                    unwanted.append(package)

            if 'perl' not in installed:
                unwanted.append('perl-base')

            logger.info('Packages we will forcibly remove (second round):')
Simon McVittie's avatar
Simon McVittie committed

            for p in sorted(unwanted):
                logger.info('- %s', p)

Simon McVittie's avatar
Simon McVittie committed
            if unwanted:
                nspawn.check_call([
                    'dpkg', '--purge', '--force-remove-essential',
                    '--force-depends',
                ] + unwanted)

            installed = nspawn.list_packages_ignore_arch()
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
            logger.info('Packages included in platform:')

            for p in sorted(installed):
                if p != 'dpkg':
                    logger.info('- %s', p)

            script = self.runtime_details.get('post_script', '')

            if script:
                logger.info('Running custom script...')
                nspawn.check_call([
                    'sh', '-c', script,
                ])
                logger.info('... done')

            script = platform_details.get('post_script', '')

            if script:
                logger.info('Running custom platform script...')
                nspawn.check_call([
                    'sh', '-c', script,
                ])
                logger.info('... done')

            logger.info('Listing packages in platform...')

Simon McVittie's avatar
Simon McVittie committed
            # We have to do this before removing dpkg :-)
            for package in nspawn.write_manifest():
                logger.info('- %s_%s', package.source, package.source_version)
                self.sources_required.add((package.source, package.source_version))
Simon McVittie's avatar
Simon McVittie committed

            # This has to be last for obvious reasons!
            nspawn.check_call([
                'dpkg', '--purge', '--force-remove-essential',
                '--force-depends',
                'dpkg',
            ])

        return installed

    def ostreeify(self, prefix, chroot, sdk=False, packages=()):
        """
        Move things around to turn a chroot into a runtime.
        """
        if sdk:
            installed = self.sdkize(chroot)
        else:
            installed = self.platformize(chroot)

        with NspawnWorker(
            self.root_worker,
            chroot,
        ) as nspawn:
            nspawn.check_call([
                'find', '/', '-xdev', '(',
                '-lname', '/etc/alternatives/*', '-o',
                '-lname', '/etc/locale.alias',
                ')', '-exec', 'sh', '-euc',

                'set -e\n'
                'while [ $# -gt 0 ]; do\n'
Simon McVittie's avatar
Simon McVittie committed
                '    if target="$(readlink -f "$1")"; then\n'
                '        echo "Making $1 a hard link to $target (was $old)"\n'
                '        rm -f "$1"\n'
                '        cp -al "$target" "$1"\n'
Simon McVittie's avatar
Simon McVittie committed
                '    fi\n'
                '    shift\n'
                'done'
                '',

                'sh', # argv[0] for the one-line shell script
                '{}', '+',
            ])
        self.root_worker.check_call([
            'chmod', '-R', '--changes', 'a-s,o-t,u=rwX,og=rX', chroot,
        ])
        self.root_worker.check_call([
            'chown', '-R', '--changes', 'root:root', chroot,
        ])
        self.root_worker.check_call([
            'rm', '-fr', '--one-file-system',
            '{}/usr/local'.format(chroot),
        ])

        # Merge /usr the hard way, if necessary.
        if not self.suite_details.get('can_merge_usr', False):
        self.root_worker.check_call([
            'rm', '-fr', '--one-file-system',
            '{}/etc/apparmor'.format(chroot),
            '{}/etc/apparmor.d'.format(chroot),
            '{}/etc/console-setup'.format(chroot),
            '{}/etc/cron.daily'.format(chroot),
            '{}/etc/cron.hourly'.format(chroot),
            '{}/etc/cron.monthly'.format(chroot),
            '{}/etc/cron.weekly'.format(chroot),
            '{}/etc/dbus-1/system.d'.format(chroot),
            '{}/etc/depmod.d'.format(chroot),
            '{}/etc/dhcp'.format(chroot),
            '{}/etc/emacs'.format(chroot),
            '{}/etc/fstab'.format(chroot),
            '{}/etc/fstab.d'.format(chroot),
            '{}/etc/group-'.format(chroot),
            '{}/etc/gshadow-'.format(chroot),
            '{}/etc/hostname'.format(chroot),
            '{}/etc/hosts'.format(chroot),
            '{}/etc/hosts.allow'.format(chroot),
            '{}/etc/hosts.deny'.format(chroot),
            '{}/etc/init'.format(chroot),
            '{}/etc/init.d'.format(chroot),
            '{}/etc/initramfs-tools'.format(chroot),
            '{}/etc/insserv'.format(chroot),
            '{}/etc/insserv.conf'.format(chroot),
            '{}/etc/insserv.conf.d'.format(chroot),
            '{}/etc/iproute2'.format(chroot),
            '{}/etc/issue'.format(chroot),
            '{}/etc/issue.net'.format(chroot),
            '{}/etc/kbd'.format(chroot),
            '{}/etc/kernel'.format(chroot),
            '{}/etc/localtime'.format(chroot),
            '{}/etc/logcheck'.format(chroot),
            '{}/etc/login.defs'.format(chroot),
            '{}/etc/logrotate.d'.format(chroot),
            '{}/etc/lsb-base'.format(chroot),
            '{}/etc/lsb-base-logging.sh'.format(chroot),
            '{}/etc/machine-id'.format(chroot),
            '{}/etc/mailname'.format(chroot),
            '{}/etc/modprobe.d'.format(chroot),
            '{}/etc/modules'.format(chroot),
            '{}/etc/network'.format(chroot),
            '{}/etc/networks'.format(chroot),
            '{}/etc/nologin'.format(chroot),
            '{}/etc/opt'.format(chroot),
            '{}/etc/pam.conf'.format(chroot),
            '{}/etc/pam.d'.format(chroot),
            '{}/etc/passwd-'.format(chroot),
            '{}/etc/ppp'.format(chroot),
            '{}/etc/rc.local'.format(chroot),
            '{}/etc/rc0.d'.format(chroot),
            '{}/etc/rc1.d'.format(chroot),
            '{}/etc/rc2.d'.format(chroot),
            '{}/etc/rc3.d'.format(chroot),
            '{}/etc/rc4.d'.format(chroot),
            '{}/etc/rc5.d'.format(chroot),
            '{}/etc/rc6.d'.format(chroot),
            '{}/etc/resolv.conf'.format(chroot),
            '{}/etc/resolvconf'.format(chroot),
            '{}/etc/rmt'.format(chroot),
            '{}/etc/rpc'.format(chroot),
            '{}/etc/rsyslog.conf'.format(chroot),
            '{}/etc/rsyslog.d'.format(chroot),
            '{}/etc/securetty'.format(chroot),
            '{}/etc/security'.format(chroot),
            '{}/etc/shadow-'.format(chroot),
            '{}/etc/shells'.format(chroot),
            '{}/etc/subgid-'.format(chroot),
            '{}/etc/subuid-'.format(chroot),
            '{}/etc/sudoers'.format(chroot),
            '{}/etc/sudoers.d'.format(chroot),
            '{}/etc/sysctl.conf'.format(chroot),
            '{}/etc/sysctl.d'.format(chroot),
            '{}/etc/systemd'.format(chroot),
            '{}/etc/timezone'.format(chroot),
            '{}/etc/udev'.format(chroot),
            '{}/etc/update-motd.d'.format(chroot),
            '{}/var/backups'.format(chroot),
            '{}/var/cache'.format(chroot),
            '{}/var/lib/dpkg/status-old'.format(chroot),
            '{}/var/lib/dpkg/statoverride'.format(chroot),
        ])

        self.root_worker.check_call([
            'mv', '{}/etc'.format(chroot),
            '{}/usr/etc'.format(chroot),
        ])

Simon McVittie's avatar
Simon McVittie committed
        if sdk:
            runtime = prefix + '.Sdk'

            self.root_worker.check_call([
                'install', '-d',
                '{}/var/cache/apt/archives/partial'.format(chroot),
                '{}/var/lib/extrausers'.format(chroot),
            ])
            self.root_worker.check_call([
                'touch', '{}/var/cache/apt/archives/partial/.exists'.format(chroot),
            ])

            # This is only useful if the SDK has libnss-extrausers
            self.root_worker.check_call([
                'cp', '{}/usr/etc/passwd'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
                '{}/var/lib/extrausers/passwd'.format(chroot),
            ])
            self.root_worker.check_call([
                'cp', '{}/usr/etc/group'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
                '{}/var/lib/extrausers/groups'.format(chroot),
            ])
            self.root_worker.check_call([
                'mv', '{}/var'.format(chroot),
                '{}/usr/var'.format(chroot),
            ])
        else:
            runtime = prefix + '.Platform'

            self.root_worker.check_call([
                'rm', '-fr', '--one-file-system',
                '{}/etc/adduser.conf'.format(chroot),
                '{}/etc/apt'.format(chroot),
                '{}/etc/bash_completion.d'.format(chroot),
                '{}/etc/dpkg'.format(chroot),
                '{}/etc/debconf.conf'.format(chroot),
                '{}/etc/default'.format(chroot),
                '{}/etc/deluser.conf'.format(chroot),
                '{}/etc/gdb'.format(chroot),
                '{}/etc/gpasswd'.format(chroot),
                '{}/etc/groff'.format(chroot),
                '{}/etc/group'.format(chroot),
                '{}/etc/mailcap'.format(chroot),
                '{}/etc/mailcap.order'.format(chroot),
                '{}/etc/manpath.config'.format(chroot),
                '{}/etc/mke2fs.conf'.format(chroot),
                '{}/etc/newt'.format(chroot),
                '{}/etc/passwd'.format(chroot),
                '{}/etc/shadow'.format(chroot),
                '{}/etc/skel'.format(chroot),
                '{}/etc/subgid'.format(chroot),
                '{}/etc/subuid'.format(chroot),
                '{}/etc/ucf.conf'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
                '{}/share/bash-completion'.format(chroot),
                '{}/share/bug'.format(chroot),
                '{}/var'.format(chroot),
            ])

        # TODO: Move lib/debug, zoneinfo, locales into extensions
        # TODO: Hook point for GL, instead of just Mesa
        # TODO: GStreamer extension
        # TODO: Icon theme, Gtk theme extension
        # TODO: VAAPI extension
        # TODO: SDK extension
        # TODO: ca-certificates extension to get newer certs?
Simon McVittie's avatar
Simon McVittie committed

        self.root_worker.check_call([
            'install', '-d', '{}/ostree/main'.format(chroot),
        ])
        self.root_worker.check_call([
            'mv', '{}/usr'.format(chroot),
            '{}/ostree/main/files'.format(chroot),
        ])

        ref = 'runtime/{}/{}/{}'.format(
            runtime, self.flatpak_arch, self.runtime_branch,
        with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
Simon McVittie's avatar
Simon McVittie committed
            metadata = os.path.join(t, 'metadata')

            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime)
            keyfile.set_string(
                'Runtime', 'runtime',
Simon McVittie's avatar
Simon McVittie committed
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
Simon McVittie's avatar
Simon McVittie committed
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                )
            )

            keyfile.set_string(
                'Environment', 'XDG_DATA_DIRS',
                ':'.join([
                    '/app/share', '/usr/share', '/usr/share/runtime/share',
                ]),
            )

            search_path = []

            for arch in self.dpkg_archs:
                search_path.append('/app/lib/{}'.format(self.multiarch_tuple(arch)))

            search_path.append('/app/lib')

            keyfile.set_string(
                'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
            )

Simon McVittie's avatar
Simon McVittie committed
            if 'libgstreamer1.0-0' in installed:
                search_path = []

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/app/lib/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/app/lib/gstreamer-1.0')

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/usr/lib/extensions/gstreamer-1.0')

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/usr/lib/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/usr/lib/gstreamer-1.0')

Simon McVittie's avatar
Simon McVittie committed
                keyfile.set_string(
                    'Environment', 'GST_PLUGIN_SYSTEM_PATH',
Simon McVittie's avatar
Simon McVittie committed
                )

            if 'libgirepository-1.0-1' in installed:
                search_path = []

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/app/lib/{}/girepository-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/app/lib/girepository-1.0')

Simon McVittie's avatar
Simon McVittie committed
                keyfile.set_string(
                    'Environment', 'GI_TYPELIB_PATH',
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )

            for ext, detail in self.runtime_details.get(
                    'add-extensions', {}
                    ).items():
                group = 'Extension {}'.format(ext)

                self.root_worker.check_call([
                    'install', '-d',
                    '{}/ostree/main/files/{}'.format(chroot, detail['directory']),
                ])

                for k, v in detail.items():
                    if isinstance(v, str):
                        keyfile.set_string(group, k, v)
                    elif isinstance(v, bool):
                        keyfile.set_boolean(group, k, v)
                    else:
                        raise RuntimeError(
                            'Unknown type {} in {}'.format(v, ext))

Simon McVittie's avatar
Simon McVittie committed
            keyfile.save_to_file(metadata)

            self.root_worker.install_file(
                metadata,
                '{}/ostree/main/metadata'.format(chroot),
            )

        self.worker.check_call([
            'time',
            'ostree',
            '--repo=' + self.remote_repo,
            'commit',
            '--branch=' + ref,
            '--subject=Update',
            '--tree=dir={}/ostree/main'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
            '--fsync=false',
        ])

        # Don't keep the history in this working repository:
        # if history is desired, mirror the commits into a public
        # repository and maintain history there.
        self.worker.check_call([
            'time',
            'ostree',
            '--repo=' + self.remote_repo,
            'prune',
            '--refs-only',
            '--depth=1',
        ])

        if not isinstance(self.worker, HostWorker):
            self.worker.check_call([
                'time',
                'flatpak',
                'build-update-repo',
                self.remote_repo,
            ])

            with self.worker.remote_dir_context(self.remote_repo) as mount:
                self.host_worker.call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'remote',
                    'delete',
                    'flatdeb-worker',
                ])
                print('^ It is OK if that failed with "remote not found"')
                self.host_worker.check_call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'remote',
                    'add',
                    '--no-gpg-verify',
                    'flatdeb-worker',
                    'file://' + urllib.parse.quote(mount),
                ])
                self.host_worker.check_call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'pull',
                    '--disable-fsync',
                    '--mirror',
                    '--untrusted',
                    'flatdeb-worker',
                    'runtime/{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                self.host_worker.check_call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'remote',
                    'delete',
                    'flatdeb-worker',
                ])

        self.host_worker.check_call([
            'time',
            'flatpak',
            'build-update-repo',
            self.repo,
        ])
    def command_app(self, *, app_branch, yaml_manifest, **kwargs):
        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed

        with open(yaml_manifest) as reader:
Simon McVittie's avatar
Simon McVittie committed
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            stack.enter_context(self.worker)
            self.ensure_build_area()
            self.ensure_remote_repo()
            t = stack.enter_context(
                TemporaryDirectory(prefix='flatpak-app.')
            )
Simon McVittie's avatar
Simon McVittie committed

            self.worker.check_call([
                'mkdir', '-p', '{}/home'.format(self.remote_build_area),
Simon McVittie's avatar
Simon McVittie committed
            ])
            if isinstance(self.worker, SshWorker):
                # ostree over sshfs is painfully slow so do something different
                self.host_worker.call([
                    'time',
                    'rsync',
                    '-aW',
                    '--no-i-r',
                    '--info=progress2',
                    '{}/'.format(self.repo),
                    '{}:{}/'.format(self.worker.remote, self.remote_repo),
                ])
            elif not isinstance(self.worker, HostWorker):
                with self.worker.remote_dir_context(self.remote_repo) as mount:
                    self.host_worker.call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'remote',
                        'delete',
                        'flatdeb-host',
                    ])
                    print('^ It is OK if that failed with "remote not found"')
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'remote',
                        'add',
                        '--no-gpg-verify',
                        'flatdeb-host',
                        'file://' + urllib.parse.quote(self.repo),
                    ])
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'pull',
                        '--disable-fsync',
                        '--mirror',
                        'flatdeb-host',
                        'runtime/{}/{}/{}'.format(
                            manifest['sdk'],
                            self.flatpak_arch,
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'pull',
                        '--disable-fsync',
                        '--mirror',
                        'flatdeb-host',
                        'runtime/{}/{}/{}'.format(
                            manifest['runtime'],
                            self.flatpak_arch,
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'remote',
                        'delete',
                        'flatdeb-host',
                    ])
                    self.worker.check_call([
                        'time',
                        'flatpak',
                        'build-update-repo',
                        self.remote_repo,
                    ])
            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
                'flatpak', '--user',
                'remote-add', '--if-not-exists', '--no-gpg-verify',
                'file://{}'.format(urllib.parse.quote(self.remote_repo)),
Simon McVittie's avatar
Simon McVittie committed
            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
Simon McVittie's avatar
Simon McVittie committed
                'flatpak', '--user',
                'remote-modify', '--no-gpg-verify',
                '--url=file://{}'.format(urllib.parse.quote(self.remote_repo)),
                'flatdeb',
Simon McVittie's avatar
Simon McVittie committed
            ])

            for runtime in (manifest['sdk'], manifest['runtime']):
                if self.worker.call([
                            'env',
                            'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
                            'flatpak', '--user',
                            'install', 'flatdeb',
                            '{}/{}/{}'.format(
                                runtime,
                                self.flatpak_arch,
                                self.runtime_branch,
                            ),
                        ]) != 0:
                    self.worker.check_call([
                        'env',
                        'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
                        'flatpak', '--user',
                        'update',
                        '{}/{}/{}'.format(
                            runtime,
                            self.flatpak_arch,
                            self.runtime_branch,
                        ),
                    ])
Simon McVittie's avatar
Simon McVittie committed

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = self.worker.check_output([
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-git.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                                ]).decode('utf-8').rstrip('\n')
                                uploader = self.host_worker.Popen([
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
                                self.worker.check_call([
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = self.worker.check_output([
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-path.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                                ]).decode('utf-8').rstrip('\n')
                                clone = '{}/{}'.format(
                                    d, os.path.basename(source['path']),
                                )

                                permissions = 0o644

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    permissions = 0o755

                                self.worker.install_file(
                                    source['path'],
                                    clone,
                                    permissions,
                                )
                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = self.worker.check_output([
                            'mktemp', '-d',
                            '-p', self.worker.scratch,
                            'flatdeb-debs.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').rstrip('\n')

                        self.worker.check_call([
                            'env',
                            'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
Simon McVittie's avatar
Simon McVittie committed
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'DEBIAN_FRONTEND=noninteractive',
Simon McVittie's avatar
Simon McVittie committed
                            'http_proxy=http://192.168.122.1:3142',
                            'export={}'.format(packages),
                            'sh',
                            '-euc',

                            'cp -a /usr/var /\n'
                            'install -d /var/cache/apt/archives/partial\n'
                            'fakeroot apt-get update\n'
                            'fakeroot apt-get -y --download-only \\\n'
                            '    --no-install-recommends install "$@"\n'
Simon McVittie's avatar
Simon McVittie committed
                            'mv /var/cache/apt/archives/*.deb "$export"\n'
                            'mv /var/lib/apt/lists "$export"\n'
                            '',

                            'sh',   # argv[0]
                        ] + module['x-flatdeb-apt-packages'])

                        obtained = self.worker.check_output([
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').splitlines()

                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
                            path = '{}/{}'.format(packages, f)

                            if f.endswith('.deb'):
                                sources.append({
                                    'dest': '.',
                                    'type': 'file',
                                    'sha256': sha256,
                                    'url': urllib.parse.urlunsplit((
                                        'file',
                                        '',
                                        urllib.parse.quote(path),
                                        '',
                                        '',
                                    ))
            remote_manifest = '{}/{}.json'.format(
                self.worker.scratch,
                manifest['id'],
            )
Simon McVittie's avatar
Simon McVittie committed

            self.worker.check_call([
                'mkdir', '-p',
                '{}/.flatpak-builder'.format(self.remote_build_area),
            ])
            self.worker.check_call([
                'ln', '-nsf',
                '{}/.flatpak-builder'.format(self.remote_build_area),
                '{}/'.format(self.worker.scratch),
            ])

            with TemporaryDirectory(prefix='flatdeb-manifest.') as t:
                json_manifest = os.path.join(t, manifest['id'] + '.json')
Simon McVittie's avatar
Simon McVittie committed

                with open(
                        json_manifest, 'w', encoding='utf-8',
                ) as writer:
                    json.dump(manifest, writer, indent=2, sort_keys=True)

                self.worker.install_file(json_manifest, remote_manifest)

            self.worker.check_call([
                'env',
                'DEBIAN_FRONTEND=noninteractive',
                'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
                'http_proxy=http://192.168.122.1:3142',
Simon McVittie's avatar
Simon McVittie committed
                'flatpak-builder',
                '--arch={}'.format(self.flatpak_arch),
Simon McVittie's avatar
Simon McVittie committed
                '--repo={}'.format(self.remote_repo),
                '{}/workdir'.format(self.worker.scratch),
                remote_manifest,
            ])

            if not isinstance(self.worker, HostWorker):
                self.worker.check_call([
                    'time',
                    'flatpak',
                    'build-update-repo',
                    self.remote_repo,
                ])

                with self.worker.remote_dir_context(self.remote_repo) as mount:
                    self.host_worker.call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'remote',
                        'delete',
                        'flatdeb-worker',
                    ])
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'remote',
                        'add',
                        '--no-gpg-verify',
                        'flatdeb-worker',
                        'file://' + urllib.parse.quote(mount),
                    ])
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'pull',
                        '--disable-fsync',
                        '--mirror',
                        '--untrusted',
                        'flatdeb-worker',
                        'app/{}/{}/{}'.format(
                            manifest['id'],
                            self.flatpak_arch,
                            manifest['branch'],
                        ),
                    ])
                    self.host_worker.check_call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'remote',
                        'delete',
                        'flatdeb-worker',
                    ])

                self.host_worker.check_call([
                    'time',
                    'flatpak',
                    'build-update-repo',
                    self.repo,
                ])

            if self.export_bundles:
Simon McVittie's avatar
Simon McVittie committed
                self.worker.check_call([
                    'XDG_DATA_HOME={}/home'.format(self.remote_build_area),
                    'flatpak',
                    'build-bundle',
                    self.remote_repo,
Simon McVittie's avatar
Simon McVittie committed
                    '{}/bundle'.format(self.worker.scratch),
                    manifest['id'],
                    manifest['branch'],
                ])
Simon McVittie's avatar
Simon McVittie committed

                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)

                with open(output + '.new', 'wb') as writer:
                    self.worker.check_call([
                        'cat',
                        '{}/bundle'.format(self.worker.scratch),
                    ], stdout=writer)

                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

    def usrmerge(self, chroot):
        self.root_worker.check_call([
            'time',
            'chroot', chroot,
            'sh',
            '-euc',

            'usrmerge () {\n'
            '    local f="$1"\n'
            '\n'
            '    ls -dl "$f" "/usr$f" >&2 || true\n'
            '    if [ "$(readlink "$f")" = "/usr$f" ]; then\n'
            '        echo "Removing $f in favour of /usr$f" >&2\n'
            '        rm -v -f "$f"\n'
            '    elif [ "$(readlink "/usr$f")" = "$f" ]; then\n'
            '        echo "Removing /usr$f in favour of $f" >&2\n'
            '        rm -v -f "/usr$f"\n'
            '    elif [ "$(readlink -f "/usr$f")" = \\\n'
            '           "$(readlink -f "$f")" ]; then\n'
            '        echo "/usr$f and $f are functionally identical" >&2\n'
            '        rm -v -f "$f"\n'
            '    else\n'
            '        echo "Cannot merge $f with /usr$f" >&2\n'
            '        exit 1\n'
            '    fi\n'
            '}\n'
            '\n'
            'find /bin /sbin /lib* -not -xtype d |\n'
            'while read f; do\n'
            '    if [ -e "/usr$f" ]; then\n'
            '        usrmerge "$f"\n'
            '    fi\n'
            'done\n'
            '',

            'sh',   # argv[0]
            chroot,
        ])
        self.root_worker.check_call([
            'time',
            'sh', '-euc',
            'cd "$1"; tar -cf- bin sbin lib* | tar -C usr -xf-',
            'sh', chroot,
        ])
        self.root_worker.check_call([
            'time',
            'sh', '-euc', 'cd "$1"; rm -fr bin sbin lib*',
            'sh', chroot,
        ])
        self.root_worker.check_call([
            'time',
            'sh', '-euc', 'cd "$1"; ln -vs usr/bin usr/sbin usr/lib* .',
            'sh', chroot,
        ])

Simon McVittie's avatar
Simon McVittie committed
if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
            pass
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    else:
        logging.basicConfig()

    logging.getLogger().setLevel(logging.DEBUG)
Simon McVittie's avatar
Simon McVittie committed

    try:
        Builder().run_command_line()
    except KeyboardInterrupt:
        raise SystemExit(130)
    except subprocess.CalledProcessError as e:
        logger.error('%s', e)
        raise SystemExit(1)