Skip to content
Snippets Groups Projects
run.py 63.7 KiB
Newer Older
            script = platform_details.get('post_script', '')

            if script:
                logger.info('Running custom platform script...')
                nspawn.check_call([
                    'sh', '-c', script,
                ])
                logger.info('... done')

            logger.info('Listing packages in platform...')

Simon McVittie's avatar
Simon McVittie committed
            # We have to do this before removing dpkg :-)
            self.platform_packages = nspawn.write_manifest()

            for package in self.platform_packages:
                logger.info(
                    '- %s from %s_%s',
                    package.binary, package.source, package.source_version)
                self.sources_required.add((package.source, package.source_version))
Simon McVittie's avatar
Simon McVittie committed

            logger.info('Listing Built-Using fields for platform...')

            for package, source, version in nspawn.list_built_using():
                logger.info(
                    '- %s was Built-Using %s_%s',
                    package, source, version)
                self.sources_required.add((source, version))

Simon McVittie's avatar
Simon McVittie committed
            # This has to be last for obvious reasons!
            nspawn.check_call([
                'dpkg', '--purge', '--force-remove-essential',
                '--force-depends',
                'dpkg',
            ])

        return installed

    def ostreeify(self, prefix, chroot, sdk=False, packages=()):
        """
        Move things around to turn a chroot into a runtime.
        """
        if sdk:
            installed = self.sdkize(chroot)
        else:
            installed = self.platformize(chroot)

        with NspawnWorker(
            self.root_worker,
            chroot,
        ) as nspawn:
            nspawn.check_call([
                'find', '/', '-xdev', '(',
                '-lname', '/etc/alternatives/*', '-o',
                '-lname', '/etc/locale.alias',
                ')', '-exec', 'sh', '-euc',

                'set -e\n'
                'while [ $# -gt 0 ]; do\n'
Simon McVittie's avatar
Simon McVittie committed
                '    if target="$(readlink -f "$1")"; then\n'
                '        echo "Making $1 a hard link to $target (was $old)"\n'
                '        rm -f "$1"\n'
                '        cp -al "$target" "$1"\n'
Simon McVittie's avatar
Simon McVittie committed
                '    fi\n'
                '    shift\n'
                'done'
                '',

                'sh', # argv[0] for the one-line shell script
                '{}', '+',
            ])
            # Flatpak wants to be able to run ldconfig without specifying
            # an absolute path
            nspawn.check_call([
                'sh', '-euc',
                'test -e /bin/ldconfig || ln -s /sbin/ldconfig /bin/ldconfig',
            ])

Simon McVittie's avatar
Simon McVittie committed
        self.root_worker.check_call([
            'chmod', '-R', '--changes', 'a-s,o-t,u=rwX,og=rX', chroot,
        ])
        self.root_worker.check_call([
            'chown', '-R', '--changes', 'root:root', chroot,
        ])
        self.root_worker.check_call([
            'rm', '-fr', '--one-file-system',
            '{}/usr/local'.format(chroot),
        ])

        self.root_worker.install_file(
            os.path.join(
                os.path.dirname(__file__), 'flatdeb', 'usrmerge'),
            '{}/usrmerge'.format(self.root_worker.scratch),
            permissions=0o755,
        )
        self.root_worker.check_call([
            '{}/usrmerge'.format(self.root_worker.scratch),
            chroot,
        ])
        self.root_worker.check_call([
            'rm', '-fr', '--one-file-system',
            '{}/etc/apparmor'.format(chroot),
            '{}/etc/apparmor.d'.format(chroot),
            '{}/etc/console-setup'.format(chroot),
            '{}/etc/cron.daily'.format(chroot),
            '{}/etc/cron.hourly'.format(chroot),
            '{}/etc/cron.monthly'.format(chroot),
            '{}/etc/cron.weekly'.format(chroot),
            '{}/etc/dbus-1/system.d'.format(chroot),
            '{}/etc/depmod.d'.format(chroot),
            '{}/etc/dhcp'.format(chroot),
            '{}/etc/emacs'.format(chroot),
            '{}/etc/fstab'.format(chroot),
            '{}/etc/fstab.d'.format(chroot),
            '{}/etc/group-'.format(chroot),
            '{}/etc/gshadow-'.format(chroot),
            '{}/etc/hostname'.format(chroot),
            '{}/etc/hosts'.format(chroot),
            '{}/etc/hosts.allow'.format(chroot),
            '{}/etc/hosts.deny'.format(chroot),
            '{}/etc/init'.format(chroot),
            '{}/etc/init.d'.format(chroot),
            '{}/etc/initramfs-tools'.format(chroot),
            '{}/etc/insserv'.format(chroot),
            '{}/etc/insserv.conf'.format(chroot),
            '{}/etc/insserv.conf.d'.format(chroot),
            '{}/etc/iproute2'.format(chroot),
            '{}/etc/issue'.format(chroot),
            '{}/etc/issue.net'.format(chroot),
            '{}/etc/kbd'.format(chroot),
            '{}/etc/kernel'.format(chroot),
            '{}/etc/localtime'.format(chroot),
            '{}/etc/logcheck'.format(chroot),
            '{}/etc/login.defs'.format(chroot),
            '{}/etc/logrotate.d'.format(chroot),
            '{}/etc/lsb-base'.format(chroot),
            '{}/etc/lsb-base-logging.sh'.format(chroot),
            '{}/etc/machine-id'.format(chroot),
            '{}/etc/mailname'.format(chroot),
            '{}/etc/modprobe.d'.format(chroot),
            '{}/etc/modules'.format(chroot),
            '{}/etc/network'.format(chroot),
            '{}/etc/networks'.format(chroot),
            '{}/etc/nologin'.format(chroot),
            '{}/etc/opt'.format(chroot),
            '{}/etc/pam.conf'.format(chroot),
            '{}/etc/pam.d'.format(chroot),
            '{}/etc/passwd-'.format(chroot),
            '{}/etc/ppp'.format(chroot),
            '{}/etc/rc.local'.format(chroot),
            '{}/etc/rc0.d'.format(chroot),
            '{}/etc/rc1.d'.format(chroot),
            '{}/etc/rc2.d'.format(chroot),
            '{}/etc/rc3.d'.format(chroot),
            '{}/etc/rc4.d'.format(chroot),
            '{}/etc/rc5.d'.format(chroot),
            '{}/etc/rc6.d'.format(chroot),
            '{}/etc/resolv.conf'.format(chroot),
            '{}/etc/resolvconf'.format(chroot),
            '{}/etc/rmt'.format(chroot),
            '{}/etc/rpc'.format(chroot),
            '{}/etc/rsyslog.conf'.format(chroot),
            '{}/etc/rsyslog.d'.format(chroot),
            '{}/etc/securetty'.format(chroot),
            '{}/etc/security'.format(chroot),
            '{}/etc/shadow-'.format(chroot),
            '{}/etc/shells'.format(chroot),
            '{}/etc/subgid-'.format(chroot),
            '{}/etc/subuid-'.format(chroot),
            '{}/etc/sudoers'.format(chroot),
            '{}/etc/sudoers.d'.format(chroot),
            '{}/etc/sysctl.conf'.format(chroot),
            '{}/etc/sysctl.d'.format(chroot),
            '{}/etc/systemd'.format(chroot),
            '{}/etc/timezone'.format(chroot),
            '{}/etc/udev'.format(chroot),
            '{}/etc/update-motd.d'.format(chroot),
            '{}/var/backups'.format(chroot),
            '{}/var/cache'.format(chroot),
            '{}/var/lib/dpkg/status-old'.format(chroot),
            '{}/var/lib/dpkg/statoverride'.format(chroot),
            '{}/var/local'.format(chroot),
            '{}/var/lock'.format(chroot),
            '{}/var/log'.format(chroot),
            '{}/var/mail'.format(chroot),
            '{}/var/opt'.format(chroot),
            '{}/var/run'.format(chroot),
            '{}/var/spool'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
        if sdk:
            runtime = prefix + '.Sdk'

            self.root_worker.check_call([
                'install', '-d',
                '{}/var/cache/apt/archives/partial'.format(chroot),
                '{}/var/lib/extrausers'.format(chroot),
            ])
            self.root_worker.check_call([
                'touch', '{}/var/cache/apt/archives/partial/.exists'.format(chroot),
            ])

            # This is only useful if the SDK has libnss-extrausers
            self.root_worker.check_call([
                'cp', '{}/etc/passwd'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
                '{}/var/lib/extrausers/passwd'.format(chroot),
            ])
            self.root_worker.check_call([
                'cp', '{}/etc/group'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
                '{}/var/lib/extrausers/groups'.format(chroot),
            ])
            self.root_worker.check_call([
                'mv', '{}/var'.format(chroot),
                '{}/usr/var'.format(chroot),
            ])
        else:
            runtime = prefix + '.Platform'

            self.root_worker.check_call([
                'rm', '-fr', '--one-file-system',
                '{}/etc/adduser.conf'.format(chroot),
                '{}/etc/apt'.format(chroot),
                '{}/etc/bash_completion.d'.format(chroot),
                '{}/etc/dpkg'.format(chroot),
                '{}/etc/debconf.conf'.format(chroot),
                '{}/etc/default'.format(chroot),
                '{}/etc/deluser.conf'.format(chroot),
                '{}/etc/gdb'.format(chroot),
                '{}/etc/gpasswd'.format(chroot),
                '{}/etc/groff'.format(chroot),
                '{}/etc/group'.format(chroot),
                '{}/etc/mailcap'.format(chroot),
                '{}/etc/mailcap.order'.format(chroot),
                '{}/etc/manpath.config'.format(chroot),
                '{}/etc/mke2fs.conf'.format(chroot),
                '{}/etc/newt'.format(chroot),
                '{}/etc/passwd'.format(chroot),
                '{}/etc/shadow'.format(chroot),
                '{}/etc/skel'.format(chroot),
                '{}/etc/subgid'.format(chroot),
                '{}/etc/subuid'.format(chroot),
                '{}/etc/ucf.conf'.format(chroot),
Simon McVittie's avatar
Simon McVittie committed
                '{}/share/bash-completion'.format(chroot),
                '{}/share/bug'.format(chroot),
                '{}/var'.format(chroot),
            ])

        self.root_worker.check_call([
            'mv', '{}/etc'.format(chroot),
            '{}/usr/etc'.format(chroot),
        ])

Simon McVittie's avatar
Simon McVittie committed
        # TODO: Move lib/debug, zoneinfo, locales into extensions
        # TODO: Hook point for GL, instead of just Mesa
        # TODO: GStreamer extension
        # TODO: Icon theme, Gtk theme extension
        # TODO: VAAPI extension
        # TODO: SDK extension
        # TODO: ca-certificates extension to get newer certs?
Simon McVittie's avatar
Simon McVittie committed

        self.root_worker.check_call([
            'install', '-d', '{}/ostree/main'.format(chroot),
        ])
        self.root_worker.check_call([
            'mv', '{}/usr'.format(chroot),
            '{}/ostree/main/files'.format(chroot),
        ])

        ref = 'runtime/{}/{}/{}'.format(
            runtime, self.flatpak_arch, self.runtime_branch,
        with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
Simon McVittie's avatar
Simon McVittie committed
            metadata = os.path.join(t, 'metadata')

            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime)
            keyfile.set_string(
                'Runtime', 'runtime',
Simon McVittie's avatar
Simon McVittie committed
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
Simon McVittie's avatar
Simon McVittie committed
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
            keyfile.set_string(
                'Runtime', 'x-flatdeb-sources',
                '{}.Sdk.Sources/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                ),
            )

Simon McVittie's avatar
Simon McVittie committed
            keyfile.set_string(
                'Environment', 'XDG_DATA_DIRS',
                ':'.join([
                    '/app/share', '/usr/share', '/usr/share/runtime/share',
                ]),
            )

            search_path = []

            for arch in self.dpkg_archs:
                search_path.append('/app/lib/{}'.format(self.multiarch_tuple(arch)))

            search_path.append('/app/lib')

            keyfile.set_string(
                'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
            )

Simon McVittie's avatar
Simon McVittie committed
            if 'libgstreamer1.0-0' in installed:
                search_path = []

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/app/lib/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/app/lib/gstreamer-1.0')

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/usr/lib/extensions/gstreamer-1.0')

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/usr/lib/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/usr/lib/gstreamer-1.0')

Simon McVittie's avatar
Simon McVittie committed
                keyfile.set_string(
                    'Environment', 'GST_PLUGIN_SYSTEM_PATH',
Simon McVittie's avatar
Simon McVittie committed
                )

            if 'libgirepository-1.0-1' in installed:
                search_path = []

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/app/lib/{}/girepository-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/app/lib/girepository-1.0')

Simon McVittie's avatar
Simon McVittie committed
                keyfile.set_string(
                    'Environment', 'GI_TYPELIB_PATH',
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )

            for ext, detail in self.runtime_details.get(
                    'add-extensions', {}
                    ).items():
                group = 'Extension {}'.format(ext)

                self.root_worker.check_call([
                    'install', '-d',
                    '{}/ostree/main/files/{}'.format(chroot, detail['directory']),
                ])

                for k, v in detail.items():
                    if isinstance(v, str):
                        keyfile.set_string(group, k, v)
                    elif isinstance(v, bool):
                        keyfile.set_boolean(group, k, v)
                    else:
                        raise RuntimeError(
                            'Unknown type {} in {}'.format(v, ext))

Simon McVittie's avatar
Simon McVittie committed
            keyfile.save_to_file(metadata)

            self.root_worker.install_file(
                metadata,
                '{}/ostree/main/metadata'.format(chroot),
            )

        tarball = '{}-ostree-{}-{}.tar.gz'.format(
            runtime,
            self.flatpak_arch,
            self.runtime_branch,
        )

        self.worker.check_call([
            'tar', '-zcf',
            '{}/{}{}'.format(
                self.build_area,
            ),
            '-C', '{}/ostree/main'.format(chroot),
            '.',
        ])
        output = os.path.join(self.build_area, tarball)
        os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed
        self.worker.check_call([
            'time',
            'ostree',
            '--repo=' + self.repo,
Simon McVittie's avatar
Simon McVittie committed
            'commit',
            '--branch=' + ref,
            '--subject=Update',
            '--tree=tar={}/{}'.format(self.build_area, tarball),
Simon McVittie's avatar
Simon McVittie committed
            '--fsync=false',
        ])

            source_ref = 'runtime/{}/{}/{}'.format(
                runtime + '.Sources',
                self.flatpak_arch,
                self.runtime_branch,
            )

            with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
                metadata = os.path.join(t, 'metadata')

                keyfile = GLib.KeyFile()
                keyfile.set_string('Runtime', 'name', runtime + '.Sources')
                keyfile.set_string(
                    'Runtime', 'runtime',
                    '{}.Platform/{}/{}'.format(
                        prefix,
                        self.flatpak_arch,
                        self.runtime_branch,
                    )
                )
                keyfile.set_string(
                    'Runtime', 'sdk',
                    '{}.Sdk/{}/{}'.format(
                        prefix,
                        self.flatpak_arch,
                        self.runtime_branch,
                    )
                )

                keyfile.set_string(
                    'Runtime', 'x-flatdeb-version', VERSION,
                )

                keyfile.save_to_file(metadata)

                self.root_worker.install_file(
                    metadata,
                    '{}/ostree/source/metadata'.format(chroot),
                )

            self.worker.check_call([
                'time',
                'ostree',
                '--repo=' + self.repo,
                'commit',
                '--branch=' + source_ref,
                '--subject=Update',
                '--tree=dir={}/ostree/source'.format(chroot),
                '--fsync=false',
            ])
Simon McVittie's avatar
Simon McVittie committed
        # Don't keep the history in this working repository:
        # if history is desired, mirror the commits into a public
        # repository and maintain history there.
        self.worker.check_call([
            'time',
            'ostree',
            '--repo=' + self.repo,
Simon McVittie's avatar
Simon McVittie committed
            'prune',
            '--refs-only',
            '--depth=1',
        ])

        self.host_worker.check_call([
            'time',
            'flatpak',
            'build-update-repo',
            self.repo,
        ])
    def command_app(self, *, app_branch, yaml_manifest, **kwargs):
        self.worker.require_extended_attributes()
        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed

        with open(yaml_manifest, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            stack.enter_context(self.worker)
            self.ensure_build_area()
            self.ensure_local_repo()
            t = stack.enter_context(
                TemporaryDirectory(prefix='flatpak-app.')
            )
Simon McVittie's avatar
Simon McVittie committed

            self.worker.check_call([
                'mkdir', '-p', '{}/home'.format(self.worker.scratch),
Simon McVittie's avatar
Simon McVittie committed
            ])
            self.worker.check_call([
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                'flatpak', '--user',
                'remote-add', '--if-not-exists', '--no-gpg-verify',
                'http://192.168.122.1:3142/local/flatdeb/repo',
Simon McVittie's avatar
Simon McVittie committed
            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
Simon McVittie's avatar
Simon McVittie committed
                'flatpak', '--user',
                'remote-modify', '--no-gpg-verify',
                '--url=http://192.168.122.1:3142/local/flatdeb/repo',
Simon McVittie's avatar
Simon McVittie committed
            ])

            for runtime in (manifest['sdk'], manifest['runtime']):
                # This may fail: we might already have it.
                self.worker.call([
                    'env',
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                    'flatpak', '--user',
                    'install', 'flatdeb',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                self.worker.check_call([
                    'env',
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                    'flatpak', '--user',
                    'update',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
Simon McVittie's avatar
Simon McVittie committed

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = self.worker.check_output([
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-git.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                                ]).decode('utf-8').rstrip('\n')
                                uploader = self.host_worker.Popen([
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
                                self.worker.check_call([
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = self.worker.check_output([
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-path.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                                ]).decode('utf-8').rstrip('\n')
                                clone = '{}/{}'.format(
                                    d, os.path.basename(source['path']),
                                )

                                permissions = 0o644

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    permissions = 0o755

                                self.worker.install_file(
                                    source['path'],
                                    clone,
                                    permissions,
                                )
                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = self.worker.check_output([
                            'mktemp', '-d',
                            '-p', self.worker.scratch,
                            'flatdeb-debs.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').rstrip('\n')

                        self.worker.check_call([
                            'env',
                            'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
Simon McVittie's avatar
Simon McVittie committed
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'DEBIAN_FRONTEND=noninteractive',
Simon McVittie's avatar
Simon McVittie committed
                            'http_proxy=http://192.168.122.1:3142',
                            'export={}'.format(packages),
                            'sh',
                            '-euc',

                            'cp -a /usr/var /\n'
                            'install -d /var/cache/apt/archives/partial\n'
                            'fakeroot apt-get update\n'
                            'fakeroot apt-get -y --download-only \\\n'
                            '    --no-install-recommends install "$@"\n'
                            'for x in /var/cache/apt/archives/*.deb; do\n'
                            '    package="$(dpkg-deb -f "$x" Package)"\n'
                            '    source="$(dpkg-deb -f "$x" Source)"\n'
                            '    bu="$(dpkg-deb -f "$x" Built-Using)"\n'
                            '    version="$(dpkg-deb -f "$x" Version)"\n'
                            '    if [ -z "$source" ]; then\n'
                            '        source="$package"\n'
                            '    fi\n'
                            '    if [ "${source% (*}" != "$source" ]; then\n'
                            '        version="${source#* (}"\n'
                            '        version="${version%)}"\n'
                            '        source="${source% (*}"\n'
                            '    fi\n'
                            '    ( cd "$export" && \\\n'
                            '         apt-get -y --download-only \\\n'
                            '         -oAPT::Get::Only-Source=true source \\\n'
                            '         "$source=$version"\n'
                            '    )\n'
                            '    if [ -n "$bu" ]; then\n'
                            '        oldIFS="$IFS"\n'
                            '        IFS=","\n'
                            '        for dep in $bu; do\n'
                            '            bu="$(echo "$bu" | tr -d " ")"\n'
                            '            version="${bu#*(=}"\n'
                            '            version="${version%)}"\n'
                            '            source="${bu%(*}"\n'
                            '            ( cd "$export" && \\\n'
                            '                 apt-get -y --download-only \\\n'
                            '                 -oAPT::Get::Only-Source=true \\\n'
                            '                 source "$source=$version"\n'
                            '            )\n'
                            '        done\n'
                            '        IFS="$oldIFS"\n'
                            '    fi\n'
Simon McVittie's avatar
Simon McVittie committed
                            'mv /var/cache/apt/archives/*.deb "$export"\n'
                            'mv /var/lib/apt/lists "$export"\n'
                            '',

                            'sh',   # argv[0]
                        ] + module['x-flatdeb-apt-packages'])

                        obtained = self.worker.check_output([
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').splitlines()

                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
                            path = '{}/{}'.format(packages, f)

                            sources.append({
                                'dest': (os.path.dirname(f) or '.'),
                                'type': 'file',
                                'sha256': sha256,
                                'url': urllib.parse.urlunsplit((
                                    'file',
                                    '',
                                    urllib.parse.quote(path),
                                    '',
                                    '',
                                ))
                            })
Simon McVittie's avatar
Simon McVittie committed

            remote_manifest = '{}/{}.json'.format(
                self.worker.scratch,
                manifest['id'],
            )
Simon McVittie's avatar
Simon McVittie committed

            self.worker.check_call([
                'mkdir', '-p',
                '{}/.flatpak-builder'.format(self.build_area),
            if self.build_area != self.worker.scratch:
                self.worker.check_call([
                    'ln', '-nsf',
                    '{}/.flatpak-builder'.format(self.build_area),
                    '{}/'.format(self.worker.scratch),
                ])
            with TemporaryDirectory(prefix='flatdeb-manifest.') as t:
                json_manifest = os.path.join(t, manifest['id'] + '.json')
Simon McVittie's avatar
Simon McVittie committed

                with open(
                        json_manifest, 'w', encoding='utf-8',
                ) as writer:
                    json.dump(manifest, writer, indent=2, sort_keys=True)

                self.worker.install_file(json_manifest, remote_manifest)

            self.worker.check_call([
                'env',
                'DEBIAN_FRONTEND=noninteractive',
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                'http_proxy=http://192.168.122.1:3142',
                'sh', '-euc',
                'cd "$1"; shift; exec "$@"',
                'sh',                   # argv[0]
                self.worker.scratch,    # directory to cd into
Simon McVittie's avatar
Simon McVittie committed
                'flatpak-builder',
                '--arch={}'.format(self.flatpak_arch),
                '--repo={}'.format(self.repo),
Simon McVittie's avatar
Simon McVittie committed
                '{}/workdir'.format(self.worker.scratch),
                remote_manifest,
            ])

            if self.export_bundles:
Simon McVittie's avatar
Simon McVittie committed
                self.worker.check_call([
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                    'flatpak',
                    'build-bundle',
                    self.repo,
Simon McVittie's avatar
Simon McVittie committed
                    '{}/bundle'.format(self.worker.scratch),
                    manifest['id'],
                    manifest['branch'],
                ])
Simon McVittie's avatar
Simon McVittie committed

                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)

                with open(output + '.new', 'wb') as writer:
                    self.worker.check_call([
                        'cat',
                        '{}/bundle'.format(self.worker.scratch),
                    ], stdout=writer)

                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
            pass
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    else:
        logging.basicConfig()

    logging.getLogger().setLevel(logging.DEBUG)
Simon McVittie's avatar
Simon McVittie committed

    try:
        Builder().run_command_line()
    except KeyboardInterrupt:
        raise SystemExit(130)
    except subprocess.CalledProcessError as e:
        logger.error('%s', e)
        raise SystemExit(1)