Skip to content
Snippets Groups Projects
run.py 70 KiB
Newer Older
                'make-flatpak-friendly',
                'platformize',
                'prepare-runtime',
                'purge-conffiles',
                'put-ldconfig-in-path',
                'set-build-id',
                'usrmerge',
                'write-manifest',
            ):
                dest = os.path.join(scratch, helper)
                shutil.copyfile(
                    os.path.join(
                        os.path.dirname(__file__),
                        'flatdeb',
                        helper,
                    ),
                    dest,
                )
Simon McVittie's avatar
Simon McVittie committed

            prefix = self.runtime_details['id_prefix']

            # Do the Platform first, because we download its source
            # packages as part of preparing the Sdk
            for sdk in (False, True):
                if sdk and not self.do_sdk:
                    continue

                if not sdk and not self.do_platform:
                    continue

                packages = list(self.get_runtime_packages(
                    self.runtime_details.get('add_packages', [])
                ))
                packages.extend(self.get_runtime_packages(
                    self.runtime_details.get('add_packages_multiarch', []),
                    multiarch=True,
                ))

                if sdk:
                    runtime = prefix + '.Sdk'
                else:
                    runtime = prefix + '.Platform'

                artifact_prefix = '{}-{}-{}'.format(
                    runtime,
                    ','.join(self.dpkg_archs),
                    self.runtime_branch,
                )
                ostree_prefix = artifact_prefix + '-runtime'
                out_tarball = ostree_prefix + '.tar.gz'
                sources_prefix = artifact_prefix + '-sources'

                argv = [
                    'debos',
                    '--artifactdir={}'.format(self.build_area),
                    '--scratchsize=8G',
                    '-t', 'architecture:{}'.format(self.primary_dpkg_arch),
                    '-t', 'foreignarchs:{}'.format(
                        ' '.join(self.dpkg_archs[1:]),
                    ),
                    '-t', 'flatpak_arch:{}'.format(self.flatpak_arch),
                    '-t', 'suite:{}'.format(self.apt_suite),
                    '-t', 'ospack:{}'.format(tarball),
                    '-t', 'artifact_prefix:{}'.format(artifact_prefix),
                    '-t', 'ostree_prefix:{}'.format(ostree_prefix),
                    '-t', 'ostree_tarball:{}'.format(out_tarball + '.new'),
                    '-t', 'runtime:{}'.format(runtime),
                    '-t', 'runtime_branch:{}'.format(self.runtime_branch),
                    '-t', 'strip_source_version_suffix:{}'.format(
                        self.strip_source_version_suffix),
                ]

                if self.apt_debug:
                    argv.append('-t')
                    argv.append('apt_debug:true')

                if self.build_id is not None:
                    argv.append('-t')
                    argv.append('build_id:{}'.format(self.build_id))

                if sdk:
                    variant_name = self.sdk_variant_name
                    variant_id = self.sdk_variant_id

                    if variant_name is None and self.variant_name is not None:
                        variant_name = self.variant_name + ' (SDK)'
                else:
                    variant_name = self.variant_name
                    variant_id = self.variant_id

                if variant_name is None:
                    variant_name = artifact_prefix

                if variant_id is None:
                    variant_id = artifact_prefix

                argv.append('-t')
                argv.append('variant:{}'.format(variant_name))
                argv.append('-t')
                argv.append('variant_id:{}'.format(
                    self.escape_variant_id(variant_id)
                ))

                if packages:
                    logger.info('Installing packages:')
                    packages.sort()

                    for p in packages:
                        logger.info('- %s', p)

                    argv.append('-t')
                    argv.append('packages:{}'.format(
                        self.yaml_dump_one_line(packages)))

                    dest = os.path.join(scratch, 'runtimes', runtime)
                    os.makedirs(dest, 0o755, exist_ok=True)
                    dest = os.path.join(dest, 'packages.yaml')

                    with open(dest, 'w', encoding='utf-8') as writer:
                        yaml.safe_dump(packages, stream=writer)

                script = self.runtime_details.get('post_script', '')

                if script:
                    dest = os.path.join(scratch, 'post_script')

                    with open(dest, 'w', encoding='utf-8') as writer:
                        writer.write('#!/bin/sh\n')
                        writer.write(script)
                        writer.write('\n')

                    os.chmod(dest, 0o755)
                    argv.append('-t')
                    argv.append('post_script:post_script')

Frédéric Danis's avatar
Frédéric Danis committed
                pre_apt_script = self.runtime_details.get('pre_apt_script', '')

                if pre_apt_script:
                    dest = os.path.join(scratch, 'pre_apt_script')

                    with open(dest, 'w', encoding='utf-8') as writer:
                        writer.write('#!/bin/sh\n')
                        writer.write(pre_apt_script)
                        writer.write('\n')

                    os.chmod(dest, 0o755)
                    argv.append('-t')
                    argv.append('pre_apt_script:pre_apt_script')

                if sdk:
                    sources_tarball = sources_prefix + '.tar.gz'
                    debug_prefix = artifact_prefix + '-debug'
                    debug_tarball = debug_prefix + '.tar.gz'

                    sysroot_prefix = None       # type: typing.Optional[str]
                    sysroot_tarball = None      # type: typing.Optional[str]

                        sysroot_prefix = artifact_prefix + '-sysroot'
                        sysroot_tarball = sysroot_prefix + '.tar.gz'
                        argv.append('-t')
                        argv.append('sysroot_prefix:{}'.format(sysroot_prefix))
                        argv.append('-t')
                        argv.append(
                            'sysroot_tarball:{}'.format(
                                sysroot_tarball + '.new'))

                    if generate_source_directory:
                        os.makedirs(
                            os.path.join(
                                self.build_area,
                                generate_source_directory,
                            ),
                            0o755,
                            exist_ok=True,
                        )
                        argv.append('-t')
                        argv.append(
                            'sources_directory:{}'.format(
                                generate_source_directory))

                    if generate_source_tarball is None:
                        generate_source_tarball = not generate_source_directory

                    if self.collect_source_code and generate_source_tarball:
                        sources_tarball = sources_prefix + '.tar.gz'
                        argv.append('-t')
                        argv.append(
                            'sources_tarball:{}'.format(
                                sources_tarball + '.new'))

                    sdk_details = self.runtime_details.get('sdk', {})
                    argv.append('-t')
                    argv.append('sdk:yes')
                    argv.append('-t')

                    if self.debug_symbols:
                        argv.append('debug_symbols:yes')
                    else:
                        argv.append('debug_symbols:')

                    argv.append('-t')

                    if self.collect_source_code:
                        argv.append('collect_source_code:yes')
                    else:
                        argv.append('collect_source_code:')

                    argv.append('-t')

                    if self.automatic_dbgsym:
                        argv.append('automatic_dbgsym:yes')
                    else:
                        argv.append('automatic_dbgsym:')

                    argv.append('-t')
                    argv.append('debug_tarball:' + debug_tarball + '.new')
                    argv.append('-t')
                    argv.append('debug_prefix:' + debug_prefix)
                    argv.append('-t')
                    argv.append('sources_prefix:' + sources_prefix)
                    sdk_packages = list(self.get_runtime_packages(
                        sdk_details.get('add_packages', [])
                    ))
                    sdk_packages.extend(self.get_runtime_packages(
                        sdk_details.get('add_packages_multiarch', []),
                        multiarch=True,
                    ))
                    # We probably have this anyway, but we need it for
                    # dpkg-scansources
                    if 'dpkg-dev' not in sdk_packages:
                        sdk_packages.append('dpkg-dev')

                    if sdk_packages:
                        logger.info('Installing extra packages for SDK:')
                        sdk_packages.sort()

                        for p in sdk_packages:
                            logger.info('- %s', p)

                        argv.append('-t')
                        argv.append(
                            'sdk_packages:{}'.format(
                                self.yaml_dump_one_line(sdk_packages)))

                        dest = os.path.join(scratch, 'runtimes', runtime)
                        os.makedirs(dest, 0o755, exist_ok=True)
                        dest = os.path.join(dest, 'sdk_packages.yaml')

                        with open(dest, 'w', encoding='utf-8') as writer:
                            yaml.safe_dump(sdk_packages, stream=writer)

                    script = sdk_details.get('post_script', '')

                    if script:
                        dest = os.path.join(scratch, 'sdk_post_script')

                        with open(dest, 'w', encoding='utf-8') as writer:
                            writer.write('#!/bin/sh\n')
                            writer.write(script)
                            writer.write('\n')

                        os.chmod(dest, 0o755)
                        argv.append('-t')
                        argv.append('sdk_post_script:sdk_post_script')
                else:   # not sdk
                    platform_details = self.runtime_details.get('platform', {})
                    script = platform_details.get('post_script', '')

                    if script:
                        dest = os.path.join(scratch, 'platform_post_script')

                        with open(dest, 'w', encoding='utf-8') as writer:
                            writer.write('#!/bin/sh\n')
                            writer.write(script)
                            writer.write('\n')

                        os.chmod(dest, 0o755)
                        argv.append('-t')
                        argv.append(
                            'platform_post_script:platform_post_script')
                overlay = os.path.join(
                    scratch, 'runtimes', runtime, 'flatpak-overlay')
                self.create_flatpak_manifest_overlay(
                    overlay, prefix, runtime, sdk=sdk)
                overlay = os.path.join(
                    scratch, 'runtimes', runtime, 'apt-overlay')
                self.configure_apt(overlay, self.final_apt_sources)

                argv.append(dest_recipe)
                    output = os.path.join(
                        self.build_area,
                        sources_prefix + '.MISSING.txt',
                    )

                    if self.collect_source_code:
                        if os.path.exists(output) and self.strict:
                            raise SystemExit(
                                'Some source code was missing: aborting'
                            )
                    else:
                        with open(output, 'w') as writer:
                            writer.write('EVERYTHING\n')
                    if sysroot_prefix is not None:
                        assert sysroot_tarball is not None
                        output = os.path.join(self.build_area, sysroot_tarball)
                        os.rename(output + '.new', output)

                        output = os.path.join(
                            self.build_area, sysroot_prefix + '.Dockerfile')

                        with open(
                            os.path.join(
                                os.path.dirname(__file__), 'flatdeb',
                                'Dockerfile.in'),
                            'r',
                            encoding='utf-8',
                        ) as reader:
                            content = reader.read()

                        content = content.replace(
                            '@sysroot_tarball@', sysroot_tarball)

                        with open(
                            output + '.new', 'w', encoding='utf-8'
                        ) as writer:
                            writer.write(content)

                        os.rename(output + '.new', output)

                    if self.debug_symbols:
                        output = os.path.join(self.build_area, debug_tarball)
                        os.rename(output + '.new', output)
                    if self.ostree_commit and self.debug_symbols:
                        logger.info('Committing %s to OSTree', debug_tarball)
                        subprocess.check_call([
                            'time',
                            'ostree',
                            '--repo=' + self.ostree_repo,
                            'commit',
                            '--branch=runtime/{}.Debug/{}/{}'.format(
                                runtime,
                                self.flatpak_arch,
                                self.runtime_branch,
                            ),
                            '--subject=Update',
                            '--tree=tar={}'.format(output),
                            '--fsync=false',
                            '--tar-autocreate-parents',
                            '--add-metadata-string',
                            'xa.metadata=' + self.metadata_debug.to_data()[0],
                    if self.collect_source_code and generate_source_tarball:
                        output = os.path.join(self.build_area, sources_tarball)
                        os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed
                            logger.info(
                                'Committing %s to OSTree', sources_tarball)
                            subprocess.check_call([
                                'time',
                                'ostree',
                                '--repo=' + self.ostree_repo,
                                'commit',
                                '--branch=runtime/{}.Sources/{}/{}'.format(
                                    runtime,
                                    self.flatpak_arch,
                                    self.runtime_branch,
                                ),
                                '--subject=Update',
                                '--tree=tar={}'.format(output),
                                '--fsync=false',
                                '--tar-autocreate-parents',
                                '--add-metadata-string',
Simon McVittie's avatar
Simon McVittie committed
                                ('xa.metadata='
                                 + self.metadata_sources.to_data()[0]),
                output = os.path.join(self.build_area, out_tarball)
                os.rename(output + '.new', output)

                if self.ostree_commit:
                    logger.info('Committing %s to OSTree', out_tarball)
                    subprocess.check_call([
                        'time',
                        'ostree',
                        '--repo=' + self.ostree_repo,
                        '--branch=runtime/{}/{}/{}'.format(
                            runtime,
                            self.flatpak_arch,
                            self.runtime_branch,
                        ),
                        '--subject=Update',
                        '--tree=tar={}'.format(output),
                        '--fsync=false',
                        '--tar-autocreate-parents',
                        '--add-metadata-string',
                        'xa.metadata=' + self.metadata.to_data()[0],
            if self.ostree_commit:
                # Don't keep the history in this working repository:
                # if history is desired, mirror the commits into a public
                # repository and maintain history there.
                subprocess.check_call([
                    'time',
                    'ostree',
                    '--repo=' + self.ostree_repo,
                subprocess.check_call([
                    'time',
                    'flatpak',
                    'build-update-repo',
                    self.ostree_repo,
                ])
                if self.export_bundles:
                    for suffix in ('.Platform', '.Sdk'):
                        bundle = '{}-{}-{}.bundle'.format(
                            prefix + suffix,
                            ','.join(self.dpkg_archs),
                            self.runtime_branch,
                        )
                        output = os.path.join(self.build_area, bundle)

                        subprocess.check_call([
                            'time',
                            'flatpak',
                            'build-bundle',
                            '--runtime',
                            self.ostree_repo,
                            output + '.new',
                            prefix + suffix,
                            self.runtime_branch,
                        ])
Simon McVittie's avatar
Simon McVittie committed

                        os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

    def configure_apt(self, overlay, apt_sources):
        # type: (str, typing.Iterable[AptSource]) -> None
Simon McVittie's avatar
Simon McVittie committed
        """
        Configure apt. We only do this once, so that all chroots
        created from the same base have their version numbers
        aligned.
        """
        os.makedirs(
            os.path.join(overlay, 'etc', 'apt', 'trusted.gpg.d'),
            0o755,
            exist_ok=True,
        )
Simon McVittie's avatar
Simon McVittie committed

        with open(
            os.path.join(overlay, 'etc', 'apt', 'sources.list'),
            'w',
            encoding='utf-8'
        ) as writer:
                writer.write('{}\n'.format(source))

            for keyring in self.apt_keyrings:
                if os.path.exists(os.path.join('suites', keyring)):
                    keyring = os.path.join('suites', keyring)
                elif os.path.exists(keyring):
                    pass
                    raise RuntimeError('Cannot open {}'.format(keyring))

                shutil.copyfile(
                    os.path.abspath(keyring),
                    os.path.join(
                        overlay,
                        'etc', 'apt', 'trusted.gpg.d',
                        os.path.basename(keyring),
                    ),
                )
Simon McVittie's avatar
Simon McVittie committed

    def create_flatpak_manifest_overlay(
        self,
        overlay,        # type: str
        prefix,         # type: str
        runtime,        # type: str
        # type: (...) -> None
        metadata = os.path.join(overlay, 'metadata')
        os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)

        keyfile = self.metadata
        keyfile.set_string('Runtime', 'name', runtime)
        keyfile.set_string(
            'Runtime', 'runtime',
            '{}.Platform/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
            )
        )
        keyfile.set_string(
            'Runtime', 'sdk',
            '{}.Sdk/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
        keyfile.set_string(
            'Runtime', 'x-flatdeb-sources',
            '{}.Sdk.Sources/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
            ),
        )
Simon McVittie's avatar
Simon McVittie committed

        keyfile.set_string(
            'Environment', 'XDG_DATA_DIRS',
            ':'.join([
                '/app/share', '/usr/share', '/usr/share/runtime/share',
            ]),
        )
        if sdk:
            keyfile.set_string(
                'Extension {}.Sdk.Debug'.format(prefix),
                'directory', 'lib/debug',
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Debug'.format(prefix),
                'autodelete', True,
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Debug'.format(prefix),
                'no-autodownload', True,
            )

            keyfile.set_string(
                'Extension {}.Sdk.Sources'.format(prefix),
                'directory', 'runtime/src',
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Sources'.format(prefix),
                'autodelete', True,
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Sources'.format(prefix),
                'no-autodownload', True,
            )

        for arch in self.dpkg_archs:
            search_path.append('/app/lib/{}'.format(
                self.multiarch_tuple(arch)))
        search_path.append('/app/lib')
        keyfile.set_string(
            'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
        )
        if True:    # TODO: 'libgstreamer1.0-0' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/extensions/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/gstreamer-1.0')
Simon McVittie's avatar
Simon McVittie committed

            keyfile.set_string(
                'Environment', 'GST_PLUGIN_SYSTEM_PATH',
                ':'.join(search_path),
            )
        if True:    # TODO: 'libgirepository-1.0-1' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/girepository-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/girepository-1.0')
Simon McVittie's avatar
Simon McVittie committed

                'Environment', 'GI_TYPELIB_PATH',
                ':'.join(search_path),
        keyfile.set_string(
            'Runtime', 'x-flatdeb-version', VERSION,
        )
        if self.build_id is not None:
            keyfile.set_string(
                'Runtime', 'x-flatdeb-build-id', self.build_id,
            )

        for ext, detail in self.runtime_details.get(
                'add-extensions', {}
                ).items():
            group = 'Extension {}'.format(ext)
            for k, v in detail.items():
                if isinstance(v, str):
                    keyfile.set_string(group, k, v)
                elif isinstance(v, bool):
                    keyfile.set_boolean(group, k, v)
                else:
                    raise RuntimeError(
                        'Unknown type {} in {}'.format(v, ext))
Simon McVittie's avatar
Simon McVittie committed

        keyfile.save_to_file(metadata)
Simon McVittie's avatar
Simon McVittie committed

            metadata = os.path.join(overlay, 'debug', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)

            keyfile = self.metadata_debug
            keyfile.set_string('Runtime', 'name', runtime + '.Debug')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )

            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )

            if self.build_id is not None:
                keyfile.set_string(
                    'Runtime', 'x-flatdeb-build-id', self.build_id,
                )

            metadata = os.path.join(overlay, 'src', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
            keyfile = self.metadata_sources
            keyfile.set_string('Runtime', 'name', runtime + '.Sources')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )
            if self.build_id is not None:
                keyfile.set_string(
                    'Runtime', 'x-flatdeb-build-id', self.build_id,
                )

            keyfile.save_to_file(metadata)
    def command_app(
        self,
        *,
        app_branch,         # type: str
        yaml_manifest,      # type: str
        **kwargs
    ):
        # type: (...) -> None

        if not self.ostree_commit:
            logger.error(
                'flatdeb app --no-ostree-commit cannot work')
            raise SystemExit(1)

        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed

        with open(yaml_manifest, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

        if self.remote_url is None:
            self.remote_url = 'file://{}'.format(
                urllib.parse.quote(self.ostree_repo))
Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            # We assume the build area has xattr support
            self.ensure_build_area()
            self.ensure_local_repo()
            scratch = stack.enter_context(
                TemporaryDirectory(
                    prefix='flatdeb.',
                    dir=os.path.join(self.build_area, 'tmp'),
                )
            )
            os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
            subprocess.check_call([
                'XDG_DATA_HOME={}/home'.format(scratch),
                'flatpak', '--user',
                'remote-add', '--if-not-exists', '--no-gpg-verify',
                '{}'.format(self.remote_url),
Simon McVittie's avatar
Simon McVittie committed
                'env',
                'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                'flatpak', '--user',
                'remote-modify', '--no-gpg-verify',
                '--url={}'.format(self.remote_url),
Simon McVittie's avatar
Simon McVittie committed
            ])

            for runtime in (manifest['sdk'], manifest['runtime']):
                # This may fail: we might already have it.
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'install', '-y', 'flatdeb',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'update',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
Simon McVittie's avatar
Simon McVittie committed

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-git.',
                                        dir=scratch,
                                    ),
                                )
                                uploader = subprocess.Popen([
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-path.',
                                        dir=scratch,
                                    ),
                                )
                                clone = os.path.join(
Simon McVittie's avatar
Simon McVittie committed
                                    d, os.path.basename(source['path']),
                                )
Simon McVittie's avatar
Simon McVittie committed

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    os.chmod(clone, 0o755)
                                else:
                                    os.chmod(clone, 0o644)
Simon McVittie's avatar
Simon McVittie committed

                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = stack.enter_context(
                            TemporaryDirectory(
                                prefix='flatdeb-debs.',
                                dir=scratch,
                            ),
                        )
                        shutil.copy2(
                            os.path.join(
                                os.path.dirname(__file__),
                                'flatdeb',
                                'collect-app-source-code',
                            ),
                            packages
                        )
Simon McVittie's avatar
Simon McVittie committed
                            'env',
                            'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'DEBIAN_FRONTEND=noninteractive',
                            '{}/collect-app-source-code'.format(packages),
                            '--export={}'.format(packages),
                            '--strip-source-version-suffix={}'.format(
                                self.strip_source_version_suffix),
Simon McVittie's avatar
Simon McVittie committed
                        ] + module['x-flatdeb-apt-packages'])
                        os.remove(
                            os.path.join(packages, 'collect-app-source-code')
                        )
Simon McVittie's avatar
Simon McVittie committed

                        obtained = subprocess.check_output([
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').splitlines()

                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
                            path = '{}/{}'.format(packages, f)

                            sources.append({
                                'dest': (os.path.dirname(f) or '.'),
                                'type': 'file',
                                'sha256': sha256,
                                'url': urllib.parse.urlunsplit((
                                    'file',
                                    '',
                                    urllib.parse.quote(path),
                                    '',
                                    '',
                                ))
                            })
Simon McVittie's avatar
Simon McVittie committed

            json_manifest = os.path.join(scratch, manifest['id'] + '.json')
            os.makedirs(
                os.path.join(self.build_area, '.flatpak-builder'),
                exist_ok=True,
            )
Simon McVittie's avatar
Simon McVittie committed

            if self.build_area != scratch:
                subprocess.check_call([
                    os.path.join(self.build_area, '.flatpak-builder'),
                    '{}/'.format(scratch),
            with open(json_manifest, 'w', encoding='utf-8') as writer:
                json.dump(manifest, writer, indent=2, sort_keys=True)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
                'env',
                'DEBIAN_FRONTEND=noninteractive',
                'XDG_DATA_HOME={}/home'.format(scratch),
                'sh', '-euc',
                'cd "$1"; shift; exec "$@"',
                'sh',                   # argv[0]
                scratch,                # directory to cd into
Simon McVittie's avatar
Simon McVittie committed
                'flatpak-builder',
                '--arch={}'.format(self.flatpak_arch),
                '--repo={}'.format(self.ostree_repo),
                os.path.join(scratch, 'workdir'),
                json_manifest,
            if self.export_bundles:
                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)
                subprocess.check_call([
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak',
                    'build-bundle',
                    self.ostree_repo,
                    manifest['id'],
                    manifest['branch'],
                ])
                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
Simon McVittie's avatar
Simon McVittie committed
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    else:
        logging.basicConfig()

    logging.getLogger().setLevel(logging.DEBUG)