Skip to content
Snippets Groups Projects
run.py 93.9 KiB
Newer Older
                        if self.ostree_commit:
                            logger.info(
                                'Committing %s to OSTree', debug_tarball,
                            )
                            subprocess.check_call([
                                'time',
                                'ostree',
                                '--repo=' + self.ostree_repo,
                                'commit',
                                '--branch=runtime/{}.Debug/{}/{}'.format(
                                    runtime,
                                    self.flatpak_arch,
                                    self.runtime_branch,
                                ),
                                '--subject=Update',
                                '--tree=tar={}'.format(output),
                                '--fsync=false',
                                '--tar-autocreate-parents',
                                '--add-metadata-string',
                                ('xa.metadata='
                                 + self.metadata_debug.to_data()[0]),
                            ])
                    if self.collect_source_code and generate_source_tarball:
                        output = os.path.join(self.build_area, sources_tarball)
                        os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed
                            logger.info(
                                'Committing %s to OSTree', sources_tarball)
                            subprocess.check_call([
                                'time',
                                'ostree',
                                '--repo=' + self.ostree_repo,
                                'commit',
                                '--branch=runtime/{}.Sources/{}/{}'.format(
                                    runtime,
                                    self.flatpak_arch,
                                    self.runtime_branch,
                                ),
                                '--subject=Update',
                                '--tree=tar={}'.format(output),
                                '--fsync=false',
                                '--tar-autocreate-parents',
                                '--add-metadata-string',
Simon McVittie's avatar
Simon McVittie committed
                                ('xa.metadata='
                                 + self.metadata_sources.to_data()[0]),
                output = os.path.join(self.build_area, out_tarball)
                os.rename(output + '.new', output)

                mtree = os.path.join(
                    self.build_area,
                    ostree_prefix + '.mtree.gz')

                if self.do_mtree:
                    self.generate_mtree(output, mtree)

                if self.ostree_commit:
                    logger.info('Committing %s to OSTree', out_tarball)
                    subprocess.check_call([
                        'time',
                        'ostree',
                        '--repo=' + self.ostree_repo,
                        '--branch=runtime/{}/{}/{}'.format(
                            runtime,
                            self.flatpak_arch,
                            self.runtime_branch,
                        ),
                        '--subject=Update',
                        '--tree=tar={}'.format(output),
                        '--fsync=false',
                        '--tar-autocreate-parents',
                        '--add-metadata-string',
                        'xa.metadata=' + self.metadata.to_data()[0],
            if self.ostree_commit:
                # Don't keep the history in this working repository:
                # if history is desired, mirror the commits into a public
                # repository and maintain history there.
                subprocess.check_call([
                    'time',
                    'ostree',
                    '--repo=' + self.ostree_repo,
                subprocess.check_call([
                    'time',
                    'flatpak',
                    'build-update-repo',
                    self.ostree_repo,
                ])
                if self.export_bundles:
                    for suffix in ('.Platform', '.Sdk'):
                        bundle = '{}-{}-{}.bundle'.format(
                            prefix + suffix,
                            ','.join(self.dpkg_archs),
                            self.runtime_branch,
                        )
                        output = os.path.join(self.build_area, bundle)

                        subprocess.check_call([
                            'time',
                            'flatpak',
                            'build-bundle',
                            '--runtime',
                            self.ostree_repo,
                            output + '.new',
                            prefix + suffix,
                            self.runtime_branch,
                        ])
Simon McVittie's avatar
Simon McVittie committed

                        os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

    def configure_apt(
        self,
        overlay,        # type: str
        apt_sources,    # type: typing.Iterable[AptSource]
        apt_keyrings,   # type: typing.Iterable[str]
Simon McVittie's avatar
Simon McVittie committed
        """
        Configure apt. We only do this once, so that all chroots
        created from the same base have their version numbers
        aligned.
        """
        os.makedirs(
            os.path.join(overlay, 'etc', 'apt', 'apt.conf.d'),
            0o755,
            exist_ok=True,
        )
        os.makedirs(
            os.path.join(overlay, 'etc', 'apt', 'sources.list.d'),
            0o755,
            exist_ok=True,
        )
        os.makedirs(
            os.path.join(overlay, 'etc', 'apt', 'trusted.gpg.d'),
            0o755,
            exist_ok=True,
        )
Simon McVittie's avatar
Simon McVittie committed

        with open(
            os.path.join(overlay, 'etc', 'apt', 'sources.list'),
            'w',
            encoding='utf-8'
        ) as writer:
                writer.write('{}\n'.format(source))

                if os.path.exists(os.path.join('suites', keyring)):
                    keyring = os.path.join('suites', keyring)
                elif os.path.exists(keyring):
                    pass
                    raise RuntimeError('Cannot open {}'.format(keyring))

                shutil.copyfile(
                    os.path.abspath(keyring),
                    os.path.join(
                        overlay,
                        'etc', 'apt', 'trusted.gpg.d',
                        apt_keyring_prefix + os.path.basename(keyring),
Simon McVittie's avatar
Simon McVittie committed

    def create_flatpak_manifest_overlay(
        self,
        overlay,        # type: str
        prefix,         # type: str
        runtime,        # type: str
        # type: (...) -> None
        metadata = os.path.join(overlay, 'metadata')
        os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)

        keyfile = self.metadata
        keyfile.set_string('Runtime', 'name', runtime)
        keyfile.set_string(
            'Runtime', 'runtime',
            '{}.Platform/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
            )
        )
        keyfile.set_string(
            'Runtime', 'sdk',
            '{}.Sdk/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
        keyfile.set_string(
            'Runtime', 'x-flatdeb-sources',
            '{}.Sdk.Sources/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
            ),
        )
Simon McVittie's avatar
Simon McVittie committed

        keyfile.set_string(
            'Environment', 'XDG_DATA_DIRS',
            ':'.join([
                '/app/share', '/usr/share', '/usr/share/runtime/share',
            ]),
        )
        if sdk:
            keyfile.set_string(
                'Extension {}.Sdk.Debug'.format(prefix),
                'directory', 'lib/debug',
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Debug'.format(prefix),
                'autodelete', True,
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Debug'.format(prefix),
                'no-autodownload', True,
            )

            keyfile.set_string(
                'Extension {}.Sdk.Sources'.format(prefix),
                'directory', 'runtime/src',
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Sources'.format(prefix),
                'autodelete', True,
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Sources'.format(prefix),
                'no-autodownload', True,
            )

        for arch in self.dpkg_archs:
            search_path.append('/app/lib/{}'.format(
                self.multiarch_tuple(arch)))
        search_path.append('/app/lib')
        keyfile.set_string(
            'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
        )
        if True:    # TODO: 'libgstreamer1.0-0' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/extensions/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/gstreamer-1.0')
Simon McVittie's avatar
Simon McVittie committed

            keyfile.set_string(
                'Environment', 'GST_PLUGIN_SYSTEM_PATH',
                ':'.join(search_path),
            )
        if True:    # TODO: 'libgirepository-1.0-1' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/girepository-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/girepository-1.0')
Simon McVittie's avatar
Simon McVittie committed

                'Environment', 'GI_TYPELIB_PATH',
                ':'.join(search_path),
        keyfile.set_string(
            'Runtime', 'x-flatdeb-version', VERSION,
        )
        if self.build_id is not None:
            keyfile.set_string(
                'Runtime', 'x-flatdeb-build-id', self.build_id,
            )

        for ext, detail in self.runtime_details.get(
                'add-extensions', {}
                ).items():
            group = 'Extension {}'.format(ext)
            for k, v in detail.items():
                if isinstance(v, str):
                    keyfile.set_string(group, k, v)
                elif isinstance(v, bool):
                    keyfile.set_boolean(group, k, v)
                else:
                    raise RuntimeError(
                        'Unknown type {} in {}'.format(v, ext))
Simon McVittie's avatar
Simon McVittie committed

        keyfile.save_to_file(metadata)
Simon McVittie's avatar
Simon McVittie committed

            metadata = os.path.join(overlay, 'debug', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)

            keyfile = self.metadata_debug
            keyfile.set_string('Runtime', 'name', runtime + '.Debug')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )

            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )

            if self.build_id is not None:
                keyfile.set_string(
                    'Runtime', 'x-flatdeb-build-id', self.build_id,
                )

            metadata = os.path.join(overlay, 'src', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
            keyfile = self.metadata_sources
            keyfile.set_string('Runtime', 'name', runtime + '.Sources')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )
            if self.build_id is not None:
                keyfile.set_string(
                    'Runtime', 'x-flatdeb-build-id', self.build_id,
                )

            keyfile.save_to_file(metadata)
    def command_app(
        self,
        *,
        app_branch,         # type: str
        yaml_manifest,      # type: str
        **kwargs
    ):
        # type: (...) -> None

        if not self.ostree_commit:
            logger.error(
                'flatdeb app --no-ostree-commit cannot work')
            raise SystemExit(1)

        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed

        with open(yaml_manifest, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

        if self.remote_url is None:
            self.remote_url = 'file://{}'.format(
                urllib.parse.quote(self.ostree_repo))
Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            # We assume the build area has xattr support
            self.ensure_build_area()
            self.ensure_local_repo()
            scratch = stack.enter_context(
                TemporaryDirectory(
                    prefix='flatdeb.',
                    dir=os.path.join(self.build_area, 'tmp'),
                )
            )
            os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
            subprocess.check_call([
                'XDG_DATA_HOME={}/home'.format(scratch),
                'flatpak', '--user',
                'remote-add', '--if-not-exists', '--no-gpg-verify',
                '{}'.format(self.remote_url),
Simon McVittie's avatar
Simon McVittie committed
                'env',
                'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                'flatpak', '--user',
                'remote-modify', '--no-gpg-verify',
                '--url={}'.format(self.remote_url),
Simon McVittie's avatar
Simon McVittie committed
            ])

            for runtime in (manifest['sdk'], manifest['runtime']):
                # This may fail: we might already have it.
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'install', '-y', 'flatdeb',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'update',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
Simon McVittie's avatar
Simon McVittie committed

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-git.',
                                        dir=scratch,
                                    ),
                                )
                                uploader = subprocess.Popen([
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-path.',
                                        dir=scratch,
                                    ),
                                )
                                clone = os.path.join(
Simon McVittie's avatar
Simon McVittie committed
                                    d, os.path.basename(source['path']),
                                )
Simon McVittie's avatar
Simon McVittie committed

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    os.chmod(clone, 0o755)
                                else:
                                    os.chmod(clone, 0o644)
Simon McVittie's avatar
Simon McVittie committed

                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = stack.enter_context(
                            TemporaryDirectory(
                                prefix='flatdeb-debs.',
                                dir=scratch,
                            ),
                        )
                        shutil.copy2(
                            os.path.join(
                                os.path.dirname(__file__),
                                'flatdeb',
                                'collect-app-source-code',
                            ),
                            packages
                        )
Simon McVittie's avatar
Simon McVittie committed
                            'env',
                            'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'DEBIAN_FRONTEND=noninteractive',
                            '{}/collect-app-source-code'.format(packages),
                            '--export={}'.format(packages),
                            '--strip-source-version-suffix={}'.format(
                                self.strip_source_version_suffix),
Simon McVittie's avatar
Simon McVittie committed
                        ] + module['x-flatdeb-apt-packages'])
                        os.remove(
                            os.path.join(packages, 'collect-app-source-code')
                        )
Simon McVittie's avatar
Simon McVittie committed

                        obtained = subprocess.check_output([
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').splitlines()

                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
                            path = '{}/{}'.format(packages, f)

                            sources.append({
                                'dest': (os.path.dirname(f) or '.'),
                                'type': 'file',
                                'sha256': sha256,
                                'url': urllib.parse.urlunsplit((
                                    'file',
                                    '',
                                    urllib.parse.quote(path),
                                    '',
                                    '',
                                ))
                            })
Simon McVittie's avatar
Simon McVittie committed

            json_manifest = os.path.join(scratch, manifest['id'] + '.json')
            os.makedirs(
                os.path.join(self.build_area, '.flatpak-builder'),
                exist_ok=True,
            )
Simon McVittie's avatar
Simon McVittie committed

            if self.build_area != scratch:
                subprocess.check_call([
                    os.path.join(self.build_area, '.flatpak-builder'),
                    '{}/'.format(scratch),
            with open(json_manifest, 'w', encoding='utf-8') as writer:
                json.dump(manifest, writer, indent=2, sort_keys=True)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
                'env',
                'DEBIAN_FRONTEND=noninteractive',
                'XDG_DATA_HOME={}/home'.format(scratch),
                'sh', '-euc',
                'cd "$1"; shift; exec "$@"',
                'sh',                   # argv[0]
                scratch,                # directory to cd into
Simon McVittie's avatar
Simon McVittie committed
                'flatpak-builder',
                '--arch={}'.format(self.flatpak_arch),
                '--repo={}'.format(self.ostree_repo),
                os.path.join(scratch, 'workdir'),
                json_manifest,
            if self.export_bundles:
                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)
                subprocess.check_call([
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak',
                    'build-bundle',
                    self.ostree_repo,
                    manifest['id'],
                    manifest['branch'],
                ])
                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
Simon McVittie's avatar
Simon McVittie committed
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    else:
        logging.basicConfig()

    logging.getLogger().setLevel(logging.DEBUG)
Simon McVittie's avatar
Simon McVittie committed

    try:
        Builder().run_command_line()
    except KeyboardInterrupt:
        raise SystemExit(130)
    except subprocess.CalledProcessError as e:
        logger.error('%s', e)
        raise SystemExit(1)