Skip to content
Snippets Groups Projects
run.py 62 KiB
Newer Older
                                self.yaml_dump_one_line(sdk_packages)))

                        dest = os.path.join(scratch, 'runtimes', runtime)
                        os.makedirs(dest, 0o755, exist_ok=True)
                        dest = os.path.join(dest, 'sdk_packages.yaml')

                        with open(dest, 'w', encoding='utf-8') as writer:
                            yaml.safe_dump(sdk_packages, stream=writer)

                    script = sdk_details.get('post_script', '')

                    if script:
                        dest = os.path.join(scratch, 'sdk_post_script')

                        with open(dest, 'w', encoding='utf-8') as writer:
                            writer.write('#!/bin/sh\n')
                            writer.write(script)
                            writer.write('\n')

                        os.chmod(dest, 0o755)
                        argv.append('-t')
                        argv.append('sdk_post_script:sdk_post_script')
                else:   # not sdk
                    platform_details = self.runtime_details.get('platform', {})
                    script = platform_details.get('post_script', '')

                    if script:
                        dest = os.path.join(scratch, 'platform_post_script')

                        with open(dest, 'w', encoding='utf-8') as writer:
                            writer.write('#!/bin/sh\n')
                            writer.write(script)
                            writer.write('\n')

                        os.chmod(dest, 0o755)
                        argv.append('-t')
                        argv.append(
                            'platform_post_script:platform_post_script')
                overlay = os.path.join(scratch, 'runtimes', runtime, 'overlay')
                self.create_flatpak_manifest_overlay(
                    overlay, prefix, runtime, sdk=sdk)
                self.configure_apt(
                    os.path.join(scratch, 'runtimes', runtime, 'overlay'),
                    self.final_apt_sources,
                )

                argv.append(dest_recipe)
                    if sysroot_prefix is not None:
                        assert sysroot_tarball is not None
                        output = os.path.join(self.build_area, sysroot_tarball)
                        os.rename(output + '.new', output)

                        output = os.path.join(
                            self.build_area, sysroot_prefix + '.Dockerfile')

                        with open(
                            os.path.join(
                                os.path.dirname(__file__), 'flatdeb',
                                'Dockerfile.in'),
                            'r',
                            encoding='utf-8',
                        ) as reader:
                            content = reader.read()

                        content = content.replace(
                            '@sysroot_tarball@', sysroot_tarball)

                        with open(
                            output + '.new', 'w', encoding='utf-8'
                        ) as writer:
                            writer.write(content)

                        os.rename(output + '.new', output)

                    output = os.path.join(self.build_area, debug_tarball)
                    os.rename(output + '.new', output)

                    if self.ostree_commit:
                        logger.info('Committing %s to OSTree', debug_tarball)
                        subprocess.check_call([
                            'time',
                            'ostree',
                            '--repo=' + self.ostree_repo,
                            'commit',
                            '--branch=runtime/{}.Debug/{}/{}'.format(
                                runtime,
                                self.flatpak_arch,
                                self.runtime_branch,
                            ),
                            '--subject=Update',
                            '--tree=tar={}'.format(output),
                            '--fsync=false',
                            '--tar-autocreate-parents',
                        ])

                    output = os.path.join(self.build_area, sources_tarball)
                    os.rename(output + '.new', output)

                    if self.ostree_commit:
                        logger.info('Committing %s to OSTree', sources_tarball)
                        subprocess.check_call([
                            'time',
                            'ostree',
                            '--repo=' + self.ostree_repo,
                            'commit',
                            '--branch=runtime/{}.Sources/{}/{}'.format(
                                runtime,
                                self.flatpak_arch,
                                self.runtime_branch,
                            ),
                            '--subject=Update',
                            '--tree=tar={}'.format(output),
                            '--fsync=false',
                            '--tar-autocreate-parents',
                        ])

                output = os.path.join(self.build_area, out_tarball)
                os.rename(output + '.new', output)

                if self.ostree_commit:
                    logger.info('Committing %s to OSTree', out_tarball)
                    subprocess.check_call([
                        'time',
                        'ostree',
                        '--repo=' + self.ostree_repo,
                        '--branch=runtime/{}/{}/{}'.format(
                            runtime,
                            self.flatpak_arch,
                            self.runtime_branch,
                        ),
                        '--subject=Update',
                        '--tree=tar={}'.format(output),
                        '--fsync=false',
                        '--tar-autocreate-parents',
                    ])
            if self.ostree_commit:
                # Don't keep the history in this working repository:
                # if history is desired, mirror the commits into a public
                # repository and maintain history there.
                subprocess.check_call([
                    'time',
                    'ostree',
                    '--repo=' + self.ostree_repo,
                subprocess.check_call([
                    'time',
                    'flatpak',
                    'build-update-repo',
                    self.ostree_repo,
                ])
                if self.export_bundles:
                    for suffix in ('.Platform', '.Sdk'):
                        bundle = '{}-{}-{}.bundle'.format(
                            prefix + suffix,
                            ','.join(self.dpkg_archs),
                            self.runtime_branch,
                        )
                        output = os.path.join(self.build_area, bundle)

                        subprocess.check_call([
                            'time',
                            'flatpak',
                            'build-bundle',
                            '--runtime',
                            self.ostree_repo,
                            output + '.new',
                            prefix + suffix,
                            self.runtime_branch,
                        ])
Simon McVittie's avatar
Simon McVittie committed

                        os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

    def configure_apt(self, overlay, apt_sources):
        # type: (str, typing.Iterable[AptSource]) -> None
Simon McVittie's avatar
Simon McVittie committed
        """
        Configure apt. We only do this once, so that all chroots
        created from the same base have their version numbers
        aligned.
        """
        os.makedirs(
            os.path.join(overlay, 'etc', 'apt', 'trusted.gpg.d'),
            0o755,
            exist_ok=True,
        )
Simon McVittie's avatar
Simon McVittie committed

        with open(
            os.path.join(overlay, 'etc', 'apt', 'sources.list'),
            'w',
            encoding='utf-8'
        ) as writer:
                writer.write('{}\n'.format(source))

            for keyring in self.apt_keyrings:
                if os.path.exists(os.path.join('suites', keyring)):
                    keyring = os.path.join('suites', keyring)
                elif os.path.exists(keyring):
                    pass
                    raise RuntimeError('Cannot open {}'.format(keyring))

                shutil.copyfile(
                    os.path.abspath(keyring),
                    os.path.join(
                        overlay,
                        'etc', 'apt', 'trusted.gpg.d',
                        os.path.basename(keyring),
                    ),
                )
Simon McVittie's avatar
Simon McVittie committed

    def create_flatpak_manifest_overlay(
        self,
        overlay,        # type: str
        prefix,         # type: str
        runtime,        # type: str
        # type: (...) -> None
        metadata = os.path.join(overlay, 'metadata')
        os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)

        keyfile = GLib.KeyFile()
        keyfile.set_string('Runtime', 'name', runtime)
        keyfile.set_string(
            'Runtime', 'runtime',
            '{}.Platform/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
            )
        )
        keyfile.set_string(
            'Runtime', 'sdk',
            '{}.Sdk/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
        keyfile.set_string(
            'Runtime', 'x-flatdeb-sources',
            '{}.Sdk.Sources/{}/{}'.format(
                prefix,
                self.flatpak_arch,
                self.runtime_branch,
            ),
        )
Simon McVittie's avatar
Simon McVittie committed

        keyfile.set_string(
            'Environment', 'XDG_DATA_DIRS',
            ':'.join([
                '/app/share', '/usr/share', '/usr/share/runtime/share',
            ]),
        )
        if sdk:
            keyfile.set_string(
                'Extension {}.Sdk.Debug'.format(prefix),
                'directory', 'lib/debug',
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Debug'.format(prefix),
                'autodelete', True,
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Debug'.format(prefix),
                'no-autodownload', True,
            )

            keyfile.set_string(
                'Extension {}.Sdk.Sources'.format(prefix),
                'directory', 'runtime/src',
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Sources'.format(prefix),
                'autodelete', True,
            )
            keyfile.set_boolean(
                'Extension {}.Sdk.Sources'.format(prefix),
                'no-autodownload', True,
            )

        for arch in self.dpkg_archs:
            search_path.append('/app/lib/{}'.format(
                self.multiarch_tuple(arch)))
        search_path.append('/app/lib')
        keyfile.set_string(
            'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
        )
        if True:    # TODO: 'libgstreamer1.0-0' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/extensions/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/gstreamer-1.0')
Simon McVittie's avatar
Simon McVittie committed

            keyfile.set_string(
                'Environment', 'GST_PLUGIN_SYSTEM_PATH',
                ':'.join(search_path),
            )
        if True:    # TODO: 'libgirepository-1.0-1' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/girepository-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/girepository-1.0')
Simon McVittie's avatar
Simon McVittie committed

                'Environment', 'GI_TYPELIB_PATH',
                ':'.join(search_path),
        keyfile.set_string(
            'Runtime', 'x-flatdeb-version', VERSION,
        )
        if self.build_id is not None:
            keyfile.set_string(
                'Runtime', 'x-flatdeb-build-id', self.build_id,
            )

        for ext, detail in self.runtime_details.get(
                'add-extensions', {}
                ).items():
            group = 'Extension {}'.format(ext)
            for k, v in detail.items():
                if isinstance(v, str):
                    keyfile.set_string(group, k, v)
                elif isinstance(v, bool):
                    keyfile.set_boolean(group, k, v)
                else:
                    raise RuntimeError(
                        'Unknown type {} in {}'.format(v, ext))
Simon McVittie's avatar
Simon McVittie committed

        keyfile.save_to_file(metadata)
Simon McVittie's avatar
Simon McVittie committed

            metadata = os.path.join(overlay, 'debug', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)

            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime + '.Debug')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )

            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )

            if self.build_id is not None:
                keyfile.set_string(
                    'Runtime', 'x-flatdeb-build-id', self.build_id,
                )

            metadata = os.path.join(overlay, 'src', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime + '.Sources')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )
            if self.build_id is not None:
                keyfile.set_string(
                    'Runtime', 'x-flatdeb-build-id', self.build_id,
                )

            keyfile.save_to_file(metadata)
    def command_app(
        self,
        *,
        app_branch,         # type: str
        yaml_manifest,      # type: str
        **kwargs
    ):
        # type: (...) -> None

        if not self.ostree_commit:
            logger.error(
                'flatdeb app --no-ostree-commit cannot work')
            raise SystemExit(1)

        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed

        with open(yaml_manifest, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            # We assume the build area has xattr support
            self.ensure_build_area()
            self.ensure_local_repo()
            scratch = stack.enter_context(
                TemporaryDirectory(
                    prefix='flatdeb.',
                    dir=os.path.join(self.build_area, 'tmp'),
                )
            )
            os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
            subprocess.check_call([
                'XDG_DATA_HOME={}/home'.format(scratch),
                'flatpak', '--user',
                'remote-add', '--if-not-exists', '--no-gpg-verify',
                'file://{}'.format(urllib.parse.quote(self.ostree_repo)),
Simon McVittie's avatar
Simon McVittie committed
                'env',
                'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                'flatpak', '--user',
                'remote-modify', '--no-gpg-verify',
                '--url=file://{}'.format(urllib.parse.quote(self.ostree_repo)),
Simon McVittie's avatar
Simon McVittie committed
            ])

            for runtime in (manifest['sdk'], manifest['runtime']):
                # This may fail: we might already have it.
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'install', '-y', 'flatdeb',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'update',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
Simon McVittie's avatar
Simon McVittie committed

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-git.',
                                        dir=scratch,
                                    ),
                                )
                                uploader = subprocess.Popen([
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-path.',
                                        dir=scratch,
                                    ),
                                )
                                clone = os.path.join(
Simon McVittie's avatar
Simon McVittie committed
                                    d, os.path.basename(source['path']),
                                )
Simon McVittie's avatar
Simon McVittie committed

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    os.chmod(clone, 0o755)
                                else:
                                    os.chmod(clone, 0o644)
Simon McVittie's avatar
Simon McVittie committed

                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = stack.enter_context(
                            TemporaryDirectory(
                                prefix='flatdeb-debs.',
                                dir=scratch,
                            ),
                        )
                        subprocess.check_call([
Simon McVittie's avatar
Simon McVittie committed
                            'env',
                            'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'DEBIAN_FRONTEND=noninteractive',
Simon McVittie's avatar
Simon McVittie committed
                            'export={}'.format(packages),
                            'sh',
                            '-euc',

                            'cp -PRp /usr/var /\n'
Simon McVittie's avatar
Simon McVittie committed
                            'install -d /var/cache/apt/archives/partial\n'
                            'fakeroot apt-get update\n'
                            'fakeroot apt-get -y --download-only \\\n'
                            '    --no-install-recommends install "$@"\n'
                            'for x in /var/cache/apt/archives/*.deb; do\n'
                            '    package="$(dpkg-deb -f "$x" Package)"\n'
                            '    source="$(dpkg-deb -f "$x" Source)"\n'
                            '    bu="$(dpkg-deb -f "$x" Built-Using)"\n'
                            '    version="$(dpkg-deb -f "$x" Version)"\n'
                            '    if [ -z "$source" ]; then\n'
                            '        source="$package"\n'
                            '    fi\n'
                            '    if [ "${source% (*}" != "$source" ]; then\n'
                            '        version="${source#* (}"\n'
                            '        version="${version%)}"\n'
                            '        source="${source% (*}"\n'
                            '    fi\n'
                            '    ( cd "$export" && \\\n'
                            '         apt-get -y --download-only \\\n'
                            '         -oAPT::Get::Only-Source=true source \\\n'
                            '         "$source=$version"\n'
                            '    )\n'
                            '    if [ -n "$bu" ]; then\n'
                            '        oldIFS="$IFS"\n'
                            '        IFS=","\n'
                            '        for dep in $bu; do\n'
                            '            bu="$(echo "$bu" | tr -d " ")"\n'
                            '            version="${bu#*(=}"\n'
                            '            version="${version%)}"\n'
                            '            source="${bu%(*}"\n'
                            '            ( cd "$export" && \\\n'
                            '               apt-get -y --download-only \\\n'
                            '               -oAPT::Get::Only-Source=true \\\n'
                            '               source "$source=$version"\n'
Simon McVittie's avatar
Simon McVittie committed
                            'mv /var/cache/apt/archives/*.deb "$export"\n'
                            'mv /var/lib/apt/lists "$export"\n'
                            '',

                            'sh',   # argv[0]
                        ] + module['x-flatdeb-apt-packages'])

                        obtained = subprocess.check_output([
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').splitlines()

                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
                            path = '{}/{}'.format(packages, f)

                            sources.append({
                                'dest': (os.path.dirname(f) or '.'),
                                'type': 'file',
                                'sha256': sha256,
                                'url': urllib.parse.urlunsplit((
                                    'file',
                                    '',
                                    urllib.parse.quote(path),
                                    '',
                                    '',
                                ))
                            })
Simon McVittie's avatar
Simon McVittie committed

            json_manifest = os.path.join(scratch, manifest['id'] + '.json')
            os.makedirs(
                os.path.join(self.build_area, '.flatpak-builder'),
                exist_ok=True,
            )
Simon McVittie's avatar
Simon McVittie committed

            if self.build_area != scratch:
                subprocess.check_call([
                    os.path.join(self.build_area, '.flatpak-builder'),
                    '{}/'.format(scratch),
            with open(json_manifest, 'w', encoding='utf-8') as writer:
                json.dump(manifest, writer, indent=2, sort_keys=True)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
                'env',
                'DEBIAN_FRONTEND=noninteractive',
                'XDG_DATA_HOME={}/home'.format(scratch),
                'sh', '-euc',
                'cd "$1"; shift; exec "$@"',
                'sh',                   # argv[0]
                scratch,                # directory to cd into
Simon McVittie's avatar
Simon McVittie committed
                'flatpak-builder',
                '--arch={}'.format(self.flatpak_arch),
                '--repo={}'.format(self.ostree_repo),
                os.path.join(scratch, 'workdir'),
                json_manifest,
            if self.export_bundles:
                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)
                subprocess.check_call([
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak',
                    'build-bundle',
                    self.ostree_repo,
                    manifest['id'],
                    manifest['branch'],
                ])
                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
            pass
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    else:
        logging.basicConfig()

    logging.getLogger().setLevel(logging.DEBUG)
Simon McVittie's avatar
Simon McVittie committed

    try:
        Builder().run_command_line()
    except KeyboardInterrupt:
        raise SystemExit(130)
    except subprocess.CalledProcessError as e:
        logger.error('%s', e)
        raise SystemExit(1)