Skip to content
Snippets Groups Projects
run.py 48.4 KiB
Newer Older
            search_path.append('/app/lib/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/extensions/gstreamer-1.0')
            for arch in self.dpkg_archs:
                search_path.append(
                    '/usr/lib/{}/gstreamer-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/usr/lib/gstreamer-1.0')
Simon McVittie's avatar
Simon McVittie committed

            keyfile.set_string(
                'Environment', 'GST_PLUGIN_SYSTEM_PATH',
                ':'.join(search_path),
            )
        if True:    # TODO: 'libgirepository-1.0-1' in installed:
            search_path = []
            for arch in self.dpkg_archs:
                search_path.append(
                    '/app/lib/{}/girepository-1.0'.format(
                        self.multiarch_tuple(arch)))
            search_path.append('/app/lib/girepository-1.0')
Simon McVittie's avatar
Simon McVittie committed

                'Environment', 'GI_TYPELIB_PATH',
                ':'.join(search_path),
        keyfile.set_string(
            'Runtime', 'x-flatdeb-version', VERSION,
        )
        for ext, detail in self.runtime_details.get(
                'add-extensions', {}
                ).items():
            group = 'Extension {}'.format(ext)
            os.makedirs(
                os.path.join(
                    overlay, 'ostree', 'main', 'files',
                    detail['directory'],
                ),
                0o755,
                exist_ok=True,
            )
            for k, v in detail.items():
                if isinstance(v, str):
                    keyfile.set_string(group, k, v)
                elif isinstance(v, bool):
                    keyfile.set_boolean(group, k, v)
                else:
                    raise RuntimeError(
                        'Unknown type {} in {}'.format(v, ext))
Simon McVittie's avatar
Simon McVittie committed

        keyfile.save_to_file(metadata)
Simon McVittie's avatar
Simon McVittie committed

            metadata = os.path.join(overlay, 'ostree', 'source', 'metadata')
            os.makedirs(os.path.dirname(metadata), 0o755, exist_ok=True)
            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime + '.Sources')
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                )
            )
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )
            keyfile.save_to_file(metadata)
    def command_app(self, *, app_branch, yaml_manifest, **kwargs):
        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed

        with open(yaml_manifest, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            # We assume the build area has xattr support
            self.ensure_build_area()
            self.ensure_local_repo()
            scratch = stack.enter_context(
                TemporaryDirectory(
                    prefix='flatdeb.',
                    dir=os.path.join(self.build_area, 'tmp'),
                )
            )
            os.makedirs(os.path.join(scratch, 'home'), 0o755, exist_ok=True)
            subprocess.check_call([
                'XDG_DATA_HOME={}/home'.format(scratch),
                'flatpak', '--user',
                'remote-add', '--if-not-exists', '--no-gpg-verify',
                'file://{}'.format(urllib.parse.quote(self.ostree_repo)),
Simon McVittie's avatar
Simon McVittie committed
                'env',
                'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                'flatpak', '--user',
                'remote-modify', '--no-gpg-verify',
                '--url=file://{}'.format(urllib.parse.quote(self.ostree_repo)),
Simon McVittie's avatar
Simon McVittie committed
            ])

            for runtime in (manifest['sdk'], manifest['runtime']):
                # This may fail: we might already have it.
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'install', '-y', 'flatdeb',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak', '--user',
                    'update',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
Simon McVittie's avatar
Simon McVittie committed

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-git.',
                                        dir=scratch,
                                    ),
                                )
                                uploader = subprocess.Popen([
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
Simon McVittie's avatar
Simon McVittie committed
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = stack.enter_context(
                                    TemporaryDirectory(
                                        prefix='flatdeb-path.',
                                        dir=scratch,
                                    ),
                                )
                                clone = os.path.join(
Simon McVittie's avatar
Simon McVittie committed
                                    d, os.path.basename(source['path']),
                                )
Simon McVittie's avatar
Simon McVittie committed

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    os.chmod(clone, 0o755)
                                else:
                                    os.chmod(clone, 0o644)
Simon McVittie's avatar
Simon McVittie committed

                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = stack.enter_context(
                            TemporaryDirectory(
                                prefix='flatdeb-debs.',
                                dir=scratch,
                            ),
                        )
                        subprocess.check_call([
Simon McVittie's avatar
Simon McVittie committed
                            'env',
                            'XDG_DATA_HOME={}/home'.format(scratch),
Simon McVittie's avatar
Simon McVittie committed
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'DEBIAN_FRONTEND=noninteractive',
Simon McVittie's avatar
Simon McVittie committed
                            'export={}'.format(packages),
                            'sh',
                            '-euc',

                            'cp -PRp /usr/var /\n'
Simon McVittie's avatar
Simon McVittie committed
                            'install -d /var/cache/apt/archives/partial\n'
                            'fakeroot apt-get update\n'
                            'fakeroot apt-get -y --download-only \\\n'
                            '    --no-install-recommends install "$@"\n'
                            'for x in /var/cache/apt/archives/*.deb; do\n'
                            '    package="$(dpkg-deb -f "$x" Package)"\n'
                            '    source="$(dpkg-deb -f "$x" Source)"\n'
                            '    bu="$(dpkg-deb -f "$x" Built-Using)"\n'
                            '    version="$(dpkg-deb -f "$x" Version)"\n'
                            '    if [ -z "$source" ]; then\n'
                            '        source="$package"\n'
                            '    fi\n'
                            '    if [ "${source% (*}" != "$source" ]; then\n'
                            '        version="${source#* (}"\n'
                            '        version="${version%)}"\n'
                            '        source="${source% (*}"\n'
                            '    fi\n'
                            '    ( cd "$export" && \\\n'
                            '         apt-get -y --download-only \\\n'
                            '         -oAPT::Get::Only-Source=true source \\\n'
                            '         "$source=$version"\n'
                            '    )\n'
                            '    if [ -n "$bu" ]; then\n'
                            '        oldIFS="$IFS"\n'
                            '        IFS=","\n'
                            '        for dep in $bu; do\n'
                            '            bu="$(echo "$bu" | tr -d " ")"\n'
                            '            version="${bu#*(=}"\n'
                            '            version="${version%)}"\n'
                            '            source="${bu%(*}"\n'
                            '            ( cd "$export" && \\\n'
                            '               apt-get -y --download-only \\\n'
                            '               -oAPT::Get::Only-Source=true \\\n'
                            '               source "$source=$version"\n'
Simon McVittie's avatar
Simon McVittie committed
                            'mv /var/cache/apt/archives/*.deb "$export"\n'
                            'mv /var/lib/apt/lists "$export"\n'
                            '',

                            'sh',   # argv[0]
                        ] + module['x-flatdeb-apt-packages'])

                        obtained = subprocess.check_output([
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').splitlines()

                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
                            path = '{}/{}'.format(packages, f)

                            sources.append({
                                'dest': (os.path.dirname(f) or '.'),
                                'type': 'file',
                                'sha256': sha256,
                                'url': urllib.parse.urlunsplit((
                                    'file',
                                    '',
                                    urllib.parse.quote(path),
                                    '',
                                    '',
                                ))
                            })
Simon McVittie's avatar
Simon McVittie committed

            json_manifest = os.path.join(scratch, manifest['id'] + '.json')
            os.makedirs(
                os.path.join(self.build_area, '.flatpak-builder'),
                exist_ok=True,
            )
Simon McVittie's avatar
Simon McVittie committed

            if self.build_area != scratch:
                subprocess.check_call([
                    os.path.join(self.build_area, '.flatpak-builder'),
                    '{}/'.format(scratch),
            with open(json_manifest, 'w', encoding='utf-8') as writer:
                json.dump(manifest, writer, indent=2, sort_keys=True)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
                'env',
                'DEBIAN_FRONTEND=noninteractive',
                'XDG_DATA_HOME={}/home'.format(scratch),
                'sh', '-euc',
                'cd "$1"; shift; exec "$@"',
                'sh',                   # argv[0]
                scratch,                # directory to cd into
Simon McVittie's avatar
Simon McVittie committed
                'flatpak-builder',
                '--arch={}'.format(self.flatpak_arch),
                '--repo={}'.format(self.ostree_repo),
                os.path.join(scratch, 'workdir'),
                json_manifest,
            if self.export_bundles:
                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)
                subprocess.check_call([
                    'XDG_DATA_HOME={}/home'.format(scratch),
                    'flatpak',
                    'build-bundle',
                    self.ostree_repo,
                    manifest['id'],
                    manifest['branch'],
                ])
                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

Simon McVittie's avatar
Simon McVittie committed
if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
            pass
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    else:
        logging.basicConfig()

    logging.getLogger().setLevel(logging.DEBUG)
Simon McVittie's avatar
Simon McVittie committed

    try:
        Builder().run_command_line()
    except KeyboardInterrupt:
        raise SystemExit(130)
    except subprocess.CalledProcessError as e:
        logger.error('%s', e)
        raise SystemExit(1)