Skip to content
Snippets Groups Projects
run.py 51.1 KiB
Newer Older
Simon McVittie's avatar
Simon McVittie committed
        # TODO: SDK extension

        self.root_worker.check_call([
            'install', '-d', '{}/ostree/main'.format(chroot),
        ])
        self.root_worker.check_call([
            'mv', '{}/usr'.format(chroot),
            '{}/ostree/main/files'.format(chroot),
        ])

        ref = 'runtime/{}/{}/{}'.format(
            runtime, self.flatpak_arch, self.runtime_branch,
        with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
Simon McVittie's avatar
Simon McVittie committed
            metadata = os.path.join(t, 'metadata')

            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime)
            keyfile.set_string(
                'Runtime', 'runtime',
                '{}.Platform/{}/{}\n'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
                '{}.Sdk/{}/{}\n'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                )
            )

            keyfile.set_string(
                'Environment', 'XDG_DATA_DIRS',
                ':'.join([
                    '/app/share', '/usr/share', '/usr/share/runtime/share',
                ]),
            )

            if 'libgstreamer1.0-0' in installed:
                keyfile.set_string(
                    'Environment', 'GST_PLUGIN_SYSTEM_PATH',
                    ':'.join([
                        '/app/lib/gstreamer-1.0',
                        '/usr/lib/extensions/gstreamer-1.0',
                        '/usr/lib/gstreamer-1.0',
                    ]),
                )

            if 'libgirepository-1.0-1' in installed:
                keyfile.set_string(
                    'Environment', 'GI_TYPELIB_PATH',
                    ':'.join([
                        '/app/lib/girepository-1.0',
                    ]),
                )

            keyfile.save_to_file(metadata)

            self.root_worker.install_file(
                metadata,
                '{}/ostree/main/metadata'.format(chroot),
            )

        tarball = '{}-ostree-{}-{}.tar.gz'.format(
            runtime,
            self.flatpak_arch,
            self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
        )

        self.root_worker.check_call([
            'tar', '-zcf',
            '{}/{}'.format(
                self.remote_build_area,
                tarball,
            ),
            '-C', '{}/ostree/main'.format(chroot),
            '.',
        ])

        self.worker.check_call([
            'time',
            'ostree',
            '--repo=' + self.remote_repo,
            'commit',
            '--branch=' + ref,
            '--subject=Update',
            '--tree=tar={}/{}'.format(self.remote_build_area, tarball),
            '--fsync=false',
        ])

        # Don't keep the history in this working repository:
        # if history is desired, mirror the commits into a public
        # repository and maintain history there.
        self.worker.check_call([
            'time',
            'ostree',
            '--repo=' + self.remote_repo,
            'prune',
            '--refs-only',
            '--depth=1',
        ])

        if not isinstance(self.worker, HostWorker):
            self.worker.check_call([
                'time',
                'flatpak',
                'build-update-repo',
                self.remote_repo,
            ])

            with self.worker.remote_dir_context(self.remote_repo) as mount:
                subprocess.call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'remote',
                    'delete',
                    'flatdeb-worker',
                ])
                print('^ It is OK if that failed with "remote not found"')
                subprocess.check_call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'remote',
                    'add',
                    '--no-gpg-verify',
                    'flatdeb-worker',
                    'file://' + urllib.parse.quote(mount),
                ])
                subprocess.check_call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'pull',
                    '--depth=1',
                    '--disable-fsync',
                    '--mirror',
                    '--untrusted',
                    'flatdeb-worker',
                    'runtime/{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                subprocess.check_call([
                    'ostree',
                    '--repo={}'.format(self.repo),
                    'remote',
                    'delete',
                    'flatdeb-worker',
                ])

Simon McVittie's avatar
Simon McVittie committed
            output = os.path.join(self.build_area, tarball)

            with open(output + '.new', 'wb') as writer:
                self.worker.check_call([
                    'cat',
                    '{}/{}'.format(self.remote_build_area, tarball),
                ], stdout=writer)

            os.rename(output + '.new', output)

    def command_app(self, *, app_branch, prefix, **kwargs):
        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed
        self.ensure_remote_repo()

        # Be nice to people using tab-completion
        if prefix.endswith('.yaml'):
            prefix = prefix[:-5]

        with open(prefix + '.yaml') as reader:
            manifest = yaml.safe_load(reader)

        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

Simon McVittie's avatar
Simon McVittie committed
        with ExitStack() as stack:
            stack.enter_context(self.worker)
            t = stack.enter_context(
                TemporaryDirectory(prefix='flatpak-app.')
            )
Simon McVittie's avatar
Simon McVittie committed

            self.worker.check_call([
                'mkdir', '-p', '{}/home'.format(self.worker.scratch),
            ])

            if not isinstance(self.worker, HostWorker):
                with self.worker.remote_dir_context(self.remote_repo) as mount:
                    subprocess.call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'remote',
                        'delete',
                        'flatdeb-host',
                    ])
                    print('^ It is OK if that failed with "remote not found"')
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'remote',
                        'add',
                        '--no-gpg-verify',
                        'flatdeb-host',
                        'file://' + urllib.parse.quote(self.repo),
                    ])
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'pull',
                        '--depth=1',
                        '--disable-fsync',
                        '--mirror',
                        'flatdeb-host',
                        'runtime/{}/{}/{}'.format(
                            manifest['sdk'],
                            self.flatpak_arch,
                            manifest['runtime-version'],
                        ),
                    ])
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'pull',
                        '--depth=1',
                        '--disable-fsync',
                        '--mirror',
                        'flatdeb-host',
                        'runtime/{}/{}/{}'.format(
                            manifest['runtime'],
                            self.flatpak_arch,
                            manifest['runtime-version'],
                        ),
                    ])
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(mount),
                        'remote',
                        'delete',
                        'flatdeb-host',
                    ])

Simon McVittie's avatar
Simon McVittie committed
            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                'flatpak', '--user',
                'remote-add', '--no-gpg-verify',
                'flatdeb', '{}'.format(self.remote_repo),
            ])
            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                'flatpak', '--user',
                'install', 'flatdeb',
                '{}/{}/{}'.format(
                    manifest['sdk'],
                    self.flatpak_arch,
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                ),
            ])
            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                'flatpak', '--user',
                'install', 'flatdeb',
                '{}/{}/{}'.format(
                    manifest['runtime'],
                    self.flatpak_arch,
                    manifest['runtime-version'],
                ),
            ])

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = self.worker.check_output([
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-git.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                                ]).decode('utf-8').rstrip('\n')
                                uploader = subprocess.Popen([
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
                                self.worker.check_call([
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = self.worker.check_output([
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-path.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                                ]).decode('utf-8').rstrip('\n')
                                clone = '{}/{}'.format(
                                    d, os.path.basename(source['path']),
                                )

                                permissions = 0o644

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    permissions = 0o755

                                self.worker.install_file(
                                    source['path'],
                                    clone,
                                    permissions,
                                )
                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = self.worker.check_output([
                            'mktemp', '-d',
                            '-p', self.worker.scratch,
                            'flatdeb-debs.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
                        ]).decode('utf-8').rstrip('\n')

                        self.worker.check_call([
                            'env',
                            'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
                            ),
                            'http_proxy=http://192.168.122.1:3142',
                            'export={}'.format(packages),
                            'sh',
                            '-euc',

                            'cp -a /usr/var /\n'
                            'install -d /var/cache/apt/archives/partial\n'
                            'fakeroot apt-get update\n'
                            'fakeroot apt-get -y --download-only install "$@"\n'
                            'mv /var/cache/apt/archives/*.deb "$export"\n'
                            'mv /var/lib/apt/lists "$export"\n'
                            '',

                            'sh',   # argv[0]
                        ] + module['x-flatdeb-apt-packages'])

                        obtained = self.worker.check_output([
                            'ls', packages,
                        ]).decode('utf-8').splitlines()

                        for f in obtained:
                            path = '{}/{}'.format(packages, f)

                            if f.endswith('.deb'):
                                sources.append({
                                    'dest': '.',
                                    'type': 'file',
                                    'path': path,
                                })

            remote_manifest = '{}/{}.json'.format(self.worker.scratch, prefix)

            with TemporaryDirectory(prefix='flatdeb-manifest.') as t:
Simon McVittie's avatar
Simon McVittie committed
                json_manifest = os.path.join(t, prefix + '.json')

                with open(
                        json_manifest, 'w', encoding='utf-8',
                ) as writer:
                    json.dump(manifest, writer, indent=2, sort_keys=True)

                self.worker.install_file(json_manifest, remote_manifest)

            self.worker.check_call([
                'env',
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                'flatpak-builder',
                '--repo={}'.format(self.remote_repo),
                '{}/workdir'.format(self.worker.scratch),
                remote_manifest,
            ])

            if not isinstance(self.worker, HostWorker):
                self.worker.check_call([
                    'time',
                    'flatpak',
                    'build-update-repo',
                    self.remote_repo,
                ])

                with self.worker.remote_dir_context(self.remote_repo) as mount:
                    subprocess.call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'remote',
                        'delete',
                        'flatdeb-worker',
                    ])
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'remote',
                        'add',
                        '--no-gpg-verify',
                        'flatdeb-worker',
                        'file://' + urllib.parse.quote(mount),
                    ])
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'pull',
                        '--depth=1',
                        '--disable-fsync',
                        '--mirror',
                        '--untrusted',
                        'app/{}/{}/{}'.format(
                            manifest['id'],
                            self.flatpak_arch,
                            manifest['branch'],
                        ),
                    ])
                    subprocess.check_call([
                        'ostree',
                        '--repo={}'.format(self.repo),
                        'remote',
                        'delete',
                        'flatdeb-worker',
                    ])

            if self.export_bundles:
Simon McVittie's avatar
Simon McVittie committed
                self.worker.check_call([
                    'time',
                    'env',
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
                    'flatpak',
                    'build-bundle',
                    self.remote_repo,
Simon McVittie's avatar
Simon McVittie committed
                    '{}/bundle'.format(self.worker.scratch),
                    manifest['id'],
                    manifest['branch'],
                ])
Simon McVittie's avatar
Simon McVittie committed

                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)

                with open(output + '.new', 'wb') as writer:
                    self.worker.check_call([
                        'cat',
                        '{}/bundle'.format(self.worker.scratch),
                    ], stdout=writer)

                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed

if __name__ == '__main__':
    Builder().run_command_line()