diff --git a/pressure-vessel/meson.build b/pressure-vessel/meson.build index ea107e4dae80249e3ae02ffa5634e39db71bb7e3..3aff8c68c82c92030b7f69f8e4e3c7898d259053 100644 --- a/pressure-vessel/meson.build +++ b/pressure-vessel/meson.build @@ -57,6 +57,7 @@ endforeach # Headers to scan for enum/flags types. headers = [ + 'mtree.h', 'runtime.h', ] @@ -83,11 +84,13 @@ pressure_vessel_utils = static_library( 'flatpak-utils-base-private.h', 'flatpak-utils.c', 'flatpak-utils-private.h', + 'mtree.c', + 'mtree.h', 'tree-copy.c', 'tree-copy.h', 'utils.c', 'utils.h', - ], + ] + enums, c_args : pv_c_args, dependencies : [ threads, @@ -208,7 +211,7 @@ executable( 'wrap-flatpak.h', 'wrap-setup.c', 'wrap-setup.h', - ] + enums, + ], c_args : pv_c_args, dependencies : [ pressure_vessel_utils_dep, diff --git a/pressure-vessel/mtree.c b/pressure-vessel/mtree.c new file mode 100644 index 0000000000000000000000000000000000000000..2a9a47a10e0365709a043d49b2da6b44fd5c5b36 --- /dev/null +++ b/pressure-vessel/mtree.c @@ -0,0 +1,560 @@ +/* + * Copyright © 2021 Collabora Ltd. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "mtree.h" + +#include "steam-runtime-tools/profiling-internal.h" +#include "steam-runtime-tools/resolve-in-sysroot-internal.h" +#include "steam-runtime-tools/utils-internal.h" + +#include <gio/gunixinputstream.h> + +#include "enumtypes.h" + +/* Enabling debug logging for this is rather too verbose, so only + * enable it when actively debugging this module */ +#if 0 +#define trace(...) g_debug (__VA_ARGS__) +#else +#define trace(...) do { } while (0) +#endif + +gboolean +pv_mtree_entry_parse (const char *line, + PvMtreeEntry *entry, + const char *filename, + guint line_number, + GError **error) +{ + PvMtreeEntry blank = PV_MTREE_ENTRY_BLANK; + g_auto(GStrv) tokens = NULL; + gsize i; + + *entry = blank; + + if (line[0] == '\0' || line[0] == '#') + return TRUE; + + if (line[0] == '/') + return glnx_throw (error, + "%s:%u: Special commands not supported", + filename, line_number); + + if (line[0] != '.' || (line[1] != ' ' && line[1] != '/' && line[1] != '\0')) + return glnx_throw (error, + "%s:%u: Filenames not relative to top level not supported", + filename, line_number); + + if (g_str_has_suffix (line, "\\")) + return glnx_throw (error, + "%s:%u: Continuation lines not supported", + filename, line_number); + + for (i = 0; line[i] != '\0'; i++) + { + if (line[i] == '\\') + { + if (line[i + 1] >= '0' && line[i + 1] <= '9') + continue; + + switch (line[i + 1]) + { + /* g_strcompress() documents these to work */ + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '"': + case '\\': + i += 1; + continue; + + /* \M, \^, \a, \s, \E, \x, \$, escaped whitespace and escaped + * newline are not supported here */ + default: + return glnx_throw (error, + "%s:%u: Unsupported backslash escape: \"\\%c\"", + filename, line_number, line[i + 1]); + } + } + } + + tokens = g_strsplit_set (line, " \t", -1); + + if (tokens == NULL) + return glnx_throw (error, "%s:%u: Line is empty", filename, line_number); + + entry->name = g_strcompress (tokens[0]); + + for (i = 1; tokens[i] != NULL; i++) + { + const char *equals; + char *endptr; + + equals = strchr (tokens[i], '='); + + if (equals == NULL) + return glnx_throw (error, + "%s:%u: No value for keyword '%s'", + filename, line_number, tokens[i]); + + if (g_str_has_prefix (tokens[i], "cksum=") + || g_str_has_prefix (tokens[i], "device=") + || g_str_has_prefix (tokens[i], "flags=") + || g_str_has_prefix (tokens[i], "gid=") + || g_str_has_prefix (tokens[i], "gname=") + || g_str_has_prefix (tokens[i], "ignore=") + || g_str_has_prefix (tokens[i], "inode=") + || g_str_has_prefix (tokens[i], "md5=") + || g_str_has_prefix (tokens[i], "md5digest=") + || g_str_has_prefix (tokens[i], "nlink=") + || g_str_has_prefix (tokens[i], "nochange=") + || g_str_has_prefix (tokens[i], "optional=") + || g_str_has_prefix (tokens[i], "resdevice=") + || g_str_has_prefix (tokens[i], "ripemd160digest=") + || g_str_has_prefix (tokens[i], "rmd160=") + || g_str_has_prefix (tokens[i], "rmd160digest=") + || g_str_has_prefix (tokens[i], "sha1=") + || g_str_has_prefix (tokens[i], "sha1digest=") + || g_str_has_prefix (tokens[i], "sha384=") + || g_str_has_prefix (tokens[i], "sha384digest=") + || g_str_has_prefix (tokens[i], "sha512=") + || g_str_has_prefix (tokens[i], "sha512digest=") + || g_str_has_prefix (tokens[i], "uid=") + || g_str_has_prefix (tokens[i], "uname=")) + continue; + + if (g_str_has_prefix (tokens[i], "link=")) + { + entry->link = g_strcompress (equals + 1); + continue; + } + + if (g_str_has_prefix (tokens[i], "contents=")) + { + entry->contents = g_strcompress (equals + 1); + continue; + } + + if (g_str_has_prefix (tokens[i], "sha256=") + || g_str_has_prefix (tokens[i], "sha256digest=")) + { + if (entry->sha256 == NULL) + entry->sha256 = g_strdup (equals + 1); + else if (strcmp (entry->sha256, equals + 1) != 0) + return glnx_throw (error, + "%s:%u: sha256 and sha256digest not consistent", + filename, line_number); + + continue; + } + + if (g_str_has_prefix (tokens[i], "mode=")) + { + gint64 value = g_ascii_strtoll (equals + 1, &endptr, 8); + + if (equals[1] == '\0' || *endptr != '\0') + return glnx_throw (error, + "%s:%u: Invalid mode %s", + filename, line_number, equals + 1); + + entry->mode = value & 07777; + continue; + } + + if (g_str_has_prefix (tokens[i], "size=")) + { + gint64 value = g_ascii_strtoll (equals + 1, &endptr, 10); + + if (equals[1] == '\0' || *endptr != '\0') + return glnx_throw (error, + "%s:%u: Invalid size %s", + filename, line_number, equals + 1); + + entry->size = value; + continue; + } + + if (g_str_has_prefix (tokens[i], "time=")) + { + gdouble value = g_ascii_strtod (equals + 1, &endptr); + + if (equals[1] == '\0' || *endptr != '\0') + return glnx_throw (error, + "%s:%u: Invalid time %s", + filename, line_number, equals + 1); + + entry->mtime_usec = (value * G_TIME_SPAN_SECOND); + continue; + } + + if (g_str_has_prefix (tokens[i], "type=")) + { + int value; + + if (srt_enum_from_nick (PV_TYPE_MTREE_ENTRY_KIND, equals + 1, + &value, NULL)) + entry->kind = value; + else + entry->kind = PV_MTREE_ENTRY_KIND_UNKNOWN; + + continue; + } + + g_warning ("%s:%u: Unknown mtree keyword %s", + filename, line_number, tokens[i]); + } + + if (entry->kind == PV_MTREE_ENTRY_KIND_UNKNOWN) + return glnx_throw (error, + "%s:%u: Unknown mtree entry type", + filename, line_number); + + if (entry->link != NULL && entry->kind != PV_MTREE_ENTRY_KIND_LINK) + return glnx_throw (error, + "%s:%u: Non-symlink cannot have a symlink target", + filename, line_number); + + if (entry->link == NULL && entry->kind == PV_MTREE_ENTRY_KIND_LINK) + return glnx_throw (error, + "%s:%u: Symlink must have a symlink target", + filename, line_number); + + return TRUE; +} + +/* + * pv_mtree_apply: + * @mtree: (type filename): Path to a mtree(5) manifest + * @sysroot: (type filename): A directory + * @sysroot_fd: A fd opened on @sysroot + * @source_files: (optional): A directory from which files will be + * hard-linked or copied when populating @sysroot. The `content` + * or filename in @mtree is taken to be relative to @source_files. + * @flags: Flags affecting how this is done + * + * Make the container root filesystem @sysroot conform to @mtree. + * + * @mtree must contain a subset of BSD mtree(5) syntax: + * + * - one entry per line + * - no device nodes, fifos, sockets or other special devices + * - strings are escaped using octal (for example \040 for space) + * - filenames other than "." start with "./" + * + * For regular files, we assert that the file exists, set its mtime, + * and set its permissions to either 0644 or 0755. + * + * For directories, we create the directory with 0755 permissions. + * + * For symbolic links, we create the symbolic link if it does not + * already exist. + * + * A suitable mtree file can be created from a tarball or the filesystem + * with `bsdtar(1)` from the `libarchive-tools` Debian package: + * + * |[ + * bsdtar -cf - \ + * --format=mtree \ + * --options "!all,type,link,mode,size,time" \ + * @- < foo.tar.gz + * bsdtar -cf - \ + * --format=mtree \ + * --options "!all,type,link,mode,size,time" \ + * -C files/ . + * ]| + * + * A suitable mtree file can also be created by `mtree(8)` from the + * `netbsd-mtree` Debian package if the filenames happen to be ASCII + * (although this implementation does not support all escaped non-ASCII + * filenames produced by `netbsd-mtree`): + * + * |[ + * mtree -p files -c | mtree -C + * ]| + * + * Because hard links are used whenever possible, the permissions or + * modification time of a source file in @source_files might be modified + * to conform to the @mtree. + * + * Returns: %TRUE on success + */ +gboolean +pv_mtree_apply (const char *mtree, + const char *sysroot, + int sysroot_fd, + const char *source_files, + PvMtreeApplyFlags flags, + GError **error) +{ + glnx_autofd int mtree_fd = -1; + g_autoptr(GInputStream) istream = NULL; + g_autoptr(GDataInputStream) reader = NULL; + g_autoptr(SrtProfilingTimer) timer = NULL; + glnx_autofd int source_files_fd = -1; + guint line_number = 0; + + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + g_return_val_if_fail (mtree != NULL, FALSE); + g_return_val_if_fail (sysroot != NULL, FALSE); + g_return_val_if_fail (sysroot_fd >= 0, FALSE); + + timer = _srt_profiling_start ("Apply %s to %s", mtree, sysroot); + + if (!glnx_openat_rdonly (AT_FDCWD, mtree, TRUE, &mtree_fd, error)) + return FALSE; + + istream = g_unix_input_stream_new (glnx_steal_fd (&mtree_fd), TRUE); + + if (flags & PV_MTREE_APPLY_FLAGS_GZIP) + { + g_autoptr(GInputStream) filter = NULL; + g_autoptr(GZlibDecompressor) decompressor = NULL; + + decompressor = g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP); + filter = g_converter_input_stream_new (istream, G_CONVERTER (decompressor)); + g_clear_object (&istream); + istream = g_object_ref (filter); + } + + reader = g_data_input_stream_new (istream); + g_data_input_stream_set_newline_type (reader, G_DATA_STREAM_NEWLINE_TYPE_LF); + + if (source_files != NULL) + { + if (!glnx_opendirat (AT_FDCWD, source_files, FALSE, &source_files_fd, + error)) + return FALSE; + } + + g_info ("Applying \"%s\" to \"%s\"...", mtree, sysroot); + + while (TRUE) + { + g_autofree gchar *line = NULL; + g_autofree gchar *parent = NULL; + const char *base; + g_autoptr(GError) local_error = NULL; + g_auto(PvMtreeEntry) entry = PV_MTREE_ENTRY_BLANK; + glnx_autofd int parent_fd = -1; + glnx_autofd int fd = -1; + int adjusted_mode; + + line = g_data_input_stream_read_line (reader, NULL, NULL, &local_error); + + if (line == NULL) + { + if (local_error != NULL) + { + g_propagate_prefixed_error (error, g_steal_pointer (&local_error), + "While reading a line from %s: ", + mtree); + return FALSE; + } + else + { + /* End of file, not an error */ + break; + } + } + + g_strstrip (line); + line_number++; + + trace ("line %u: %s", line_number, line); + + if (!pv_mtree_entry_parse (line, &entry, mtree, line_number, error)) + return FALSE; + + if (entry.name == NULL || strcmp (entry.name, ".") == 0) + continue; + + trace ("mtree entry: %s", entry.name); + + parent = g_path_get_dirname (entry.name); + base = glnx_basename (entry.name); + trace ("Creating %s in %s", parent, sysroot); + parent_fd = _srt_resolve_in_sysroot (sysroot_fd, parent, + SRT_RESOLVE_FLAGS_MKDIR_P, + NULL, error); + + if (parent_fd < 0) + return glnx_prefix_error (error, + "Unable to create parent directory for \"%s\" in \"%s\"", + entry.name, sysroot); + + switch (entry.kind) + { + case PV_MTREE_ENTRY_KIND_FILE: + if (entry.size == 0) + { + /* For empty files, we can create it from nothing. */ + fd = TEMP_FAILURE_RETRY (openat (parent_fd, base, + (O_RDWR | O_CLOEXEC | O_NOCTTY + | O_NOFOLLOW | O_CREAT + | O_TRUNC), + 0644)); + + if (fd < 0) + return glnx_throw_errno_prefix (error, + "Unable to open \"%s\" in \"%s\"", + entry.name, sysroot); + } + else if (source_files_fd >= 0) + { + const char *source = entry.contents; + + if (source == NULL) + source = entry.name; + + /* If it already exists, assume it's correct */ + if (glnx_openat_rdonly (parent_fd, base, FALSE, &fd, NULL)) + { + trace ("\"%s\" already exists in \"%s\"", + entry.name, sysroot); + } + /* If we can create a hard link, that's also fine */ + else if (TEMP_FAILURE_RETRY (linkat (source_files_fd, source, + parent_fd, base, 0)) == 0) + { + trace ("Created hard link \"%s\" in \"%s\"", + entry.name, sysroot); + } + /* Or if we can copy it, that's fine too */ + else + { + g_debug ("Could not create hard link \"%s\" from \"%s/%s\" into \"%s\": %s", + entry.name, source_files, source, sysroot, + g_strerror (errno)); + + if (!glnx_file_copy_at (source_files_fd, source, NULL, + parent_fd, base, + GLNX_FILE_COPY_OVERWRITE | GLNX_FILE_COPY_NOCHOWN, + NULL, error)) + return glnx_prefix_error (error, + "Could not create copy \"%s\" from \"%s/%s\" into \"%s\"", + entry.name, source_files, + source, sysroot); + + } + } + + /* For other regular files we just assert that it already exists + * (and is not a symlink). */ + if (fd < 0 + && !glnx_openat_rdonly (parent_fd, base, FALSE, &fd, error)) + return glnx_prefix_error (error, + "Unable to open \"%s\" in \"%s\"", + entry.name, sysroot); + + break; + + case PV_MTREE_ENTRY_KIND_DIR: + /* Create directories on-demand */ + if (!glnx_ensure_dir (parent_fd, base, 0755, error)) + return glnx_prefix_error (error, + "Unable to create directory \"%s\" in \"%s\"", + entry.name, sysroot); + + /* Assert that it is in fact a directory */ + if (!glnx_opendirat (parent_fd, base, FALSE, &fd, error)) + return glnx_prefix_error (error, + "Unable to open directory \"%s\" in \"%s\"", + entry.name, sysroot); + + break; + + case PV_MTREE_ENTRY_KIND_LINK: + { + g_autofree char *target = NULL; + /* Create symlinks on-demand. To be idempotent, don't delete + * an existing symlink. */ + target = glnx_readlinkat_malloc (parent_fd, base, + NULL, NULL); + + if (target == NULL && symlinkat (entry.link, parent_fd, base) != 0) + return glnx_throw_errno_prefix (error, + "Unable to create symlink \"%s\" in \"%s\"", + entry.name, sysroot); + } + break; + + case PV_MTREE_ENTRY_KIND_BLOCK: + case PV_MTREE_ENTRY_KIND_CHAR: + case PV_MTREE_ENTRY_KIND_FIFO: + case PV_MTREE_ENTRY_KIND_SOCKET: + case PV_MTREE_ENTRY_KIND_UNKNOWN: + default: + return glnx_throw (error, + "%s:%u: Special file not supported", + mtree, line_number); + } + + if (entry.kind == PV_MTREE_ENTRY_KIND_DIR + || (entry.mode >= 0 && entry.mode & 0111)) + adjusted_mode = 0755; + else + adjusted_mode = 0644; + + if (fd >= 0 && !glnx_fchmod (fd, adjusted_mode, error)) + { + g_prefix_error (error, + "Unable to set mode of \"%s\" in \"%s\": ", + entry.name, sysroot); + return FALSE; + } + + if (entry.mtime_usec >= 0 && fd >= 0 && entry.kind == PV_MTREE_ENTRY_KIND_FILE) + { + struct timespec times[2] = + { + { .tv_sec = 0, .tv_nsec = UTIME_OMIT }, /* atime */ + { + .tv_sec = entry.mtime_usec / G_TIME_SPAN_SECOND, + .tv_nsec = (entry.mtime_usec % G_TIME_SPAN_SECOND) * 1000 + } /* mtime */ + }; + + if (futimens (fd, times) != 0) + g_warning ("Unable to set mtime of \"%s\" in \"%s\": %s", + entry.name, sysroot, g_strerror (errno)); + } + } + + return TRUE; +} + +/* + * Free the contents of @entry, but not @entry itself. + */ +void +pv_mtree_entry_clear (PvMtreeEntry *entry) +{ + g_clear_pointer (&entry->name, g_free); + g_clear_pointer (&entry->contents, g_free); + g_clear_pointer (&entry->link, g_free); + g_clear_pointer (&entry->sha256, g_free); +} diff --git a/pressure-vessel/mtree.h b/pressure-vessel/mtree.h new file mode 100644 index 0000000000000000000000000000000000000000..19e9a8e44bb00c57b80d3db955e1d9fdda349ff2 --- /dev/null +++ b/pressure-vessel/mtree.h @@ -0,0 +1,85 @@ +/* + * Copyright © 2021 Collabora Ltd. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include <glib.h> + +#include "steam-runtime-tools/glib-backports-internal.h" +#include "libglnx/libglnx.h" + +typedef enum +{ + PV_MTREE_APPLY_FLAGS_GZIP = (1 << 0), + PV_MTREE_APPLY_FLAGS_NONE = 0 +} PvMtreeApplyFlags; + +typedef enum +{ + PV_MTREE_ENTRY_KIND_UNKNOWN = '\0', + PV_MTREE_ENTRY_KIND_BLOCK = 'b', + PV_MTREE_ENTRY_KIND_CHAR = 'c', + PV_MTREE_ENTRY_KIND_DIR = 'd', + PV_MTREE_ENTRY_KIND_FIFO = 'p', + PV_MTREE_ENTRY_KIND_FILE = '-', + PV_MTREE_ENTRY_KIND_LINK = 'l', + PV_MTREE_ENTRY_KIND_SOCKET = 's', +} PvMtreeEntryKind; + +typedef struct _PvMtreeEntry PvMtreeEntry; + +struct _PvMtreeEntry +{ + gchar *name; + gchar *contents; + gchar *link; + gchar *sha256; + goffset size; + GTimeSpan mtime_usec; + int mode; + PvMtreeEntryKind kind; +}; + +#define PV_MTREE_ENTRY_BLANK \ +{ \ + .size = -1, \ + .mtime_usec = -1, \ + .mode = -1, \ +} + +gboolean pv_mtree_entry_parse (const char *line, + PvMtreeEntry *entry, + const char *filename, + guint line_number, + GError **error); + +void pv_mtree_entry_clear (PvMtreeEntry *entry); +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (PvMtreeEntry, pv_mtree_entry_clear) + +gboolean pv_mtree_apply (const char *mtree, + const char *sysroot, + int sysroot_fd, + const char *source_files, + PvMtreeApplyFlags flags, + GError **error); diff --git a/pressure-vessel/runtime.c b/pressure-vessel/runtime.c index 8505c1c74fae51f2b49bccdf4acd61f79fff191e..b7408809674db7ba6292dded59826cbc6faabd36 100644 --- a/pressure-vessel/runtime.c +++ b/pressure-vessel/runtime.c @@ -42,6 +42,7 @@ #include "enumtypes.h" #include "exports.h" #include "flatpak-run-private.h" +#include "mtree.h" #include "tree-copy.h" #include "utils.h" @@ -820,17 +821,10 @@ pv_runtime_garbage_collect (PvRuntime *self, if (g_str_has_prefix (dent->d_name, "deploy-")) { - /* Don't GC old deployments unless we know which one is current - * and therefore should not be deleted */ - if (self->id == NULL) - { - g_debug ("Ignoring %s/deploy-*: current ID not known", - self->variable_dir); - continue; - } - - /* Don't GC the current deployment */ - if (strcmp (dent->d_name + strlen ("deploy-"), self->id) == 0) + if (_srt_fstatat_is_same_file (self->variable_dir_fd, + dent->d_name, + AT_FDCWD, + self->deployment)) { g_debug ("Ignoring %s/%s: is the current version", self->variable_dir, dent->d_name); @@ -875,17 +869,17 @@ pv_runtime_init_variable_dir (PvRuntime *self, static gboolean pv_runtime_create_copy (PvRuntime *self, PvBwrapLock *variable_dir_lock, + const char *usr_mtree, + PvMtreeApplyFlags mtree_flags, GError **error) { g_autofree gchar *dest_usr = NULL; - g_autofree gchar *source_usr_subdir = NULL; g_autofree gchar *temp_dir = NULL; g_autoptr(GDir) dir = NULL; g_autoptr(PvBwrapLock) copy_lock = NULL; g_autoptr(PvBwrapLock) source_lock = NULL; g_autoptr(SrtProfilingTimer) timer = NULL; const char *member; - const char *source_usr; glnx_autofd int temp_dir_fd = -1; gboolean is_just_usr; @@ -906,28 +900,62 @@ pv_runtime_create_copy (PvRuntime *self, "Cannot create temporary directory \"%s\"", temp_dir); - source_usr_subdir = g_build_filename (self->source_files, "usr", NULL); dest_usr = g_build_filename (temp_dir, "usr", NULL); - is_just_usr = !g_file_test (source_usr_subdir, G_FILE_TEST_IS_DIR); + if (usr_mtree != NULL) + { + is_just_usr = TRUE; + } + else + { + g_autofree gchar *source_usr_subdir = g_build_filename (self->source_files, + "usr", NULL); + + is_just_usr = !g_file_test (source_usr_subdir, G_FILE_TEST_IS_DIR); + } if (is_just_usr) { /* ${source_files}/usr does not exist, so assume it's a merged /usr, * for example ./scout/files. Copy ${source_files}/bin to * ${temp_dir}/usr/bin, etc. */ - source_usr = self->source_files; + if (usr_mtree != NULL) + { + /* If there's a manifest available, it's actually quicker to iterate + * through the manifest and use that to populate a new copy of the + * runtime that it would be to do the equivalent of `cp -al` - + * presumably because the mtree is probably contiguous on disk, + * and the nested directories are probably not. */ + glnx_autofd int dest_usr_fd = -1; + + if (!glnx_ensure_dir (AT_FDCWD, dest_usr, 0755, error)) + return FALSE; - if (!pv_cheap_tree_copy (self->source_files, dest_usr, - PV_COPY_FLAGS_NONE, error)) - return FALSE; + if (!glnx_opendirat (AT_FDCWD, dest_usr, FALSE, &dest_usr_fd, error)) + { + g_prefix_error (error, "Unable to open \"%s\": ", dest_usr); + return FALSE; + } + + if (!pv_mtree_apply (usr_mtree, dest_usr, dest_usr_fd, + self->source_files, mtree_flags, + error)) + return FALSE; + } + else + { + /* Fall back to assuming that what's on-disk is correct. */ + if (!pv_cheap_tree_copy (self->source_files, dest_usr, + PV_COPY_FLAGS_NONE, error)) + return FALSE; + } } else { /* ${source_files}/usr exists, so assume it's a complete sysroot. * Merge ${source_files}/bin and ${source_files}/usr/bin into * ${temp_dir}/usr/bin, etc. */ - source_usr = source_usr_subdir; + g_assert (usr_mtree == NULL); if (!pv_cheap_tree_copy (self->source_files, temp_dir, PV_COPY_FLAGS_USRMERGE, error)) @@ -982,7 +1010,7 @@ pv_runtime_create_copy (PvRuntime *self, temp_dir); } - dir = g_dir_open (source_usr, 0, error); + dir = g_dir_open (dest_usr, 0, error); if (dir == NULL) return FALSE; @@ -1471,7 +1499,9 @@ pv_runtime_initable_init (GInitable *initable, g_autoptr(PvBwrapLock) mutable_lock = NULL; g_autofree gchar *contents = NULL; g_autofree gchar *os_release = NULL; + g_autofree gchar *usr_mtree = NULL; gsize len; + PvMtreeApplyFlags mtree_flags = PV_MTREE_APPLY_FLAGS_NONE; g_return_val_if_fail (PV_IS_RUNTIME (self), FALSE); g_return_val_if_fail (error == NULL || *error == NULL, FALSE); @@ -1533,11 +1563,40 @@ pv_runtime_initable_init (GInitable *initable, self->deployment); } - /* If it contains ./files/, assume it's a Flatpak-style runtime where - * ./files is a merged /usr and ./metadata is an optional GKeyFile */ + /* If the deployment contains usr-mtree.txt, assume that it's a + * Flatpak-style merged-/usr runtime, and usr-mtree.txt describes + * what's in the runtime. The content is taken from the files/ + * directory, but files not listed in the mtree are not included. + * + * The manifest compresses well (about 3:1 if sha256sums are included) + * so try to read a compressed version first, falling back to + * uncompressed. */ + usr_mtree = g_build_filename (self->deployment, "usr-mtree.txt.gz", NULL); + + if (g_file_test (usr_mtree, G_FILE_TEST_IS_REGULAR)) + { + mtree_flags |= PV_MTREE_APPLY_FLAGS_GZIP; + } + else + { + g_clear_pointer (&usr_mtree, g_free); + usr_mtree = g_build_filename (self->deployment, "usr-mtree.txt", NULL); + } + + if (!g_file_test (usr_mtree, G_FILE_TEST_IS_REGULAR)) + g_clear_pointer (&usr_mtree, g_free); + + /* Or, if it contains ./files/, assume it's a Flatpak-style runtime where + * ./files is a merged /usr and ./metadata is an optional GKeyFile. */ self->source_files = g_build_filename (self->deployment, "files", NULL); - if (g_file_test (self->source_files, G_FILE_TEST_IS_DIR)) + if (usr_mtree != NULL) + { + g_debug ("Assuming %s is a merged-/usr runtime because it has " + "a /usr mtree", + self->deployment); + } + else if (g_file_test (self->source_files, G_FILE_TEST_IS_DIR)) { g_debug ("Assuming %s is a Flatpak-style runtime", self->deployment); } @@ -1601,6 +1660,10 @@ pv_runtime_initable_init (GInitable *initable, return FALSE; } + /* Always copy the runtime into var/ before applying a manifest. */ + if (usr_mtree != NULL) + self->flags |= PV_RUNTIME_FLAGS_COPY_RUNTIME; + if (self->flags & PV_RUNTIME_FLAGS_COPY_RUNTIME) { if (self->variable_dir_fd < 0) @@ -1620,7 +1683,8 @@ pv_runtime_initable_init (GInitable *initable, if (mutable_lock == NULL) return FALSE; - if (!pv_runtime_create_copy (self, mutable_lock, error)) + if (!pv_runtime_create_copy (self, mutable_lock, usr_mtree, + mtree_flags, error)) return FALSE; } diff --git a/tests/pressure-vessel/containers.py b/tests/pressure-vessel/containers.py index 60a8f3a64bfd669b8fc1dfaf21f5e1d876838f40..cc1261aec62454c12fbb792c101e9d30bc9d343a 100755 --- a/tests/pressure-vessel/containers.py +++ b/tests/pressure-vessel/containers.py @@ -606,6 +606,15 @@ class TestContainers(BaseTest): check=True, ) + # Exercise the code path where we don't have a mtree manifest: + # this is important here because we're editing the runtime + # in-place to have the OLD-DEPLOYMENT flag-file, but the + # manifest doesn't include that, so if we're using a runtime + # with a manifest, that part will fail. + for manifest in ('usr-mtree.txt', 'usr-mtree.txt.gz'): + with contextlib.suppress(FileNotFoundError): + os.remove(os.path.join(old_dir, manifest)) + if os.path.isdir(os.path.join(old_dir, 'files')): old_dir = os.path.join(old_dir, 'files') diff --git a/tests/pressure-vessel/meson.build b/tests/pressure-vessel/meson.build index 792fd8907727f454a05fb37fd85ae17e85cd948f..d7ce3c2830e91f06f38b63b40d948e021e866413 100644 --- a/tests/pressure-vessel/meson.build +++ b/tests/pressure-vessel/meson.build @@ -36,6 +36,7 @@ tests = [ 'containers.py', 'invocation.py', 'launcher.py', + 'mtree-apply.py', 'test-locale-gen.sh', 'utils.py', ] @@ -146,6 +147,7 @@ endforeach helpers = [ 'cheap-copy', 'helper', + 'mtree-apply', ] foreach helper : helpers diff --git a/tests/pressure-vessel/mtree-apply.c b/tests/pressure-vessel/mtree-apply.c new file mode 100644 index 0000000000000000000000000000000000000000..697889d9229e52ecc654a441de2aac5e5d4a2df6 --- /dev/null +++ b/tests/pressure-vessel/mtree-apply.c @@ -0,0 +1,91 @@ +/* + * Copyright © 2021 Collabora Ltd. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "config.h" +#include "subprojects/libglnx/config.h" + +#include <locale.h> +#include <sysexits.h> + +#include "libglnx/libglnx.h" + +#include "steam-runtime-tools/glib-backports-internal.h" +#include "steam-runtime-tools/utils-internal.h" + +#include "mtree.h" +#include "utils.h" + +static GOptionEntry options[] = +{ + { NULL } +}; + +int +main (int argc, + char *argv[]) +{ + glnx_autofd int fd = -1; + g_autoptr(GOptionContext) context = NULL; + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + PvMtreeApplyFlags flags = PV_MTREE_APPLY_FLAGS_NONE; + int ret = EX_USAGE; + + setlocale (LC_ALL, ""); + _srt_setenv_disable_gio_modules (); + + context = g_option_context_new ("MTREE ROOT"); + g_option_context_add_main_entries (context, options, NULL); + + if (!g_option_context_parse (context, &argc, &argv, error)) + goto out; + + if (argc >= 2 && strcmp (argv[1], "--") == 0) + { + argv++; + argc--; + } + + if (argc < 3 || argc > 4) + { + g_printerr ("Usage: %s MTREE ROOT [SOURCE]\n", g_get_prgname ()); + goto out; + } + + ret = EX_UNAVAILABLE; + + if (!glnx_opendirat (AT_FDCWD, argv[2], TRUE, &fd, error)) + goto out; + + if (!pv_mtree_apply (argv[1], argv[2], fd, argv[3], flags, error)) + goto out; + + ret = 0; + +out: + if (local_error != NULL) + g_warning ("%s", local_error->message); + + return ret; +} diff --git a/tests/pressure-vessel/mtree-apply.py b/tests/pressure-vessel/mtree-apply.py new file mode 100755 index 0000000000000000000000000000000000000000..42fe885b51e940cb676f9fbb2fc85c5474a26cc3 --- /dev/null +++ b/tests/pressure-vessel/mtree-apply.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# Copyright 2021 Collabora Ltd. +# +# SPDX-License-Identifier: MIT + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + + +try: + import typing + typing # placate pyflakes +except ImportError: + pass + +from testutils import ( + BaseTest, + test_main, +) + + +class TestMtreeApply(BaseTest): + def setUp(self) -> None: + super().setUp() + os.environ['G_MESSAGES_DEBUG'] = 'all' + self.mtree_apply = os.path.join( + self.G_TEST_BUILDDIR, + 'test-mtree-apply', + ) + + def assert_tree_is_superset( + self, + superset, + subset, + require_hard_links: bool = True, + require_permissions: bool = True, + require_times: bool = True, + ): + for path, dirs, files in os.walk(subset): + equivalent = os.path.join(superset, os.path.relpath(path, subset)) + + for d in dirs: + in_subset = os.path.join(path, d) + in_superset = os.path.join(equivalent, d) + + if not os.path.isdir(in_superset): + raise AssertionError( + '%r should be a directory', in_superset) + + info = os.stat(in_subset) + info2 = os.stat(in_superset) + self.assertEqual(oct(info.st_mode), oct(info2.st_mode)) + + for f in files: + in_subset = os.path.join(path, f) + in_superset = os.path.join(equivalent, f) + + if ( + os.path.islink(in_subset) + or not os.path.exists(in_subset) + ): + target = os.readlink(in_subset) + target2 = os.readlink(in_superset) + self.assertEqual(target, target2) + else: + info = os.stat(in_subset) + info2 = os.stat(in_superset) + + if require_hard_links: + self.assertEqual(info.st_ino, info2.st_ino) + self.assertEqual(info.st_dev, info2.st_dev) + + if require_permissions: + self.assertEqual(oct(info.st_mode), oct(info2.st_mode)) + else: + self.assertEqual( + oct(info.st_mode & ~0o7777), + oct(info2.st_mode & ~0o7777), + ) + + self.assertEqual(info.st_size, info2.st_size) + + if require_times: + self.assertEqual( + int(info.st_mtime), + int(info2.st_mtime), + ) + + def assert_tree_is_same( + self, + left, + right, + require_hard_links: bool = True, + require_permissions: bool = True, + require_times: bool = True, + ): + self.assert_tree_is_superset( + left, right, + require_hard_links=require_hard_links, + require_permissions=require_permissions, + require_times=require_times, + ) + self.assert_tree_is_superset( + right, + left, + require_hard_links=require_hard_links, + require_permissions=require_permissions, + require_times=require_times, + ) + + def test_empty(self) -> None: + content = b'' + + with tempfile.NamedTemporaryFile( + ) as source, tempfile.TemporaryDirectory( + ) as expected, tempfile.TemporaryDirectory( + ) as dest: + source.write(content) + source.flush() + + subprocess.run( + [ + self.mtree_apply, + source.name, + dest, + ], + check=True, + stdout=2, + ) + self.assert_tree_is_same( + dest, + expected, + require_hard_links=False, + require_permissions=False, + require_times=False, + ) + + def test_populate(self) -> None: + content = b'''\ + . type=dir + ./foo/bar/\302\251 type=dir + ./sym/link type=link link=/dev/null + ./create type=file size=0 + ./make-executable type=file mode=755 + ./make-non-executable type=file mode=644 time=1597415889 + ''' + + with tempfile.NamedTemporaryFile( + ) as source, tempfile.TemporaryDirectory( + ) as expected, tempfile.TemporaryDirectory( + ) as dest: + source.write(content) + source.flush() + + os.umask(0o077) + bar = (Path(expected) / 'foo' / 'bar') + bar.mkdir(parents=True) + non_ascii_filename = str(bar).encode('ascii') + b'/\302\251' + os.mkdir(non_ascii_filename) + (Path(expected) / 'sym').mkdir() + (Path(expected) / 'sym' / 'link').symlink_to('/dev/null') + + for filename in 'make-executable', 'make-non-executable': + for d in Path(expected), Path(dest): + with open(str(d / filename), 'w') as writer: + writer.write('#!/bin/sh\n') + + with open(str(Path(expected) / 'create'), 'w') as writer: + pass + + # Paths that are not explicitly created get their permissions + # from umask + (Path(expected)).chmod(0o755) + os.chmod(non_ascii_filename, 0o755) + (Path(expected) / 'create').chmod(0o644) + (Path(expected) / 'make-executable').chmod(0o755) + (Path(expected) / 'make-non-executable').chmod(0o644) + os.utime( + str(Path(expected) / 'make-non-executable'), + times=(1597415889, 1597415889), + ) + + subprocess.run( + [ + self.mtree_apply, + source.name, + dest, + ], + check=True, + stdout=2, + ) + subprocess.run( + [ + 'find', + dest, + '-ls', + ], + check=True, + stdout=2, + ) + self.assert_tree_is_same( + dest, + expected, + require_hard_links=False, + require_permissions=True, + require_times=False, + ) + + def test_populate_copy(self) -> None: + content = b'''\ +# Content-addressed storage indexed by a truncated sha256 +./make-executable type=file mode=755 contents=a8/076d3d28d21e02012b20eaf7dbf754 +./make-non-executable type=file mode=644 time=1597415889 +''' + + with tempfile.NamedTemporaryFile( + ) as source, tempfile.TemporaryDirectory( + ) as reference, tempfile.TemporaryDirectory( + ) as expected, tempfile.TemporaryDirectory( + ) as dest: + source.write(content) + source.flush() + + os.umask(0o077) + + for filename in 'make-executable', 'make-non-executable': + with open(str(Path(expected) / filename), 'w') as writer: + writer.write('#!/bin/sh\n') + + # Avoid Path.link_to() which wasn't in Python 3.5 + os.link( + str(Path(expected) / 'make-non-executable'), + str(Path(reference) / 'make-non-executable'), + ) + (Path(reference) / 'a8').mkdir() + os.link( + str(Path(expected) / 'make-executable'), + str( + Path(reference) / 'a8' + / '076d3d28d21e02012b20eaf7dbf754' + ), + ) + + subprocess.run( + [ + self.mtree_apply, + source.name, + dest, + reference, + ], + check=True, + stdout=2, + ) + + subprocess.run( + [ + 'find', + dest, + '-ls', + ], + check=True, + stdout=2, + ) + self.assert_tree_is_same( + dest, + expected, + require_hard_links=True, + require_permissions=True, + require_times=True, + ) + + info = (Path(dest) / 'make-executable').stat() + self.assertEqual(info.st_mode & 0o7777, 0o755) + + info = (Path(dest) / 'make-non-executable').stat() + self.assertEqual(info.st_mode & 0o7777, 0o644) + self.assertEqual(info.st_mtime, 1597415889) + + def tearDown(self) -> None: + super().tearDown() + + +if __name__ == '__main__': + assert sys.version_info >= (3, 5), \ + 'Python 3.5+ is required (configure with -Dpython=python3.5 ' \ + 'if necessary)' + + test_main() + +# vi: set sw=4 sts=4 et: diff --git a/tests/pressure-vessel/utils.c b/tests/pressure-vessel/utils.c index 202431ff899bd3dabf0c12bc397d158483c76986..0856c8e24fff8591e08ff1cef39297ca3fe04043 100644 --- a/tests/pressure-vessel/utils.c +++ b/tests/pressure-vessel/utils.c @@ -34,6 +34,7 @@ #include "libglnx/libglnx.h" #include "tests/test-utils.h" +#include "mtree.h" #include "utils.h" typedef struct @@ -344,6 +345,125 @@ test_get_path_after (Fixture *f, } } +static void +test_mtree_entry_parse (Fixture *f, + gconstpointer context) +{ + static const struct + { + const char *line; + const char *name; + PvMtreeEntry expected; + gboolean error; + const char *link; + const char *sha256; + } tests[] = + { + { "#mtree", + NULL, + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_UNKNOWN }, + }, + { "", + NULL, + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_UNKNOWN }, + }, + { ". type=dir", + ".", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_DIR }, + }, + { "./foo type=file sha256=ffff mode=0640 size=42 time=1597415889.5", + "./foo", + { .size = 42, + .mtime_usec = 1597415889 * G_TIME_SPAN_SECOND + (G_TIME_SPAN_SECOND / 2), + .mode = 0640, + .kind = PV_MTREE_ENTRY_KIND_FILE }, + .sha256 = "ffff", + }, + { "./foo type=file sha256digest=ffff mode=4755", + "./foo", + { .size = -1, .mtime_usec = -1, .mode = 04755, + .kind = PV_MTREE_ENTRY_KIND_FILE }, + .sha256 = "ffff", + }, + { "./foo type=file sha256=ffff sha256digest=ffff", + "./foo", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_FILE }, + .sha256 = "ffff", + }, + { "./symlink type=link link=/dev/null", + "./symlink", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_LINK }, + .link = "/dev/null", + }, + { "./silly-name/\\001\\123\\n\\r type=link link=\\\"\\\\\\b", + "./silly-name/\001\123\n\r", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_LINK }, + .link = "\"\\\b", + }, + { ("./ignore cksum=123 device=456 contents=./ignore flags=123 gid=123 " + "gname=users ignore=1 inode=123 md5=ffff md5digest=ffff nlink=1 " + "nochange=1 optional=1 resdevice=123 " + "ripemd160digest=ffff rmd160=ffff rmd160digest=ffff " + "sha1=ffff sha1digest=ffff " + "sha384=ffff sha384digest=ffff " + "sha512=ffff sha512digest=ffff " + "uid=0 uname=root type=dir"), + "./ignore", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_DIR }, + }, + { "./foo type=file sha256=ffff sha256digest=eeee", .error = TRUE }, + { "./foo type=file mode=1a", .error = TRUE }, + { "/set type=dir", .error = TRUE }, + { "../escape type=dir", .error = TRUE }, + { "relative type=dir", .error = TRUE }, + { "./foo link", .error = TRUE }, + { "./foo type=bar", .error = TRUE }, + { "./continuation type=dir \\", .error = TRUE }, + { "./alert type=link link=\\a", .error = TRUE }, + { "./hex type=link link=\\x12", .error = TRUE }, + { "./symlink type=file link=/dev/null", .error = TRUE }, + { "./symlink type=link", .error = TRUE }, + { " ", .error = TRUE }, + }; + gsize i; + + for (i = 0; i < G_N_ELEMENTS (tests); i++) + { + const char *line = tests[i].line; + PvMtreeEntry expected = tests[i].expected; + g_auto(PvMtreeEntry) got = PV_MTREE_ENTRY_BLANK; + g_autoptr(GError) error = NULL; + + g_test_message ("%s", line); + pv_mtree_entry_parse (line, &got, "test.mtree", 1, &error); + + if (tests[i].error) + { + g_assert_nonnull (error); + g_test_message ("-> %s", error->message); + } + else + { + g_assert_no_error (error); + g_test_message ("-> OK"); + g_assert_cmpstr (got.name, ==, tests[i].name); + g_assert_cmpstr (got.link, ==, tests[i].link); + g_assert_cmpstr (got.sha256, ==, tests[i].sha256); + g_assert_cmpint (got.size, ==, expected.size); + g_assert_cmpint (got.mtime_usec, ==, expected.mtime_usec); + g_assert_cmpint (got.mode, ==, expected.mode); + g_assert_cmpint (got.kind, ==, expected.kind); + } + } +} + static void test_search_path_append (Fixture *f, gconstpointer context) @@ -389,6 +509,8 @@ main (int argc, g_test_add ("/envp-cmp", Fixture, NULL, setup, test_envp_cmp, teardown); g_test_add ("/get-path-after", Fixture, NULL, setup, test_get_path_after, teardown); + g_test_add ("/mtree-entry-parse", Fixture, NULL, + setup, test_mtree_entry_parse, teardown); g_test_add ("/search-path-append", Fixture, NULL, setup, test_search_path_append, teardown);