From 1bdf777da87d4ecd9686bad3353a7bb862d5d77b Mon Sep 17 00:00:00 2001 From: Simon McVittie <smcv@collabora.com> Date: Mon, 19 Apr 2021 15:16:56 +0100 Subject: [PATCH] pressure-vessel: Add code to parse and apply a mtree(5) manifest This can be used to create directories, symlinks and zero-byte files, create files as a clone (hard-link or copy) of files in a reference directory, assert that files exist, set executable/non-executable permissions, and set files' modification times. When cloning files from a reference directory, the default is to use the same filename as the destination, but that can be overridden (for example to deal with inconvenient/non-Windows-friendly filenames). This is also actually slightly faster than pv_cheap_tree_copy(), presumably because the manifest is contiguous on-disk and the directories are not necessarily. Setting directories' modification times is implemented, but probably won't work in practice, because any files we create in the directory will alter its mtime. Signed-off-by: Simon McVittie <smcv@collabora.com> --- pressure-vessel/meson.build | 7 +- pressure-vessel/mtree.c | 560 +++++++++++++++++++++++++++ pressure-vessel/mtree.h | 85 ++++ tests/pressure-vessel/meson.build | 2 + tests/pressure-vessel/mtree-apply.c | 91 +++++ tests/pressure-vessel/mtree-apply.py | 293 ++++++++++++++ tests/pressure-vessel/utils.c | 122 ++++++ 7 files changed, 1158 insertions(+), 2 deletions(-) create mode 100644 pressure-vessel/mtree.c create mode 100644 pressure-vessel/mtree.h create mode 100644 tests/pressure-vessel/mtree-apply.c create mode 100755 tests/pressure-vessel/mtree-apply.py diff --git a/pressure-vessel/meson.build b/pressure-vessel/meson.build index ea107e4da..3aff8c68c 100644 --- a/pressure-vessel/meson.build +++ b/pressure-vessel/meson.build @@ -57,6 +57,7 @@ endforeach # Headers to scan for enum/flags types. headers = [ + 'mtree.h', 'runtime.h', ] @@ -83,11 +84,13 @@ pressure_vessel_utils = static_library( 'flatpak-utils-base-private.h', 'flatpak-utils.c', 'flatpak-utils-private.h', + 'mtree.c', + 'mtree.h', 'tree-copy.c', 'tree-copy.h', 'utils.c', 'utils.h', - ], + ] + enums, c_args : pv_c_args, dependencies : [ threads, @@ -208,7 +211,7 @@ executable( 'wrap-flatpak.h', 'wrap-setup.c', 'wrap-setup.h', - ] + enums, + ], c_args : pv_c_args, dependencies : [ pressure_vessel_utils_dep, diff --git a/pressure-vessel/mtree.c b/pressure-vessel/mtree.c new file mode 100644 index 000000000..2a9a47a10 --- /dev/null +++ b/pressure-vessel/mtree.c @@ -0,0 +1,560 @@ +/* + * Copyright © 2021 Collabora Ltd. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "mtree.h" + +#include "steam-runtime-tools/profiling-internal.h" +#include "steam-runtime-tools/resolve-in-sysroot-internal.h" +#include "steam-runtime-tools/utils-internal.h" + +#include <gio/gunixinputstream.h> + +#include "enumtypes.h" + +/* Enabling debug logging for this is rather too verbose, so only + * enable it when actively debugging this module */ +#if 0 +#define trace(...) g_debug (__VA_ARGS__) +#else +#define trace(...) do { } while (0) +#endif + +gboolean +pv_mtree_entry_parse (const char *line, + PvMtreeEntry *entry, + const char *filename, + guint line_number, + GError **error) +{ + PvMtreeEntry blank = PV_MTREE_ENTRY_BLANK; + g_auto(GStrv) tokens = NULL; + gsize i; + + *entry = blank; + + if (line[0] == '\0' || line[0] == '#') + return TRUE; + + if (line[0] == '/') + return glnx_throw (error, + "%s:%u: Special commands not supported", + filename, line_number); + + if (line[0] != '.' || (line[1] != ' ' && line[1] != '/' && line[1] != '\0')) + return glnx_throw (error, + "%s:%u: Filenames not relative to top level not supported", + filename, line_number); + + if (g_str_has_suffix (line, "\\")) + return glnx_throw (error, + "%s:%u: Continuation lines not supported", + filename, line_number); + + for (i = 0; line[i] != '\0'; i++) + { + if (line[i] == '\\') + { + if (line[i + 1] >= '0' && line[i + 1] <= '9') + continue; + + switch (line[i + 1]) + { + /* g_strcompress() documents these to work */ + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': + case '"': + case '\\': + i += 1; + continue; + + /* \M, \^, \a, \s, \E, \x, \$, escaped whitespace and escaped + * newline are not supported here */ + default: + return glnx_throw (error, + "%s:%u: Unsupported backslash escape: \"\\%c\"", + filename, line_number, line[i + 1]); + } + } + } + + tokens = g_strsplit_set (line, " \t", -1); + + if (tokens == NULL) + return glnx_throw (error, "%s:%u: Line is empty", filename, line_number); + + entry->name = g_strcompress (tokens[0]); + + for (i = 1; tokens[i] != NULL; i++) + { + const char *equals; + char *endptr; + + equals = strchr (tokens[i], '='); + + if (equals == NULL) + return glnx_throw (error, + "%s:%u: No value for keyword '%s'", + filename, line_number, tokens[i]); + + if (g_str_has_prefix (tokens[i], "cksum=") + || g_str_has_prefix (tokens[i], "device=") + || g_str_has_prefix (tokens[i], "flags=") + || g_str_has_prefix (tokens[i], "gid=") + || g_str_has_prefix (tokens[i], "gname=") + || g_str_has_prefix (tokens[i], "ignore=") + || g_str_has_prefix (tokens[i], "inode=") + || g_str_has_prefix (tokens[i], "md5=") + || g_str_has_prefix (tokens[i], "md5digest=") + || g_str_has_prefix (tokens[i], "nlink=") + || g_str_has_prefix (tokens[i], "nochange=") + || g_str_has_prefix (tokens[i], "optional=") + || g_str_has_prefix (tokens[i], "resdevice=") + || g_str_has_prefix (tokens[i], "ripemd160digest=") + || g_str_has_prefix (tokens[i], "rmd160=") + || g_str_has_prefix (tokens[i], "rmd160digest=") + || g_str_has_prefix (tokens[i], "sha1=") + || g_str_has_prefix (tokens[i], "sha1digest=") + || g_str_has_prefix (tokens[i], "sha384=") + || g_str_has_prefix (tokens[i], "sha384digest=") + || g_str_has_prefix (tokens[i], "sha512=") + || g_str_has_prefix (tokens[i], "sha512digest=") + || g_str_has_prefix (tokens[i], "uid=") + || g_str_has_prefix (tokens[i], "uname=")) + continue; + + if (g_str_has_prefix (tokens[i], "link=")) + { + entry->link = g_strcompress (equals + 1); + continue; + } + + if (g_str_has_prefix (tokens[i], "contents=")) + { + entry->contents = g_strcompress (equals + 1); + continue; + } + + if (g_str_has_prefix (tokens[i], "sha256=") + || g_str_has_prefix (tokens[i], "sha256digest=")) + { + if (entry->sha256 == NULL) + entry->sha256 = g_strdup (equals + 1); + else if (strcmp (entry->sha256, equals + 1) != 0) + return glnx_throw (error, + "%s:%u: sha256 and sha256digest not consistent", + filename, line_number); + + continue; + } + + if (g_str_has_prefix (tokens[i], "mode=")) + { + gint64 value = g_ascii_strtoll (equals + 1, &endptr, 8); + + if (equals[1] == '\0' || *endptr != '\0') + return glnx_throw (error, + "%s:%u: Invalid mode %s", + filename, line_number, equals + 1); + + entry->mode = value & 07777; + continue; + } + + if (g_str_has_prefix (tokens[i], "size=")) + { + gint64 value = g_ascii_strtoll (equals + 1, &endptr, 10); + + if (equals[1] == '\0' || *endptr != '\0') + return glnx_throw (error, + "%s:%u: Invalid size %s", + filename, line_number, equals + 1); + + entry->size = value; + continue; + } + + if (g_str_has_prefix (tokens[i], "time=")) + { + gdouble value = g_ascii_strtod (equals + 1, &endptr); + + if (equals[1] == '\0' || *endptr != '\0') + return glnx_throw (error, + "%s:%u: Invalid time %s", + filename, line_number, equals + 1); + + entry->mtime_usec = (value * G_TIME_SPAN_SECOND); + continue; + } + + if (g_str_has_prefix (tokens[i], "type=")) + { + int value; + + if (srt_enum_from_nick (PV_TYPE_MTREE_ENTRY_KIND, equals + 1, + &value, NULL)) + entry->kind = value; + else + entry->kind = PV_MTREE_ENTRY_KIND_UNKNOWN; + + continue; + } + + g_warning ("%s:%u: Unknown mtree keyword %s", + filename, line_number, tokens[i]); + } + + if (entry->kind == PV_MTREE_ENTRY_KIND_UNKNOWN) + return glnx_throw (error, + "%s:%u: Unknown mtree entry type", + filename, line_number); + + if (entry->link != NULL && entry->kind != PV_MTREE_ENTRY_KIND_LINK) + return glnx_throw (error, + "%s:%u: Non-symlink cannot have a symlink target", + filename, line_number); + + if (entry->link == NULL && entry->kind == PV_MTREE_ENTRY_KIND_LINK) + return glnx_throw (error, + "%s:%u: Symlink must have a symlink target", + filename, line_number); + + return TRUE; +} + +/* + * pv_mtree_apply: + * @mtree: (type filename): Path to a mtree(5) manifest + * @sysroot: (type filename): A directory + * @sysroot_fd: A fd opened on @sysroot + * @source_files: (optional): A directory from which files will be + * hard-linked or copied when populating @sysroot. The `content` + * or filename in @mtree is taken to be relative to @source_files. + * @flags: Flags affecting how this is done + * + * Make the container root filesystem @sysroot conform to @mtree. + * + * @mtree must contain a subset of BSD mtree(5) syntax: + * + * - one entry per line + * - no device nodes, fifos, sockets or other special devices + * - strings are escaped using octal (for example \040 for space) + * - filenames other than "." start with "./" + * + * For regular files, we assert that the file exists, set its mtime, + * and set its permissions to either 0644 or 0755. + * + * For directories, we create the directory with 0755 permissions. + * + * For symbolic links, we create the symbolic link if it does not + * already exist. + * + * A suitable mtree file can be created from a tarball or the filesystem + * with `bsdtar(1)` from the `libarchive-tools` Debian package: + * + * |[ + * bsdtar -cf - \ + * --format=mtree \ + * --options "!all,type,link,mode,size,time" \ + * @- < foo.tar.gz + * bsdtar -cf - \ + * --format=mtree \ + * --options "!all,type,link,mode,size,time" \ + * -C files/ . + * ]| + * + * A suitable mtree file can also be created by `mtree(8)` from the + * `netbsd-mtree` Debian package if the filenames happen to be ASCII + * (although this implementation does not support all escaped non-ASCII + * filenames produced by `netbsd-mtree`): + * + * |[ + * mtree -p files -c | mtree -C + * ]| + * + * Because hard links are used whenever possible, the permissions or + * modification time of a source file in @source_files might be modified + * to conform to the @mtree. + * + * Returns: %TRUE on success + */ +gboolean +pv_mtree_apply (const char *mtree, + const char *sysroot, + int sysroot_fd, + const char *source_files, + PvMtreeApplyFlags flags, + GError **error) +{ + glnx_autofd int mtree_fd = -1; + g_autoptr(GInputStream) istream = NULL; + g_autoptr(GDataInputStream) reader = NULL; + g_autoptr(SrtProfilingTimer) timer = NULL; + glnx_autofd int source_files_fd = -1; + guint line_number = 0; + + g_return_val_if_fail (error == NULL || *error == NULL, FALSE); + g_return_val_if_fail (mtree != NULL, FALSE); + g_return_val_if_fail (sysroot != NULL, FALSE); + g_return_val_if_fail (sysroot_fd >= 0, FALSE); + + timer = _srt_profiling_start ("Apply %s to %s", mtree, sysroot); + + if (!glnx_openat_rdonly (AT_FDCWD, mtree, TRUE, &mtree_fd, error)) + return FALSE; + + istream = g_unix_input_stream_new (glnx_steal_fd (&mtree_fd), TRUE); + + if (flags & PV_MTREE_APPLY_FLAGS_GZIP) + { + g_autoptr(GInputStream) filter = NULL; + g_autoptr(GZlibDecompressor) decompressor = NULL; + + decompressor = g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP); + filter = g_converter_input_stream_new (istream, G_CONVERTER (decompressor)); + g_clear_object (&istream); + istream = g_object_ref (filter); + } + + reader = g_data_input_stream_new (istream); + g_data_input_stream_set_newline_type (reader, G_DATA_STREAM_NEWLINE_TYPE_LF); + + if (source_files != NULL) + { + if (!glnx_opendirat (AT_FDCWD, source_files, FALSE, &source_files_fd, + error)) + return FALSE; + } + + g_info ("Applying \"%s\" to \"%s\"...", mtree, sysroot); + + while (TRUE) + { + g_autofree gchar *line = NULL; + g_autofree gchar *parent = NULL; + const char *base; + g_autoptr(GError) local_error = NULL; + g_auto(PvMtreeEntry) entry = PV_MTREE_ENTRY_BLANK; + glnx_autofd int parent_fd = -1; + glnx_autofd int fd = -1; + int adjusted_mode; + + line = g_data_input_stream_read_line (reader, NULL, NULL, &local_error); + + if (line == NULL) + { + if (local_error != NULL) + { + g_propagate_prefixed_error (error, g_steal_pointer (&local_error), + "While reading a line from %s: ", + mtree); + return FALSE; + } + else + { + /* End of file, not an error */ + break; + } + } + + g_strstrip (line); + line_number++; + + trace ("line %u: %s", line_number, line); + + if (!pv_mtree_entry_parse (line, &entry, mtree, line_number, error)) + return FALSE; + + if (entry.name == NULL || strcmp (entry.name, ".") == 0) + continue; + + trace ("mtree entry: %s", entry.name); + + parent = g_path_get_dirname (entry.name); + base = glnx_basename (entry.name); + trace ("Creating %s in %s", parent, sysroot); + parent_fd = _srt_resolve_in_sysroot (sysroot_fd, parent, + SRT_RESOLVE_FLAGS_MKDIR_P, + NULL, error); + + if (parent_fd < 0) + return glnx_prefix_error (error, + "Unable to create parent directory for \"%s\" in \"%s\"", + entry.name, sysroot); + + switch (entry.kind) + { + case PV_MTREE_ENTRY_KIND_FILE: + if (entry.size == 0) + { + /* For empty files, we can create it from nothing. */ + fd = TEMP_FAILURE_RETRY (openat (parent_fd, base, + (O_RDWR | O_CLOEXEC | O_NOCTTY + | O_NOFOLLOW | O_CREAT + | O_TRUNC), + 0644)); + + if (fd < 0) + return glnx_throw_errno_prefix (error, + "Unable to open \"%s\" in \"%s\"", + entry.name, sysroot); + } + else if (source_files_fd >= 0) + { + const char *source = entry.contents; + + if (source == NULL) + source = entry.name; + + /* If it already exists, assume it's correct */ + if (glnx_openat_rdonly (parent_fd, base, FALSE, &fd, NULL)) + { + trace ("\"%s\" already exists in \"%s\"", + entry.name, sysroot); + } + /* If we can create a hard link, that's also fine */ + else if (TEMP_FAILURE_RETRY (linkat (source_files_fd, source, + parent_fd, base, 0)) == 0) + { + trace ("Created hard link \"%s\" in \"%s\"", + entry.name, sysroot); + } + /* Or if we can copy it, that's fine too */ + else + { + g_debug ("Could not create hard link \"%s\" from \"%s/%s\" into \"%s\": %s", + entry.name, source_files, source, sysroot, + g_strerror (errno)); + + if (!glnx_file_copy_at (source_files_fd, source, NULL, + parent_fd, base, + GLNX_FILE_COPY_OVERWRITE | GLNX_FILE_COPY_NOCHOWN, + NULL, error)) + return glnx_prefix_error (error, + "Could not create copy \"%s\" from \"%s/%s\" into \"%s\"", + entry.name, source_files, + source, sysroot); + + } + } + + /* For other regular files we just assert that it already exists + * (and is not a symlink). */ + if (fd < 0 + && !glnx_openat_rdonly (parent_fd, base, FALSE, &fd, error)) + return glnx_prefix_error (error, + "Unable to open \"%s\" in \"%s\"", + entry.name, sysroot); + + break; + + case PV_MTREE_ENTRY_KIND_DIR: + /* Create directories on-demand */ + if (!glnx_ensure_dir (parent_fd, base, 0755, error)) + return glnx_prefix_error (error, + "Unable to create directory \"%s\" in \"%s\"", + entry.name, sysroot); + + /* Assert that it is in fact a directory */ + if (!glnx_opendirat (parent_fd, base, FALSE, &fd, error)) + return glnx_prefix_error (error, + "Unable to open directory \"%s\" in \"%s\"", + entry.name, sysroot); + + break; + + case PV_MTREE_ENTRY_KIND_LINK: + { + g_autofree char *target = NULL; + /* Create symlinks on-demand. To be idempotent, don't delete + * an existing symlink. */ + target = glnx_readlinkat_malloc (parent_fd, base, + NULL, NULL); + + if (target == NULL && symlinkat (entry.link, parent_fd, base) != 0) + return glnx_throw_errno_prefix (error, + "Unable to create symlink \"%s\" in \"%s\"", + entry.name, sysroot); + } + break; + + case PV_MTREE_ENTRY_KIND_BLOCK: + case PV_MTREE_ENTRY_KIND_CHAR: + case PV_MTREE_ENTRY_KIND_FIFO: + case PV_MTREE_ENTRY_KIND_SOCKET: + case PV_MTREE_ENTRY_KIND_UNKNOWN: + default: + return glnx_throw (error, + "%s:%u: Special file not supported", + mtree, line_number); + } + + if (entry.kind == PV_MTREE_ENTRY_KIND_DIR + || (entry.mode >= 0 && entry.mode & 0111)) + adjusted_mode = 0755; + else + adjusted_mode = 0644; + + if (fd >= 0 && !glnx_fchmod (fd, adjusted_mode, error)) + { + g_prefix_error (error, + "Unable to set mode of \"%s\" in \"%s\": ", + entry.name, sysroot); + return FALSE; + } + + if (entry.mtime_usec >= 0 && fd >= 0 && entry.kind == PV_MTREE_ENTRY_KIND_FILE) + { + struct timespec times[2] = + { + { .tv_sec = 0, .tv_nsec = UTIME_OMIT }, /* atime */ + { + .tv_sec = entry.mtime_usec / G_TIME_SPAN_SECOND, + .tv_nsec = (entry.mtime_usec % G_TIME_SPAN_SECOND) * 1000 + } /* mtime */ + }; + + if (futimens (fd, times) != 0) + g_warning ("Unable to set mtime of \"%s\" in \"%s\": %s", + entry.name, sysroot, g_strerror (errno)); + } + } + + return TRUE; +} + +/* + * Free the contents of @entry, but not @entry itself. + */ +void +pv_mtree_entry_clear (PvMtreeEntry *entry) +{ + g_clear_pointer (&entry->name, g_free); + g_clear_pointer (&entry->contents, g_free); + g_clear_pointer (&entry->link, g_free); + g_clear_pointer (&entry->sha256, g_free); +} diff --git a/pressure-vessel/mtree.h b/pressure-vessel/mtree.h new file mode 100644 index 000000000..19e9a8e44 --- /dev/null +++ b/pressure-vessel/mtree.h @@ -0,0 +1,85 @@ +/* + * Copyright © 2021 Collabora Ltd. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include <glib.h> + +#include "steam-runtime-tools/glib-backports-internal.h" +#include "libglnx/libglnx.h" + +typedef enum +{ + PV_MTREE_APPLY_FLAGS_GZIP = (1 << 0), + PV_MTREE_APPLY_FLAGS_NONE = 0 +} PvMtreeApplyFlags; + +typedef enum +{ + PV_MTREE_ENTRY_KIND_UNKNOWN = '\0', + PV_MTREE_ENTRY_KIND_BLOCK = 'b', + PV_MTREE_ENTRY_KIND_CHAR = 'c', + PV_MTREE_ENTRY_KIND_DIR = 'd', + PV_MTREE_ENTRY_KIND_FIFO = 'p', + PV_MTREE_ENTRY_KIND_FILE = '-', + PV_MTREE_ENTRY_KIND_LINK = 'l', + PV_MTREE_ENTRY_KIND_SOCKET = 's', +} PvMtreeEntryKind; + +typedef struct _PvMtreeEntry PvMtreeEntry; + +struct _PvMtreeEntry +{ + gchar *name; + gchar *contents; + gchar *link; + gchar *sha256; + goffset size; + GTimeSpan mtime_usec; + int mode; + PvMtreeEntryKind kind; +}; + +#define PV_MTREE_ENTRY_BLANK \ +{ \ + .size = -1, \ + .mtime_usec = -1, \ + .mode = -1, \ +} + +gboolean pv_mtree_entry_parse (const char *line, + PvMtreeEntry *entry, + const char *filename, + guint line_number, + GError **error); + +void pv_mtree_entry_clear (PvMtreeEntry *entry); +G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC (PvMtreeEntry, pv_mtree_entry_clear) + +gboolean pv_mtree_apply (const char *mtree, + const char *sysroot, + int sysroot_fd, + const char *source_files, + PvMtreeApplyFlags flags, + GError **error); diff --git a/tests/pressure-vessel/meson.build b/tests/pressure-vessel/meson.build index 792fd8907..d7ce3c283 100644 --- a/tests/pressure-vessel/meson.build +++ b/tests/pressure-vessel/meson.build @@ -36,6 +36,7 @@ tests = [ 'containers.py', 'invocation.py', 'launcher.py', + 'mtree-apply.py', 'test-locale-gen.sh', 'utils.py', ] @@ -146,6 +147,7 @@ endforeach helpers = [ 'cheap-copy', 'helper', + 'mtree-apply', ] foreach helper : helpers diff --git a/tests/pressure-vessel/mtree-apply.c b/tests/pressure-vessel/mtree-apply.c new file mode 100644 index 000000000..697889d92 --- /dev/null +++ b/tests/pressure-vessel/mtree-apply.c @@ -0,0 +1,91 @@ +/* + * Copyright © 2021 Collabora Ltd. + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "config.h" +#include "subprojects/libglnx/config.h" + +#include <locale.h> +#include <sysexits.h> + +#include "libglnx/libglnx.h" + +#include "steam-runtime-tools/glib-backports-internal.h" +#include "steam-runtime-tools/utils-internal.h" + +#include "mtree.h" +#include "utils.h" + +static GOptionEntry options[] = +{ + { NULL } +}; + +int +main (int argc, + char *argv[]) +{ + glnx_autofd int fd = -1; + g_autoptr(GOptionContext) context = NULL; + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + PvMtreeApplyFlags flags = PV_MTREE_APPLY_FLAGS_NONE; + int ret = EX_USAGE; + + setlocale (LC_ALL, ""); + _srt_setenv_disable_gio_modules (); + + context = g_option_context_new ("MTREE ROOT"); + g_option_context_add_main_entries (context, options, NULL); + + if (!g_option_context_parse (context, &argc, &argv, error)) + goto out; + + if (argc >= 2 && strcmp (argv[1], "--") == 0) + { + argv++; + argc--; + } + + if (argc < 3 || argc > 4) + { + g_printerr ("Usage: %s MTREE ROOT [SOURCE]\n", g_get_prgname ()); + goto out; + } + + ret = EX_UNAVAILABLE; + + if (!glnx_opendirat (AT_FDCWD, argv[2], TRUE, &fd, error)) + goto out; + + if (!pv_mtree_apply (argv[1], argv[2], fd, argv[3], flags, error)) + goto out; + + ret = 0; + +out: + if (local_error != NULL) + g_warning ("%s", local_error->message); + + return ret; +} diff --git a/tests/pressure-vessel/mtree-apply.py b/tests/pressure-vessel/mtree-apply.py new file mode 100755 index 000000000..42fe885b5 --- /dev/null +++ b/tests/pressure-vessel/mtree-apply.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# Copyright 2021 Collabora Ltd. +# +# SPDX-License-Identifier: MIT + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + + +try: + import typing + typing # placate pyflakes +except ImportError: + pass + +from testutils import ( + BaseTest, + test_main, +) + + +class TestMtreeApply(BaseTest): + def setUp(self) -> None: + super().setUp() + os.environ['G_MESSAGES_DEBUG'] = 'all' + self.mtree_apply = os.path.join( + self.G_TEST_BUILDDIR, + 'test-mtree-apply', + ) + + def assert_tree_is_superset( + self, + superset, + subset, + require_hard_links: bool = True, + require_permissions: bool = True, + require_times: bool = True, + ): + for path, dirs, files in os.walk(subset): + equivalent = os.path.join(superset, os.path.relpath(path, subset)) + + for d in dirs: + in_subset = os.path.join(path, d) + in_superset = os.path.join(equivalent, d) + + if not os.path.isdir(in_superset): + raise AssertionError( + '%r should be a directory', in_superset) + + info = os.stat(in_subset) + info2 = os.stat(in_superset) + self.assertEqual(oct(info.st_mode), oct(info2.st_mode)) + + for f in files: + in_subset = os.path.join(path, f) + in_superset = os.path.join(equivalent, f) + + if ( + os.path.islink(in_subset) + or not os.path.exists(in_subset) + ): + target = os.readlink(in_subset) + target2 = os.readlink(in_superset) + self.assertEqual(target, target2) + else: + info = os.stat(in_subset) + info2 = os.stat(in_superset) + + if require_hard_links: + self.assertEqual(info.st_ino, info2.st_ino) + self.assertEqual(info.st_dev, info2.st_dev) + + if require_permissions: + self.assertEqual(oct(info.st_mode), oct(info2.st_mode)) + else: + self.assertEqual( + oct(info.st_mode & ~0o7777), + oct(info2.st_mode & ~0o7777), + ) + + self.assertEqual(info.st_size, info2.st_size) + + if require_times: + self.assertEqual( + int(info.st_mtime), + int(info2.st_mtime), + ) + + def assert_tree_is_same( + self, + left, + right, + require_hard_links: bool = True, + require_permissions: bool = True, + require_times: bool = True, + ): + self.assert_tree_is_superset( + left, right, + require_hard_links=require_hard_links, + require_permissions=require_permissions, + require_times=require_times, + ) + self.assert_tree_is_superset( + right, + left, + require_hard_links=require_hard_links, + require_permissions=require_permissions, + require_times=require_times, + ) + + def test_empty(self) -> None: + content = b'' + + with tempfile.NamedTemporaryFile( + ) as source, tempfile.TemporaryDirectory( + ) as expected, tempfile.TemporaryDirectory( + ) as dest: + source.write(content) + source.flush() + + subprocess.run( + [ + self.mtree_apply, + source.name, + dest, + ], + check=True, + stdout=2, + ) + self.assert_tree_is_same( + dest, + expected, + require_hard_links=False, + require_permissions=False, + require_times=False, + ) + + def test_populate(self) -> None: + content = b'''\ + . type=dir + ./foo/bar/\302\251 type=dir + ./sym/link type=link link=/dev/null + ./create type=file size=0 + ./make-executable type=file mode=755 + ./make-non-executable type=file mode=644 time=1597415889 + ''' + + with tempfile.NamedTemporaryFile( + ) as source, tempfile.TemporaryDirectory( + ) as expected, tempfile.TemporaryDirectory( + ) as dest: + source.write(content) + source.flush() + + os.umask(0o077) + bar = (Path(expected) / 'foo' / 'bar') + bar.mkdir(parents=True) + non_ascii_filename = str(bar).encode('ascii') + b'/\302\251' + os.mkdir(non_ascii_filename) + (Path(expected) / 'sym').mkdir() + (Path(expected) / 'sym' / 'link').symlink_to('/dev/null') + + for filename in 'make-executable', 'make-non-executable': + for d in Path(expected), Path(dest): + with open(str(d / filename), 'w') as writer: + writer.write('#!/bin/sh\n') + + with open(str(Path(expected) / 'create'), 'w') as writer: + pass + + # Paths that are not explicitly created get their permissions + # from umask + (Path(expected)).chmod(0o755) + os.chmod(non_ascii_filename, 0o755) + (Path(expected) / 'create').chmod(0o644) + (Path(expected) / 'make-executable').chmod(0o755) + (Path(expected) / 'make-non-executable').chmod(0o644) + os.utime( + str(Path(expected) / 'make-non-executable'), + times=(1597415889, 1597415889), + ) + + subprocess.run( + [ + self.mtree_apply, + source.name, + dest, + ], + check=True, + stdout=2, + ) + subprocess.run( + [ + 'find', + dest, + '-ls', + ], + check=True, + stdout=2, + ) + self.assert_tree_is_same( + dest, + expected, + require_hard_links=False, + require_permissions=True, + require_times=False, + ) + + def test_populate_copy(self) -> None: + content = b'''\ +# Content-addressed storage indexed by a truncated sha256 +./make-executable type=file mode=755 contents=a8/076d3d28d21e02012b20eaf7dbf754 +./make-non-executable type=file mode=644 time=1597415889 +''' + + with tempfile.NamedTemporaryFile( + ) as source, tempfile.TemporaryDirectory( + ) as reference, tempfile.TemporaryDirectory( + ) as expected, tempfile.TemporaryDirectory( + ) as dest: + source.write(content) + source.flush() + + os.umask(0o077) + + for filename in 'make-executable', 'make-non-executable': + with open(str(Path(expected) / filename), 'w') as writer: + writer.write('#!/bin/sh\n') + + # Avoid Path.link_to() which wasn't in Python 3.5 + os.link( + str(Path(expected) / 'make-non-executable'), + str(Path(reference) / 'make-non-executable'), + ) + (Path(reference) / 'a8').mkdir() + os.link( + str(Path(expected) / 'make-executable'), + str( + Path(reference) / 'a8' + / '076d3d28d21e02012b20eaf7dbf754' + ), + ) + + subprocess.run( + [ + self.mtree_apply, + source.name, + dest, + reference, + ], + check=True, + stdout=2, + ) + + subprocess.run( + [ + 'find', + dest, + '-ls', + ], + check=True, + stdout=2, + ) + self.assert_tree_is_same( + dest, + expected, + require_hard_links=True, + require_permissions=True, + require_times=True, + ) + + info = (Path(dest) / 'make-executable').stat() + self.assertEqual(info.st_mode & 0o7777, 0o755) + + info = (Path(dest) / 'make-non-executable').stat() + self.assertEqual(info.st_mode & 0o7777, 0o644) + self.assertEqual(info.st_mtime, 1597415889) + + def tearDown(self) -> None: + super().tearDown() + + +if __name__ == '__main__': + assert sys.version_info >= (3, 5), \ + 'Python 3.5+ is required (configure with -Dpython=python3.5 ' \ + 'if necessary)' + + test_main() + +# vi: set sw=4 sts=4 et: diff --git a/tests/pressure-vessel/utils.c b/tests/pressure-vessel/utils.c index 202431ff8..0856c8e24 100644 --- a/tests/pressure-vessel/utils.c +++ b/tests/pressure-vessel/utils.c @@ -34,6 +34,7 @@ #include "libglnx/libglnx.h" #include "tests/test-utils.h" +#include "mtree.h" #include "utils.h" typedef struct @@ -344,6 +345,125 @@ test_get_path_after (Fixture *f, } } +static void +test_mtree_entry_parse (Fixture *f, + gconstpointer context) +{ + static const struct + { + const char *line; + const char *name; + PvMtreeEntry expected; + gboolean error; + const char *link; + const char *sha256; + } tests[] = + { + { "#mtree", + NULL, + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_UNKNOWN }, + }, + { "", + NULL, + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_UNKNOWN }, + }, + { ". type=dir", + ".", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_DIR }, + }, + { "./foo type=file sha256=ffff mode=0640 size=42 time=1597415889.5", + "./foo", + { .size = 42, + .mtime_usec = 1597415889 * G_TIME_SPAN_SECOND + (G_TIME_SPAN_SECOND / 2), + .mode = 0640, + .kind = PV_MTREE_ENTRY_KIND_FILE }, + .sha256 = "ffff", + }, + { "./foo type=file sha256digest=ffff mode=4755", + "./foo", + { .size = -1, .mtime_usec = -1, .mode = 04755, + .kind = PV_MTREE_ENTRY_KIND_FILE }, + .sha256 = "ffff", + }, + { "./foo type=file sha256=ffff sha256digest=ffff", + "./foo", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_FILE }, + .sha256 = "ffff", + }, + { "./symlink type=link link=/dev/null", + "./symlink", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_LINK }, + .link = "/dev/null", + }, + { "./silly-name/\\001\\123\\n\\r type=link link=\\\"\\\\\\b", + "./silly-name/\001\123\n\r", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_LINK }, + .link = "\"\\\b", + }, + { ("./ignore cksum=123 device=456 contents=./ignore flags=123 gid=123 " + "gname=users ignore=1 inode=123 md5=ffff md5digest=ffff nlink=1 " + "nochange=1 optional=1 resdevice=123 " + "ripemd160digest=ffff rmd160=ffff rmd160digest=ffff " + "sha1=ffff sha1digest=ffff " + "sha384=ffff sha384digest=ffff " + "sha512=ffff sha512digest=ffff " + "uid=0 uname=root type=dir"), + "./ignore", + { .size = -1, .mtime_usec = -1, .mode = -1, + .kind = PV_MTREE_ENTRY_KIND_DIR }, + }, + { "./foo type=file sha256=ffff sha256digest=eeee", .error = TRUE }, + { "./foo type=file mode=1a", .error = TRUE }, + { "/set type=dir", .error = TRUE }, + { "../escape type=dir", .error = TRUE }, + { "relative type=dir", .error = TRUE }, + { "./foo link", .error = TRUE }, + { "./foo type=bar", .error = TRUE }, + { "./continuation type=dir \\", .error = TRUE }, + { "./alert type=link link=\\a", .error = TRUE }, + { "./hex type=link link=\\x12", .error = TRUE }, + { "./symlink type=file link=/dev/null", .error = TRUE }, + { "./symlink type=link", .error = TRUE }, + { " ", .error = TRUE }, + }; + gsize i; + + for (i = 0; i < G_N_ELEMENTS (tests); i++) + { + const char *line = tests[i].line; + PvMtreeEntry expected = tests[i].expected; + g_auto(PvMtreeEntry) got = PV_MTREE_ENTRY_BLANK; + g_autoptr(GError) error = NULL; + + g_test_message ("%s", line); + pv_mtree_entry_parse (line, &got, "test.mtree", 1, &error); + + if (tests[i].error) + { + g_assert_nonnull (error); + g_test_message ("-> %s", error->message); + } + else + { + g_assert_no_error (error); + g_test_message ("-> OK"); + g_assert_cmpstr (got.name, ==, tests[i].name); + g_assert_cmpstr (got.link, ==, tests[i].link); + g_assert_cmpstr (got.sha256, ==, tests[i].sha256); + g_assert_cmpint (got.size, ==, expected.size); + g_assert_cmpint (got.mtime_usec, ==, expected.mtime_usec); + g_assert_cmpint (got.mode, ==, expected.mode); + g_assert_cmpint (got.kind, ==, expected.kind); + } + } +} + static void test_search_path_append (Fixture *f, gconstpointer context) @@ -389,6 +509,8 @@ main (int argc, g_test_add ("/envp-cmp", Fixture, NULL, setup, test_envp_cmp, teardown); g_test_add ("/get-path-after", Fixture, NULL, setup, test_get_path_after, teardown); + g_test_add ("/mtree-entry-parse", Fixture, NULL, + setup, test_mtree_entry_parse, teardown); g_test_add ("/search-path-append", Fixture, NULL, setup, test_search_path_append, teardown); -- GitLab